rust-kernel

//! System calls via the `syscall` / `sysret` instructions.
//!
//! ABI (like Linux x86_64): number in rax, arguments in rdi, rsi, rdx (r10,
//! r8 for 4th/5th), result in rax; negative results are `-errno`. The
//! `syscall` instruction itself clobbers rcx (return address) and r11 (saved
//! RFLAGS); every other register is preserved for the caller.
//!
//! Numbers are Linux's where Linux has the same call, so `strace` output and
//! man pages mostly apply. Paths are passed as pointer + length (Linux uses
//! NUL-terminated C strings); relative paths start at the process's cwd.
//!
//! | nr  | name        | arguments                           | note |
//! |-----|-------------|-------------------------------------|------|
//! |   0 | read        | fd, buf, len                        | blocks: console until Enter, pipe until data/EOF |
//! |   1 | write       | fd, buf, len                        | |
//! |   2 | open        | path, path_len, flags               | O_CREAT 0o100, O_TRUNC 0o1000; directories too (for getdents) |
//! |   3 | close       | fd                                  | |
//! |   4 | stat        | path, path_len, *Stat               | our own small `Stat {kind, size, mode}`; follows symlinks |
//! |   5 | fstat       | fd, *Stat                           | |
//! |   8 | lseek       | fd, offset, whence                  | SEEK_SET/CUR/END = 0/1/2 |
//! |  16 | ioctl       | fd, request, arg                    | console: TTY_SET_RAW (ours); /dev/fb0: FB_GET_INFO, FB_FLUSH, FB_FILL, FB_SET_CONSOLE (below) |
//! |  22 | pipe        | *[u32; 2]                           | [read end, write end] |
//! |  24 | sched_yield |                                     | |
//! |  35 | sleep_ms    | milliseconds                        | Linux's nr 35 is nanosleep(timespec); ours is simpler |
//! |  39 | getpid      |                                     | |
//! |  60 | exit        | code                                | |
//! |  61 | wait        | pid, flags                          | returns the exit code (139 if killed); WNOHANG = 1 -> EAGAIN if running. Linux wait4 is fancier |
//! |  63 | uname       | *utsname                            | Linux's `struct utsname` (6 x 65-byte strings) |
//! |  79 | getcwd      | buf, len                            | returns the length (no NUL) |
//! |  80 | chdir       | path, path_len                      | |
//! |  83 | mkdir       | path, path_len                      | |
//! |  84 | rmdir       | path, path_len                      | only empty directories |
//! |  87 | unlink      | path, path_len                      | files and symlinks (not what a symlink points to) |
//! |  89 | readlink    | path, path_len, buf, len            | a symlink's target; returns its length (cut to `len`, no NUL) |
//! |  41 | socket      | domain, type, protocol              | AF_INET (2) only; SOCK_STREAM (1) = TCP, SOCK_DGRAM (2) = UDP, SOCK_DGRAM + IPPROTO_ICMP (1) = ping socket |
//! |  42 | connect     | fd, *sockaddr_in, len               | TCP: handshake, blocks (10 s timeout); UDP/ICMP: set default destination |
//! |  43 | accept      | fd, *sockaddr_in or 0, *u32 len or 0 | blocks for a connection (SO_RCVTIMEO applies); returns the new fd |
//! |  44 | sendto      | fd, buf, len, flags, *sockaddr_in or 0 | Linux has a 6th arg (addrlen); we only take 5, the address is always 16 bytes |
//! |  45 | recvfrom    | fd, buf, len, flags, *sockaddr_in or 0 | blocks (SO_RCVTIMEO -> EAGAIN); 0 = TCP peer closed. No addrlen pointer |
//! |  49 | bind        | fd, *sockaddr_in, len               | sets the local port (address ignored: we have one) |
//! |  50 | listen      | fd, backlog                         | TCP; backlog 1..4 |
//! |  54 | setsockopt  | fd, level, name, *value, len        | only SOL_SOCKET (1) / SO_RCVTIMEO (20) with a `struct timeval` |
//! | 217 | getdents64  | fd, buf, len                        | Linux `struct linux_dirent64` records |
//! | 228 | clock_gettime | clock, *timespec                  | CLOCK_REALTIME/MONOTONIC both = time since boot (we have no RTC driver) |
//! | 318 | getrandom   | buf, len, flags                     | random bytes (virtio-rng); up to 64 KiB per call; flags ignored |
//! | 500 | spawn       | path, path_len, argv, argv_len, *SpawnOptions | ours: start a program, returns its pid (see process.rs for why not fork/exec) |
//! | 501 | mounts      | buf, len                            | ours: "path fstype\n" per mount (Linux: read /proc/mounts) |
//! | 502 | sysinfo     | *SysInfo                            | ours: CPUs, threads, memory, per-CPU busy time (Linux: /proc/stat etc.) |
//! | 503 | resolve     | name, name_len, *[[u8; 4]], max     | ours: DNS lookup (A records), returns the count. Linux does DNS in libc |
//! | 504 | netinfo     | *NetInfo                            | ours: MAC, IP, gateway, DNS, packet counters (Linux: netlink/ioctls) |
//!
//! spawn's `argv` is all arguments separated by NUL bytes ("ls\0-l\0/bin");
//! empty means just [program name]. `SpawnOptions` (or a null pointer for
//! the defaults) picks the child's stdin/stdout/stderr from the caller's fds
//! (-1 = same as mine) and flags (SPAWN_TRACE = print its syscalls).
//!
//! The framebuffer ioctls (on an fd of /dev/fb0; numbers in Linux's 'F'
//! range, but the structs are our own -- Linux's fb_var_screeninfo has 40
//! fields):
//!
//!   FB_GET_INFO     0x4600  *FbInfo {width, height, stride_bytes, bpp} (u32s)
//!   FB_FLUSH        0x46f0  *[x, y, w, h] or 0 = what `write` changed: show it
//!   FB_FILL         0x46f1  *[x, y, w, h, 0x00RRGGBB]: fill (not flushed)
//!   FB_SET_CONSOLE  0x46f2  *[x, y, w, h] or 0 = full screen: move the text
//!                           console there; returns columns << 16 | rows
//!
//! What happens on `syscall` (the CPU does steps 1-3 in one instruction):
//!   1. rcx = user RIP (next instruction), r11 = user RFLAGS
//!   2. RFLAGS &= !SFMASK  (we mask IF -> interrupts off)
//!   3. CS/SS = kernel selectors from STAR, RIP = LSTAR (`syscall_entry`), CPL = 0
//!   ...but RSP is still the *user* stack! The CPU doesn't switch stacks for
//!   `syscall` (unlike interrupts, which use TSS.RSP0). So `syscall_entry`
//!   must do it first: stash the user RSP, load this thread's kernel stack,
//!   save registers, re-enable interrupts and call `syscall_dispatch` in Rust.
//!   `sysretq` reverses it: RIP = rcx, RFLAGS = r11, CPL = 3.
//!
//! **Several CPUs.** The "kernel stack of the running thread" differs per
//! CPU, but at the entry we have no free register to compute which CPU we
//! are on. The trick every x86_64 kernel uses: the GS segment base. Each CPU
//! keeps a pointer to its own `smp::PerCpu` block in the KERNEL_GS_BASE MSR;
//! `swapgs` swaps that with the (user's) GS base, so for a few instructions
//! `gs:[0]` is *this CPU's* kernel stack and `gs:[8]` a scratch slot. We
//! swap back right after switching stacks, so the thread can be moved to
//! another CPU later without any GS confusion.
//!
//! Why not `int 0x80`? It also works (and is simpler: the CPU switches stacks
//! via TSS.RSP0 for us), but `syscall` is what Linux and every modern x86_64
//! OS use, and it's faster.

use crate::errno::Errno;
use crate::fs::file::{InodeFile, Stat};
use crate::fs::{self, InodeKind};
use crate::{memory, process, thread};
use alloc::{string::String, sync::Arc, vec::Vec};
use core::arch::global_asm;
use core::sync::atomic::Ordering;

pub const SYS_READ: u64 = 0;
pub const SYS_WRITE: u64 = 1;
pub const SYS_OPEN: u64 = 2;
pub const SYS_CLOSE: u64 = 3;
pub const SYS_STAT: u64 = 4;
pub const SYS_FSTAT: u64 = 5;
pub const SYS_LSEEK: u64 = 8;
pub const SYS_IOCTL: u64 = 16;
pub const SYS_PIPE: u64 = 22;
pub const SYS_SCHED_YIELD: u64 = 24;
pub const SYS_SLEEP_MS: u64 = 35;
pub const SYS_GETPID: u64 = 39;
pub const SYS_EXIT: u64 = 60;
pub const SYS_WAIT: u64 = 61;
pub const SYS_UNAME: u64 = 63;
pub const SYS_GETCWD: u64 = 79;
pub const SYS_CHDIR: u64 = 80;
pub const SYS_MKDIR: u64 = 83;
pub const SYS_RMDIR: u64 = 84;
pub const SYS_UNLINK: u64 = 87;
pub const SYS_READLINK: u64 = 89;
pub const SYS_SOCKET: u64 = 41;
pub const SYS_CONNECT: u64 = 42;
pub const SYS_ACCEPT: u64 = 43;
pub const SYS_SENDTO: u64 = 44;
pub const SYS_RECVFROM: u64 = 45;
pub const SYS_BIND: u64 = 49;
pub const SYS_LISTEN: u64 = 50;
pub const SYS_SETSOCKOPT: u64 = 54;
pub const SYS_GETDENTS64: u64 = 217;
pub const SYS_GETRANDOM: u64 = 318;
pub const SYS_CLOCK_GETTIME: u64 = 228;
pub const SYS_SPAWN: u64 = 500;
pub const SYS_MOUNTS: u64 = 501;
pub const SYS_SYSINFO: u64 = 502;
pub const SYS_RESOLVE: u64 = 503;
pub const SYS_NETINFO: u64 = 504;

// Socket constants (Linux's values).
pub const AF_INET: u64 = 2;
pub const SOCK_STREAM: u64 = 1;
pub const SOCK_DGRAM: u64 = 2;
pub const IPPROTO_ICMP: u64 = 1;
pub const SOL_SOCKET: u64 = 1;
pub const SO_RCVTIMEO: u64 = 20;

pub const O_CREAT: u64 = 0o100;
pub const O_TRUNC: u64 = 0o1000;
pub const O_APPEND: u64 = 0o2000;
/// ioctl request (ours): arg 1 = raw mode, 0 = cooked; returns the old mode.
pub const TTY_SET_RAW: u64 = 1;
pub const SPAWN_TRACE: u64 = 1;
/// Framebuffer ioctls (see the table at the top).
pub const FB_GET_INFO: u64 = 0x4600;
pub const FB_FLUSH: u64 = 0x46f0;
pub const FB_FILL: u64 = 0x46f1;
pub const FB_SET_CONSOLE: u64 = 0x46f2;

/// Name of a syscall number (for tracing).
pub fn name(nr: u64) -> &'static str {
    match nr {
        SYS_READ => "read",
        SYS_WRITE => "write",
        SYS_OPEN => "open",
        SYS_CLOSE => "close",
        SYS_STAT => "stat",
        SYS_FSTAT => "fstat",
        SYS_LSEEK => "lseek",
        SYS_IOCTL => "ioctl",
        SYS_PIPE => "pipe",
        SYS_SCHED_YIELD => "sched_yield",
        SYS_SLEEP_MS => "sleep_ms",
        SYS_GETPID => "getpid",
        SYS_EXIT => "exit",
        SYS_WAIT => "wait",
        SYS_UNAME => "uname",
        SYS_GETCWD => "getcwd",
        SYS_CHDIR => "chdir",
        SYS_MKDIR => "mkdir",
        SYS_RMDIR => "rmdir",
        SYS_UNLINK => "unlink",
        SYS_READLINK => "readlink",
        SYS_GETDENTS64 => "getdents64",
        SYS_GETRANDOM => "getrandom",
        SYS_SPAWN => "spawn",
        SYS_MOUNTS => "mounts",
        SYS_SYSINFO => "sysinfo",
        SYS_SOCKET => "socket",
        SYS_CONNECT => "connect",
        SYS_ACCEPT => "accept",
        SYS_SENDTO => "sendto",
        SYS_RECVFROM => "recvfrom",
        SYS_BIND => "bind",
        SYS_LISTEN => "listen",
        SYS_SETSOCKOPT => "setsockopt",
        SYS_CLOCK_GETTIME => "clock_gettime",
        SYS_RESOLVE => "resolve",
        SYS_NETINFO => "netinfo",
        _ => "?",
    }
}

/// spawn's options, as laid out in user memory.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct SpawnOptions {
    pub stdin: i64, // caller's fd to use as the child's fd 0; -1 = caller's fd 0
    pub stdout: i64,
    pub stderr: i64,
    pub flags: u64,
}

/// Called by the scheduler when switching threads: the kernel stack that
/// `syscall` on this CPU should use from now on.
pub fn set_kernel_stack(stack_top: u64) {
    crate::smp::this_cpu().syscall_kernel_rsp.store(stack_top, Ordering::Relaxed);
}

/// Program the MSRs that make `syscall` work (on the calling CPU; every CPU
/// runs this once).
pub fn init() {
    use x86_64::VirtAddr;
    use x86_64::registers::model_specific::{Efer, EferFlags, KernelGsBase, LStar, SFMask, Star};
    use x86_64::registers::rflags::RFlags;

    let s = crate::gdt::selectors();
    unsafe {
        // EFER.SCE: enable the syscall/sysret instructions at all.
        Efer::update(|flags| flags.insert(EferFlags::SYSTEM_CALL_EXTENSIONS));
    }
    // STAR: which segments syscall/sysret load (see gdt.rs for the layout rule).
    Star::write(s.user_code, s.user_data, s.kernel_code, s.kernel_data)
        .expect("GDT layout doesn't fit syscall/sysret");
    // LSTAR: where `syscall` jumps to.
    LStar::write(VirtAddr::new(syscall_entry as *const () as u64));
    // SFMASK: RFLAGS bits to clear on entry. IF: no interrupts until we're on
    // the kernel stack. DF: Rust expects the direction flag clear. TF: no
    // single-stepping into the kernel.
    SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::DIRECTION_FLAG | RFlags::TRAP_FLAG);
    // KERNEL_GS_BASE: this CPU's PerCpu block, for `swapgs` in the entry.
    KernelGsBase::write(VirtAddr::from_ptr(crate::smp::this_cpu()));
}

unsafe extern "C" {
    fn syscall_entry();
}

global_asm!(
    ".global syscall_entry",
    "syscall_entry:",
    // Interrupts are off (SFMASK). Switch to this thread's kernel stack,
    // found in this CPU's PerCpu block (gs:[0]; gs:[8] is scratch).
    "    swapgs",
    "    mov gs:[8], rsp",
    "    mov rsp, gs:[0]",
    "    push qword ptr gs:[8]", // user rsp
    "    swapgs",                // GS back to normal right away
    "    push rcx",                                // user rip
    "    push r11",                                // user rflags
    // Preserve the argument registers for the user (Rust may clobber them).
    // rbx, rbp, r12-r15 are callee-saved: the Rust code keeps them intact.
    "    push rdi",
    "    push rsi",
    "    push rdx",
    "    push r8",
    "    push r9",
    "    push r10",
    "    sub rsp, 8", // 10 pushes total -> stack 16-byte aligned for the call
    // syscall_dispatch(nr, a1, a2, a3, a4, a5) in the C calling convention:
    // shuffle rax, rdi, rsi, rdx, r10, r8 -> rdi, rsi, rdx, rcx, r8, r9.
    "    mov r9, r8",
    "    mov r8, r10",
    "    mov rcx, rdx",
    "    mov rdx, rsi",
    "    mov rsi, rdi",
    "    mov rdi, rax",
    "    sti",  // we're on our own stack now: preemption is fine
    "    call {dispatch}", // result in rax
    "    cli",  // no interrupts while we're back on the user stack in ring 0
    "    add rsp, 8",
    "    pop r10",
    "    pop r9",
    "    pop r8",
    "    pop rdx",
    "    pop rsi",
    "    pop rdi",
    "    pop r11",
    "    pop rcx",
    "    pop rsp",
    // (rcx came from the `syscall` itself, so it's a canonical address;
    // sysret with a non-canonical rcx would fault *in ring 0* -- a classic bug.)
    "    sysretq",
    dispatch = sym syscall_dispatch,
);

/// Rust side of every system call. Returns the value for rax.
extern "C" fn syscall_dispatch(nr: u64, a1: u64, a2: u64, a3: u64, a4: u64, a5: u64) -> i64 {
    let process = thread::current_process();
    let trace = process.as_ref().is_some_and(|p| p.trace);
    if trace && nr == SYS_EXIT {
        crate::kprintln!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64);
    }
    drop(process); // don't hold an Arc while blocking/exiting
    let result = match nr {
        SYS_READ => sys_read(a1, a2, a3),
        SYS_WRITE => sys_write(a1, a2, a3),
        SYS_OPEN => sys_open(a1, a2, a3),
        SYS_CLOSE => current().and_then(|p| p.close(a1)).map(|_| 0),
        SYS_STAT => sys_stat(a1, a2, a3),
        SYS_FSTAT => sys_fstat(a1, a2),
        SYS_LSEEK => current().and_then(|p| p.file(a1)).and_then(|f| f.seek(a2 as i64, a3)),
        SYS_IOCTL => sys_ioctl(a1, a2, a3),
        SYS_PIPE => sys_pipe(a1),
        SYS_SCHED_YIELD => {
            thread::yield_now();
            Ok(0)
        }
        SYS_SLEEP_MS => {
            // The PIT fires ~18.2 times a second: 1 tick = ~55 ms. Round up.
            thread::sleep_ticks(a1.div_ceil(55));
            Ok(0)
        }
        SYS_GETPID => current().map(|p| p.pid),
        SYS_EXIT => process::exit_current(process::ExitStatus::Exited(a1 as i64)),
        SYS_WAIT => current().and_then(|p| process::wait_child(p.pid, a1, a2)).map(|s| s.code() as u64),
        SYS_UNAME => sys_uname(a1),
        SYS_GETCWD => sys_getcwd(a1, a2),
        SYS_CHDIR => sys_chdir(a1, a2),
        SYS_MKDIR => user_path(a1, a2).and_then(|p| fs::create(&p, InodeKind::Directory)).map(|_| 0),
        SYS_RMDIR => user_path(a1, a2).and_then(|p| fs::remove(&p, true)).map(|_| 0),
        SYS_UNLINK => user_path(a1, a2).and_then(|p| fs::remove(&p, false)).map(|_| 0),
        SYS_READLINK => sys_readlink(a1, a2, a3, a4),
        SYS_GETDENTS64 => sys_getdents64(a1, a2, a3),
        SYS_SPAWN => sys_spawn(a1, a2, a3, a4, a5),
        SYS_MOUNTS => sys_mounts(a1, a2),
        SYS_SYSINFO => sys_sysinfo(a1),
        SYS_SOCKET => sys_socket(a1, a2, a3),
        SYS_CONNECT => with_socket(a1, |s| s.connect(read_sockaddr(a2)?)).map(|_| 0),
        SYS_ACCEPT => sys_accept(a1, a2, a3),
        SYS_SENDTO => sys_sendto(a1, a2, a3, a5),
        SYS_RECVFROM => sys_recvfrom(a1, a2, a3, a5),
        SYS_BIND => with_socket(a1, |s| s.bind(read_sockaddr(a2)?.1)).map(|_| 0),
        SYS_LISTEN => with_socket(a1, |s| s.listen(a2 as usize)).map(|_| 0),
        SYS_SETSOCKOPT => sys_setsockopt(a1, a2, a3, a4, a5),
        SYS_CLOCK_GETTIME => sys_clock_gettime(a2),
        SYS_RESOLVE => sys_resolve(a1, a2, a3, a4),
        SYS_NETINFO => copy_to_user(a1, &crate::net::info()).map(|_| 0),
        SYS_GETRANDOM => sys_getrandom(a1, a2),
        _ => Err(Errno::ENOSYS),
    };
    let ret = match result {
        Ok(value) => value as i64,
        Err(errno) => -(errno as i64),
    };
    if trace {
        let pid = current().map(|p| p.pid).unwrap_or(0);
        crate::kprintln!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret));
    }
    ret
}

/// One syscall in strace style: `open("/mnt/disk0/readme.txt", 0) = 3`.
/// Strings and buffers are read back from user memory (after the call, so
/// `read` shows what it got), shortened to 40 bytes.
fn trace_line(nr: u64, a: [u64; 4], ret: i64) -> String {
    use alloc::format;
    let text = |ptr: u64, len: u64| -> String {
        let Ok(bytes) = user_slice(ptr, len.min(40)) else { return String::from("<bad pointer>") };
        let mut s = String::from("\"");
        for &b in bytes {
            match b {
                b'\n' => s.push_str("\\n"),
                b'"' => s.push_str("\\\""),
                0x20..=0x7e => s.push(b as char),
                _ => s.push_str(&format!("\\x{:02x}", b)),
            }
        }
        s.push('"');
        if len > 40 {
            s.push_str("...");
        }
        s
    };
    let args = match nr {
        SYS_READ => format!("{}, {}, {}", a[0], if ret > 0 { text(a[1], ret as u64) } else { String::from("\"\"") }, a[2]),
        SYS_WRITE => format!("{}, {}, {}", a[0], text(a[1], a[2]), a[2]),
        SYS_OPEN => format!("{}, {:#o}", text(a[0], a[1]), a[2]),
        SYS_STAT => format!("{}, {:#x}", text(a[0], a[1]), a[2]),
        SYS_READLINK => format!("{}, {:#x}, {}", text(a[0], a[1]), a[2], a[3]),
        SYS_CHDIR | SYS_MKDIR | SYS_RMDIR | SYS_UNLINK => text(a[0], a[1]),
        SYS_SPAWN => format!("{}, {}", text(a[0], a[1]), text(a[2], a[3])),
        SYS_SCHED_YIELD | SYS_GETPID => String::new(),
        SYS_CLOSE | SYS_SLEEP_MS | SYS_EXIT => format!("{}", a[0] as i64),
        SYS_PIPE | SYS_UNAME => format!("{:#x}", a[0]),
        SYS_FSTAT | SYS_WAIT | SYS_GETCWD | SYS_MOUNTS => format!("{}, {:#x}", a[0], a[1]),
        _ => format!("{}, {:#x}, {:#x}", a[0], a[1], a[2]),
    };
    match Errno::from_ret(ret) {
        Some(e) if ret < 0 => format!("{}({}) = {} {:?}", name(nr), args, ret, e),
        _ => format!("{}({}) = {}", name(nr), args, ret),
    }
}

fn current() -> Result<Arc<process::Process>, Errno> {
    thread::current_process().ok_or(Errno::EINVAL)
}

/// Turn a user pointer + length into a slice, after checking that the user
/// may really access that memory. Never trust pointers from user space!
pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> {
    if len == 0 {
        return Ok(&[]); // nothing to access, so any pointer is fine (like Linux)
    }
    if !memory::user_range_ok(ptr, len, false) {
        return Err(Errno::EFAULT);
    }
    Ok(unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) })
}

/// Like `user_slice`, but the user must be allowed to *write* the memory.
pub fn user_slice_mut(ptr: u64, len: u64) -> Result<&'static mut [u8], Errno> {
    if len == 0 {
        return Ok(&mut []);
    }
    if !memory::user_range_ok(ptr, len, true) {
        return Err(Errno::EFAULT);
    }
    Ok(unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len as usize) })
}

fn sys_read(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?; // clone the Arc: no lock held while we block
    let buf = user_slice_mut(buf, len)?;
    file.read(buf).map(|n| n as u64)
}

fn sys_write(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    let buf = user_slice(buf, len)?;
    file.write(buf).map(|n| n as u64)
}

/// Read a path argument and make it absolute (relative to the cwd).
fn user_path(ptr: u64, len: u64) -> Result<String, Errno> {
    if len > 4096 {
        return Err(Errno::ENAMETOOLONG);
    }
    let path = core::str::from_utf8(user_slice(ptr, len)?).map_err(|_| Errno::EINVAL)?;
    current()?.resolve(path)
}

/// Copy a plain-data struct out to user memory.
fn copy_to_user<T: Copy>(ptr: u64, value: &T) -> Result<(), Errno> {
    let size = core::mem::size_of::<T>();
    let dst = user_slice_mut(ptr, size as u64)?;
    let src = unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size) };
    dst.copy_from_slice(src);
    Ok(())
}

/// Copy `N` u32s in from user memory (ioctl arguments).
fn read_user_u32s<const N: usize>(ptr: u64) -> Result<[u32; N], Errno> {
    let src = user_slice(ptr, 4 * N as u64)?;
    let mut out = [0u32; N];
    for (i, word) in out.iter_mut().enumerate() {
        *word = u32::from_le_bytes(src[4 * i..4 * i + 4].try_into().unwrap());
    }
    Ok(out)
}

fn sys_open(path_ptr: u64, path_len: u64, flags: u64) -> Result<u64, Errno> {
    let path = user_path(path_ptr, path_len)?;
    let inode = match fs::lookup(&path) {
        Err(Errno::ENOENT) if flags & O_CREAT != 0 => fs::create(&path, InodeKind::File)?,
        other => other?,
    };
    // Device files (/dev/...) come with their own File.
    if let Some(device) = inode.open_device() {
        return current()?.add_file(device);
    }
    if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File {
        inode.truncate()?;
    }
    let file = if flags & O_APPEND != 0 { InodeFile::open_append(inode)? } else { InodeFile::open(inode)? };
    current()?.add_file(Arc::new(file))
}

fn sys_stat(path_ptr: u64, path_len: u64, stat_ptr: u64) -> Result<u64, Errno> {
    let inode = fs::lookup(&user_path(path_ptr, path_len)?)?;
    copy_to_user(stat_ptr, &Stat::of(&*inode))?;
    Ok(0)
}

fn sys_fstat(fd: u64, stat_ptr: u64) -> Result<u64, Errno> {
    let stat = current()?.file(fd)?.stat()?;
    copy_to_user(stat_ptr, &stat)?;
    Ok(0)
}

fn sys_pipe(fds_ptr: u64) -> Result<u64, Errno> {
    user_slice_mut(fds_ptr, 8)?; // check before creating anything
    let process = current()?;
    let (reader, writer) = fs::pipe::new();
    let read_fd = process.add_file(Arc::new(reader))?;
    let write_fd = match process.add_file(Arc::new(writer)) {
        Ok(fd) => fd,
        Err(e) => {
            process.close(read_fd)?;
            return Err(e);
        }
    };
    copy_to_user(fds_ptr, &[read_fd as u32, write_fd as u32])?;
    Ok(0)
}

fn sys_getcwd(buf: u64, len: u64) -> Result<u64, Errno> {
    let cwd = current()?.cwd.lock().clone();
    if (len as usize) < cwd.len() {
        return Err(Errno::ERANGE);
    }
    user_slice_mut(buf, cwd.len() as u64)?.copy_from_slice(cwd.as_bytes());
    Ok(cwd.len() as u64)
}

fn sys_chdir(path_ptr: u64, path_len: u64) -> Result<u64, Errno> {
    let path = user_path(path_ptr, path_len)?;
    if fs::lookup(&path)?.kind() != InodeKind::Directory {
        return Err(Errno::ENOTDIR);
    }
    *current()?.cwd.lock() = path;
    Ok(0)
}

fn sys_readlink(path_ptr: u64, path_len: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let target = fs::readlink(&user_path(path_ptr, path_len)?)?;
    let n = target.len().min(len as usize);
    user_slice_mut(buf, n as u64)?.copy_from_slice(&target.as_bytes()[..n]);
    Ok(n as u64)
}

fn sys_getdents64(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    file.getdents(user_slice_mut(buf, len)?).map(|n| n as u64)
}

fn sys_spawn(path_ptr: u64, path_len: u64, argv_ptr: u64, argv_len: u64, opts_ptr: u64) -> Result<u64, Errno> {
    let parent = current()?;
    let path = user_path(path_ptr, path_len)?;
    if argv_len as usize > process::MAX_ARGS_SIZE {
        return Err(Errno::E2BIG);
    }
    let blob = core::str::from_utf8(user_slice(argv_ptr, argv_len)?).map_err(|_| Errno::EINVAL)?;
    let mut args: Vec<String> = blob.split('\0').filter(|a| !a.is_empty()).map(String::from).collect();
    if args.is_empty() {
        args.push(String::from(path.rsplit('/').next().unwrap_or("")));
    }
    let opts = if opts_ptr == 0 {
        SpawnOptions { stdin: -1, stdout: -1, stderr: -1, flags: 0 }
    } else {
        let bytes = user_slice(opts_ptr, core::mem::size_of::<SpawnOptions>() as u64)?;
        unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const SpawnOptions) }
    };
    // Child fd n = the caller's fd chosen in the options (-1: the caller's fd n).
    let pick = |n: u64, choice: i64| parent.file(if choice < 0 { n } else { choice as u64 });
    let stdio = [pick(0, opts.stdin)?, pick(1, opts.stdout)?, pick(2, opts.stderr)?];
    let spawn = process::SpawnArgs {
        args,
        stdio,
        cwd: parent.cwd.lock().clone(),
        parent: parent.pid,
        trace: opts.flags & SPAWN_TRACE != 0,
    };
    drop(parent);
    process::spawn_path(&path, spawn).map(|child| child.pid)
}

fn sys_ioctl(fd: u64, request: u64, arg: u64) -> Result<u64, Errno> {
    let process = current()?;
    let file = process.file(fd)?;
    if let Some(fb) = file.as_framebuffer() {
        return fb_ioctl(fb, request, arg);
    }
    if file.stat()?.kind != crate::fs::file::KIND_CONSOLE {
        return Err(Errno::ENOTTY);
    }
    match request {
        TTY_SET_RAW => Ok(crate::tty::set_raw(arg != 0, process.pid) as u64),
        _ => Err(Errno::EINVAL),
    }
}

/// The /dev/fb0 ioctls (table at the top of this file).
fn fb_ioctl(fb: &crate::dev::Framebuffer, request: u64, arg: u64) -> Result<u64, Errno> {
    let info = fb.info;
    match request {
        FB_GET_INFO => {
            copy_to_user(arg, &[info.width, info.height, info.stride * 4, 32u32])?;
            Ok(0)
        }
        FB_FLUSH if arg == 0 => fb.flush().then_some(0).ok_or(Errno::EIO),
        FB_FLUSH => {
            let [x, y, w, h] = read_user_u32s::<4>(arg)?;
            crate::virtio::gpu::flush(x, y, w, h).then_some(0).ok_or(Errno::EIO)
        }
        FB_FILL => {
            let [x, y, w, h, color] = read_user_u32s::<5>(arg)?;
            fb.fill(x, y, w, h, color);
            Ok(0)
        }
        FB_SET_CONSOLE => {
            let [x, y, w, h] = if arg == 0 { [0; 4] } else { read_user_u32s::<4>(arg)? };
            let (cols, rows) = crate::vga_buffer::set_viewport(crate::fbcon::Rect { x, y, w, h }).ok_or(Errno::ENODEV)?;
            Ok((cols as u64) << 16 | rows as u64)
        }
        _ => Err(Errno::EINVAL),
    }
}

/// getrandom(buf, len, flags): Linux 318. We always have "enough entropy"
/// (the host's), so the flags (GRND_NONBLOCK, GRND_RANDOM) change nothing.
fn sys_getrandom(buf: u64, len: u64) -> Result<u64, Errno> {
    let buf = user_slice_mut(buf, len.min(65536))?;
    crate::virtio::rng::fill(buf);
    Ok(buf.len() as u64)
}

/// Linux's `struct utsname`: six NUL-terminated strings of 65 bytes each.
fn sys_uname(buf: u64) -> Result<u64, Errno> {
    let fields = [
        "Rustbelt",                     // sysname (`uname -s`)
        "qemu",                         // nodename (hostname)
        env!("CARGO_PKG_VERSION"),      // release (`uname -r`)
        "stage 13: virtual drivers",    // version
        "x86_64",                       // machine
        "(none)",                       // domainname
    ];
    let out = user_slice_mut(buf, 6 * 65)?;
    out.fill(0);
    for (i, field) in fields.iter().enumerate() {
        out[i * 65..i * 65 + field.len()].copy_from_slice(field.as_bytes());
    }
    Ok(0)
}

/// What `sysinfo` fills in (ours; the user side is in user/ulib.rs).
#[repr(C)]
#[derive(Default)]
pub struct SysInfo {
    pub cpus: u64,
    pub this_cpu: u64,
    pub threads: u64,
    pub processes: u64,
    pub frames_in_use: u64,
    pub uptime_ticks: u64,
    /// Per CPU: local-timer ticks so far, and how many found it busy
    /// (running something other than its idle thread).
    pub ticks: [u64; 16],
    pub busy_ticks: [u64; 16],
}

fn sys_sysinfo(buf: u64) -> Result<u64, Errno> {
    let out = user_slice_mut(buf, core::mem::size_of::<SysInfo>() as u64)?;
    let mut info = SysInfo {
        cpus: crate::smp::cpu_count() as u64,
        this_cpu: crate::smp::cpu_id() as u64,
        threads: thread::thread_count() as u64,
        processes: process::count() as u64,
        frames_in_use: memory::frames_in_use() as u64,
        uptime_ticks: crate::interrupts::ticks(),
        ..Default::default()
    };
    for cpu in 0..info.cpus as usize {
        (info.ticks[cpu], info.busy_ticks[cpu]) = thread::cpu_ticks(cpu);
    }
    let bytes = unsafe {
        core::slice::from_raw_parts(&info as *const SysInfo as *const u8, core::mem::size_of::<SysInfo>())
    };
    out.copy_from_slice(bytes);
    Ok(0)
}

/// The mount table as text, one "path fstype" line per mount.
fn sys_mounts(buf: u64, len: u64) -> Result<u64, Errno> {
    let mut text = String::new();
    for (path, fs_type) in fs::mounts() {
        text.push_str(&path);
        text.push(' ');
        text.push_str(fs_type);
        text.push('\n');
    }
    if (len as usize) < text.len() {
        return Err(Errno::ERANGE);
    }
    user_slice_mut(buf, text.len() as u64)?.copy_from_slice(text.as_bytes());
    Ok(text.len() as u64)
}

// --- Networking (net/socket.rs does the real work) -----------------------------

/// Linux's `struct sockaddr_in`: family, port and address in network byte
/// order (big-endian), 8 bytes of padding.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct SockAddrIn {
    pub family: u16,
    pub port_be: u16,
    pub addr: [u8; 4],
    pub zero: [u8; 8],
}

fn read_sockaddr(ptr: u64) -> Result<crate::net::socket::Addr, Errno> {
    let bytes = user_slice(ptr, 16)?;
    let family = u16::from_ne_bytes([bytes[0], bytes[1]]);
    if family as u64 != AF_INET {
        return Err(Errno::EAFNOSUPPORT);
    }
    Ok(([bytes[4], bytes[5], bytes[6], bytes[7]], u16::from_be_bytes([bytes[2], bytes[3]])))
}

fn write_sockaddr(ptr: u64, (addr, port): crate::net::socket::Addr) -> Result<(), Errno> {
    if ptr == 0 {
        return Ok(()); // caller doesn't want it
    }
    copy_to_user(ptr, &SockAddrIn { family: AF_INET as u16, port_be: port.to_be(), addr, zero: [0; 8] })
}

/// Run `f` on the socket behind `fd` (ENOTSOCK if it's another kind of
/// file). We hold our own Arc to it, so the fd may be closed meanwhile.
fn with_socket<R>(fd: u64, f: impl FnOnce(&crate::net::socket::Socket) -> Result<R, Errno>) -> Result<R, Errno> {
    let file = current()?.file(fd)?;
    f(file.as_socket().ok_or(Errno::ENOTSOCK)?)
}

fn sys_socket(domain: u64, ty: u64, protocol: u64) -> Result<u64, Errno> {
    use crate::net::socket::{Kind, Socket};
    if domain != AF_INET {
        return Err(Errno::EAFNOSUPPORT);
    }
    let kind = match (ty & 0xf, protocol) {
        (SOCK_STREAM, 0 | 6) => Kind::Tcp,
        (SOCK_DGRAM, 0 | 17) => Kind::Udp,
        (SOCK_DGRAM, IPPROTO_ICMP) => Kind::Icmp,
        _ => return Err(Errno::EPROTONOSUPPORT),
    };
    current()?.add_file(Arc::new(Socket::new(kind)?))
}

fn sys_accept(fd: u64, addr_ptr: u64, len_ptr: u64) -> Result<u64, Errno> {
    let (socket, peer) = with_socket(fd, |s| s.accept())?;
    let new_fd = current()?.add_file(Arc::new(socket))?;
    write_sockaddr(addr_ptr, peer)?;
    if len_ptr != 0 {
        copy_to_user(len_ptr, &16u32)?;
    }
    Ok(new_fd)
}

fn sys_sendto(fd: u64, buf: u64, len: u64, addr_ptr: u64) -> Result<u64, Errno> {
    let to = if addr_ptr == 0 { None } else { Some(read_sockaddr(addr_ptr)?) };
    with_socket(fd, |s| s.send(user_slice(buf, len)?, to)).map(|n| n as u64)
}

fn sys_recvfrom(fd: u64, buf: u64, len: u64, addr_ptr: u64) -> Result<u64, Errno> {
    let (n, from) = with_socket(fd, |s| s.recv(user_slice_mut(buf, len)?))?;
    if let Some(from) = from {
        write_sockaddr(addr_ptr, from)?;
    }
    Ok(n as u64)
}

/// Only SO_RCVTIMEO: `struct timeval {i64 seconds, i64 microseconds}`;
/// zero means "block forever", like Linux.
fn sys_setsockopt(fd: u64, level: u64, name: u64, value: u64, len: u64) -> Result<u64, Errno> {
    if level != SOL_SOCKET || name != SO_RCVTIMEO || len < 16 {
        return Err(Errno::EOPNOTSUPP);
    }
    let tv = user_slice(value, 16)?;
    let secs = u64::from_ne_bytes(tv[..8].try_into().unwrap());
    let usecs = u64::from_ne_bytes(tv[8..16].try_into().unwrap());
    let total = secs.saturating_mul(1_000_000).saturating_add(usecs);
    with_socket(fd, |s| Ok(s.set_recv_timeout(if total == 0 { None } else { Some(total) })))?;
    Ok(0)
}

/// `struct timespec {i64 seconds, i64 nanoseconds}` since boot.
fn sys_clock_gettime(ts_ptr: u64) -> Result<u64, Errno> {
    let us = crate::time::now_us();
    copy_to_user(ts_ptr, &[us / 1_000_000, (us % 1_000_000) * 1000])?;
    Ok(0)
}

fn sys_resolve(name_ptr: u64, name_len: u64, out: u64, max: u64) -> Result<u64, Errno> {
    if name_len > 253 {
        return Err(Errno::ENAMETOOLONG);
    }
    let name = core::str::from_utf8(user_slice(name_ptr, name_len)?).map_err(|_| Errno::EINVAL)?;
    let addrs = crate::net::resolve(name, 5_000_000)?;
    let n = addrs.len().min(max as usize);
    let dst = user_slice_mut(out, 4 * n as u64)?;
    for (i, a) in addrs.iter().take(n).enumerate() {
        dst[4 * i..4 * i + 4].copy_from_slice(a);
    }
    Ok(n as u64)
}