rust-kernel

//! System calls via the `syscall` instruction.
//!
//! ABI (like Linux x86_64): number in rax, arguments in rdi, rsi, rdx, r10,
//! r8, r9; result in rax; negative results are `-errno`. The `syscall`
//! instruction itself clobbers rcx (return address) and r11 (saved
//! RFLAGS); every other register is preserved for the caller.
//!
//! Numbers are Linux's where Linux has the same call, so `strace` output and
//! man pages mostly apply. The process calls added in stage 14 (fork,
//! execve, wait4, mmap, signals, dup, pipe2, fcntl, process groups) follow
//! Linux exactly -- C strings, structs, flags -- so a C library can use
//! them as they are, and so do stage 15's (poll/epoll, futex, threads,
//! ttys, nanosleep, readv/writev...). Since stage 15.1 the file calls do
//! too: paths are NUL-terminated C strings (at most PATH_MAX = 4096 bytes
//! with the NUL; a bad pointer is EFAULT, never a crash), stat fills
//! Linux's 144-byte `struct stat`, and the `*at()` versions take a
//! directory fd or AT_FDCWD (-100). Relative paths start at the cwd.
//! `mode` arguments are stored where the filesystem can (stage 16: ramfs,
//! ext2) but never checked -- everyone is root -- and uid/gid are always 0. Every blocking call honours O_NONBLOCK (-> EAGAIN).
//!
//! | nr  | name        | arguments                           | note |
//! |-----|-------------|-------------------------------------|------|
//! |   0 | read        | fd, buf, len                        | blocks: tty until a line (canonical) or a byte (raw), pipe until data/EOF |
//! |   1 | write       | fd, buf, len                        | to a pipe without readers: EPIPE + SIGPIPE; a full pipe blocks |
//! |   2 | open        | path, flags, mode                   | = openat(AT_FDCWD, ...). O_RDONLY/WRONLY/RDWR 0/1/2, O_CREAT 0o100, O_EXCL 0o200, O_NOCTTY 0o400, O_TRUNC 0o1000, O_APPEND, O_NONBLOCK 0o4000, O_DIRECTORY 0o200000, O_NOFOLLOW 0o400000, O_CLOEXEC 0o2000000 |
//! |   3 | close       | fd                                  | |
//! |   4 | stat        | path, *stat                         | Linux `struct stat`; follows symlinks. st_ino is a hash of the path |
//! |   5 | fstat       | fd, *stat                           | |
//! |   6 | lstat       | path, *stat                         | doesn't follow a final symlink |
//! |   7 | poll        | *pollfd, n, timeout_ms              | -1 = forever; pipes, sockets, ttys, files, epoll fds |
//! |   8 | lseek       | fd, offset, whence                  | SEEK_SET/CUR/END = 0/1/2 |
//! |   9 | mmap        | addr, len, prot, flags, fd, offset  | MAP_PRIVATE/SHARED, MAP_ANONYMOUS, MAP_FIXED; files: private only |
//! |  10 | mprotect    | addr, len, prot                     | PROT_READ 1, PROT_WRITE 2, PROT_EXEC 4 |
//! |  11 | munmap      | addr, len                           | |
//! |  12 | brk         | addr                                | returns the (new) end of the heap; brk(0) = current |
//! |  13 | rt_sigaction | sig, *new, *old, 8                 | kernel `struct sigaction`; handlers need SA_RESTORER |
//! |  14 | rt_sigprocmask | how, *set, *old, 8               | SIG_BLOCK 0 / SIG_UNBLOCK 1 / SIG_SETMASK 2 |
//! |  15 | rt_sigreturn |                                    | only from a signal handler's restorer |
//! |  16 | ioctl       | fd, request, arg                    | any fd: FIONBIO; ttys: TCGETS/TCSETS(W/F), TIOCGPGRP/SPGRP, TIOCSCTTY, TIOCNOTTY, TIOCGWINSZ/SWINSZ, TIOCGPTN, TIOCSPTLCK, TTY_SET_RAW 1 (ours); /dev/fb0: FB_* (below) |
//! |  17 | pread64     | fd, buf, len, offset                | regular files; leaves the file position alone |
//! |  18 | pwrite64    | fd, buf, len, offset                | |
//! |  19 | readv       | fd, *iovec, n                       | `struct iovec {base, len}`, up to 1024 |
//! |  20 | writev      | fd, *iovec, n                       | |
//! |  21 | access      | path, mode                          | = faccessat(AT_FDCWD, ...): 0 if it exists (we have no permissions) |
//! |  22 | pipe        | *[u32; 2]                           | [read end, write end] |
//! |  23 | select      | n, *rd, *wr, *ex, *timeval          | fd bitmaps (n <= 1024) |
//! |  24 | sched_yield |                                     | |
//! |  28 | madvise     | addr, len, advice                   | hints accepted; MADV_DONTNEED 4 / MADV_FREE 8 drop private pages (they read back as zeros) |
//! |  32 | dup         | fd                                  | lowest free fd |
//! |  33 | dup2        | old, new                            | closes `new` first if open |
//! |  34 | pause       |                                     | sleeps until a signal; returns EINTR |
//! |  35 | nanosleep   | *timespec, *remaining               | EINTR on a signal (remaining time written back) |
//! |  39 | getpid      |                                     | |
//! |  41 | socket      | domain, type, protocol              | AF_INET (2), stage 18: AF_UNIX (1) STREAM/DGRAM/SEQPACKET (src/net/unix.rs); type may add SOCK_NONBLOCK/SOCK_CLOEXEC; SOCK_STREAM (1) = TCP, SOCK_DGRAM (2) = UDP, SOCK_DGRAM + IPPROTO_ICMP (1) = ping socket |
//! |  42 | connect     | fd, *sockaddr_in, len               | TCP: handshake, blocks (10 s timeout; nonblocking: EINPROGRESS); UDP/ICMP: set default destination. 127.x = loopback |
//! |  43 | accept      | fd, *sockaddr_in or 0, *u32 len or 0 | blocks for a connection (SO_RCVTIMEO applies); returns the new fd. Like Linux, *len is read and updated |
//! |  44 | sendto      | fd, buf, len, flags, *sockaddr_in or 0, addrlen | INET: flags and addrlen are ignored (a sockaddr_in is always 16 bytes), so MSG_NOSIGNAL is too; AF_UNIX (stage 18) honors them |
//! |  45 | recvfrom    | fd, buf, len, flags, *sockaddr_in or 0, *addrlen | blocks (SO_RCVTIMEO -> EAGAIN); 0 = TCP peer closed. The address is cut to *addrlen |
//! |  46 | sendmsg     | fd, *msghdr, flags                  | stage 18: iovecs + name; AF_UNIX: SCM_RIGHTS (<= 253 fds), SCM_CREDENTIALS, MSG_NOSIGNAL/DONTWAIT |
//! |  47 | recvmsg     | fd, *msghdr, flags                  | stage 18: AF_UNIX control data (MSG_CTRUNC, MSG_CMSG_CLOEXEC), MSG_PEEK/WAITALL/TRUNC/DONTWAIT |
//! |  49 | bind        | fd, *sockaddr_in, len               | sets the local port (address ignored: we have one); AF_UNIX: a socket file, an abstract name, or autobind |
//! |  50 | listen      | fd, backlog                         | TCP; backlog 1..4 |
//! |  54 | setsockopt  | fd, level, name, *value, len        | stage 17 (src/syscall_ext.rs): SO_RCVTIMEO is real; SO_REUSEADDR & co. stored; others ENOPROTOOPT |
//! |  56 | clone       | flags, stack, *ptid, *ctid, tls     | fork-like (SIGCHLD [+ CLONE_VFORK]) or a thread: CLONE_VM|FS|FILES|SIGHAND|THREAD|SYSVSEM [+ SETTLS, PARENT_SETTID, CHILD_CLEARTID] |
//! |  57 | fork        |                                     | copy-on-write; parent gets the pid, child 0 |
//! |  58 | vfork       |                                     | fork + the parent waits until the child execs or exits |
//! |  59 | execve      | path, argv, envp (C strings)        | Linux's stack layout: argc, argv, envp, auxv |
//! |  60 | exit        | code                                | ends this *thread* only (exit_group ends the process) |
//! |  61 | wait4       | pid, *status, options, *rusage      | WNOHANG 1, WUNTRACED 2, WCONTINUED 8; *rusage (if not 0) is zero-filled: we don't count usage |
//! |  62 | kill        | pid, sig                            | pid 0 = my group, -1 = all, < -1 = group -pid |
//! |  63 | uname       | *utsname                            | Linux's `struct utsname` (6 x 65-byte strings) |
//! |  72 | fcntl       | fd, cmd, arg                        | F_DUPFD 0, F_GETFD 1, F_SETFD 2, F_GETFL 3 (access mode, O_APPEND, O_NONBLOCK), F_SETFL 4 (O_NONBLOCK, O_APPEND), F_GETLK 5 / F_SETLK 6 / F_SETLKW 7 (record locks, fs/lock.rs), F_DUPFD_CLOEXEC 1030 |
//! |  74 | fsync       | fd                                  | fdatasync 75 too: writes already went to the disk, so just checks the fd |
//! |  76 | truncate    | path, len                           | cut or zero-extend a file; ftruncate 77 = the same on an fd open for writing |
//! |  79 | getcwd      | buf, len                            | like Linux: writes a NUL, returns the length including it |
//! |  80 | chdir       | path                                | |
//! |  81 | fchdir      | fd                                  | a directory opened with open() |
//! |  82 | rename      | old, new                            | = renameat(AT_FDCWD, old, AT_FDCWD, new); see `fs::rename` for the limits |
//! |  83 | mkdir       | path, mode                          | = mkdirat(AT_FDCWD, ...) |
//! |  84 | rmdir       | path                                | only empty directories |
//! |  87 | unlink      | path                                | files and symlinks (not what a symlink points to) |
//! |  89 | readlink    | path, buf, len                      | a symlink's target; returns its length (cut to `len`, no NUL); /proc/self/exe = my program |
//! |  90 | chmod       | path, mode                          | ramfs and ext2 keep it; FAT: EPERM; btrfs: EROFS. fchmod 91 = on an fd |
//! |  95 | umask       | mask                                | returns the old one; open(O_CREAT)/mkdir take these bits away from `mode` |
//! |  97 | getrlimit   | resource, *rlimit                   | {soft, hard}; setrlimit 160 too. RLIMIT_NOFILE is enforced (<= 64) |
//! | 102 | getuid      |                                     | stage 17 (src/syscall_ext.rs): real per-process ids, root (0) by default |
//! | 109 | setpgid     | pid, pgid                           | 0 = myself / a new group |
//! | 110 | getppid     |                                     | 0 if started by the kernel or orphaned |
//! | 111 | getpgrp     |                                     | |
//! | 112 | setsid      |                                     | new session + group, no controlling tty |
//! | 121 | getpgid     | pid                                 | |
//! | 124 | getsid      | pid                                 | |
//! | 127 | rt_sigpending | *set, 8                           | blocked signals that are waiting |
//! | 128 | rt_sigtimedwait | *set, *siginfo, *timespec, 8    | take a signal without a handler; EAGAIN on timeout |
//! | 129 | rt_sigqueueinfo | tgid, sig, *siginfo             | sigqueue(); real-time signals (32..64) queue up. rt_tgsigqueueinfo 297: to one thread |
//! | 130 | rt_sigsuspend | *mask, 8                          | sleep with `mask` until a handler ran; EINTR |
//! | 131 | sigaltstack | *new, *old                          | `stack_t` {sp, flags, size}; SS_DISABLE 2, SS_AUTODISARM; SA_ONSTACK handlers run there |
//! | 157 | prctl       | option, ...                         | PR_SET/GET_NAME 15/16, PDEATHSIG 1/2, DUMPABLE 3/4, NO_NEW_PRIVS 38/39; others EINVAL |
//! | 158 | arch_prctl  | code, addr                          | ARCH_SET_FS 0x1002 / ARCH_GET_FS 0x1003: the TLS pointer |
//! | 186 | gettid      |                                     | thread id (= pid for the first thread) |
//! | 200 | tkill       | tid, sig                            | |
//! | 202 | futex       | *u32, op, val, *timespec, *u32, val3 | FUTEX_WAIT 0, WAKE 1, WAIT_BITSET 9, WAKE_BITSET 10 (+ PRIVATE 128) |
//! | 204 | sched_getaffinity | tid, len, *mask                 | every online CPU; returns 8 (bytes written). sched_setaffinity 203: accepted, not enforced |
//! | 213 | epoll_create | size                               | = epoll_create1(0) |
//! | 217 | getdents64  | fd, buf, len                        | Linux `struct linux_dirent64` records |
//! | 218 | set_tid_address | *u32                            | cleared + futex-woken when this thread exits; returns the tid |
//! | 228 | clock_gettime | clock, *timespec                  | REALTIME 0 (CMOS clock at boot + uptime), MONOTONIC 1 (since boot); stage 17: CPU time 2/3 |
//! | 230 | clock_nanosleep | clock, flags, *timespec, *remaining | REALTIME, MONOTONIC or BOOTTIME 7; TIMER_ABSTIME 1 = sleep until that time |
//! | 231 | exit_group  | code                                | ends every thread of the process |
//! | 232 | epoll_wait  | epfd, *epoll_event, max, timeout_ms | level-triggered; EPOLLONESHOT supported |
//! | 233 | epoll_ctl   | epfd, op, fd, *epoll_event          | ADD 1, DEL 2, MOD 3 |
//! | 234 | tgkill      | tgid, tid, sig                      | a signal for one thread |
//! | 257 | openat      | dirfd, path, flags, mode            | O_NOFOLLOW on a symlink: ELOOP; O_EXCL: EEXIST; O_DIRECTORY on a file: ENOTDIR; a directory for writing: EISDIR |
//! | 258 | mkdirat     | dirfd, path, mode                   | |
//! | 262 | newfstatat  | dirfd, path, *stat, flags           | AT_SYMLINK_NOFOLLOW 0x100, AT_EMPTY_PATH 0x1000 (= fstat(dirfd)) |
//! | 263 | unlinkat    | dirfd, path, flags                  | AT_REMOVEDIR 0x200 = rmdir |
//! | 264 | renameat    | olddirfd, old, newdirfd, new        | renameat2 316 too: flags 0 or RENAME_NOREPLACE (else EINVAL) |
//! | 267 | readlinkat  | dirfd, path, buf, len               | |
//! | 268 | fchmodat    | dirfd, path, mode                   | fchmodat2 452: + flags (AT_SYMLINK_NOFOLLOW on a symlink: EOPNOTSUPP) |
//! | 269 | faccessat   | dirfd, path, mode                   | faccessat2 439 too (its flags are ignored) |
//! | 273 | set_robust_list | *head, 24                       | robust mutexes: marked FUTEX_OWNER_DIED when the thread dies. get_robust_list 274 |
//! | 270 | pselect6    | n, *rd, *wr, *ex, *timespec, *{*mask, 8} | |
//! | 271 | ppoll       | *pollfd, n, *timespec, *mask, 8     | sets the mask while waiting |
//! | 280 | utimensat   | dirfd, path or 0, *[timespec; 2], flags | UTIME_NOW / UTIME_OMIT; FAT ignores times; btrfs: EROFS |
//! | 281 | epoll_pwait | epfd, *events, max, timeout, *mask, 8 | |
//! | 288 | accept4     | fd, *addr, *len, flags              | flags: SOCK_NONBLOCK, SOCK_CLOEXEC |
//! | 291 | epoll_create1 | flags                             | EPOLL_CLOEXEC |
//! | 292 | dup3        | old, new, flags                     | flags: O_CLOEXEC |
//! | 293 | pipe2       | *[u32; 2], flags                    | flags: O_CLOEXEC |
//! | 302 | prlimit64   | pid, resource, *new, *old           | get and/or set another process's limits (0 = me) |
//! | 318 | getrandom   | buf, len, flags                     | random bytes (virtio-rng); up to 64 KiB per call; flags ignored |
//! | 500 | spawn       | path, argv, argv_len, *SpawnOptions | ours: new process running a program (fork+exec in one step, like posix_spawn) |
//! | 501 | mounts      | buf, len                            | ours: "path fstype\n" per mount (Linux: read /proc/mounts) |
//! | 502 | rb_sysinfo  | *SysInfo                            | ours: CPUs, threads, memory, per-CPU busy time (Linux: /proc/stat etc.) |
//! | 503 | resolve     | name, name_len, *[[u8; 4]], max     | ours: DNS lookup (A records), returns the count. Linux does DNS in libc |
//! | 504 | netinfo     | *NetInfo                            | ours: MAC, IP, gateway, DNS, packet counters (Linux: netlink/ioctls) |
//! | 505 | trace_me    | on                                  | ours: print my syscalls from now on (survives execve; `run --trace`) |
//!
//! Stage 17's calls (link/symlink, chown, statfs/statx, sysinfo 99, times,
//! getrusage, clock_getres, socket names/options, timers, ids, priority,
//! host names, flock, preadv/pwritev, sendfile) live in `syscall_ext.rs`,
//! which has its own table; unknown numbers fall through to it.
//!
//! spawn's `argv` is all arguments separated by NUL bytes ("ls\0-l\0/bin");
//! empty means just [program name]. `SpawnOptions` (or a null pointer for
//! the defaults) picks the child's stdin/stdout/stderr from the caller's fds
//! (-1 = same as mine) and flags (SPAWN_TRACE = print its syscalls).
//!
//! The framebuffer ioctls (on an fd of /dev/fb0; numbers in Linux's 'F'
//! range, but the structs are our own -- Linux's fb_var_screeninfo has 40
//! fields):
//!
//!   FB_GET_INFO     0x4600  *FbInfo {width, height, stride_bytes, bpp} (u32s)
//!   FB_FLUSH        0x46f0  *[x, y, w, h] or 0 = what `write` changed: show it
//!   FB_FILL         0x46f1  *[x, y, w, h, 0x00RRGGBB]: fill (not flushed)
//!   FB_SET_CONSOLE  0x46f2  *[x, y, w, h] or 0 = full screen: move the text
//!                           console there; returns columns << 16 | rows
//!
//! What happens on `syscall` (the CPU does steps 1-3 in one instruction):
//!   1. rcx = user RIP (next instruction), r11 = user RFLAGS
//!   2. RFLAGS &= !SFMASK  (we mask IF -> interrupts off)
//!   3. CS/SS = kernel selectors from STAR, RIP = LSTAR (`syscall_entry`), CPL = 0
//!   ...but RSP is still the *user* stack! The CPU doesn't switch stacks for
//!   `syscall` (unlike interrupts, which use TSS.RSP0). So `syscall_entry`
//!   must do it first: stash the user RSP, load this thread's kernel stack,
//!   save all registers as a `TrapFrame` (trap.rs), re-enable interrupts and
//!   call `syscall_dispatch` in Rust. The way back is trap.rs's common exit,
//!   `iretq` (stage 13 used the faster `sysretq`, but that can't restore
//!   rcx/r11, which a signal handler's return needs).
//!
//! **Several CPUs.** The "kernel stack of the running thread" differs per
//! CPU, but at the entry we have no free register to compute which CPU we
//! are on. The trick every x86_64 kernel uses: the GS segment base. Each CPU
//! keeps a pointer to its own `smp::PerCpu` block in the KERNEL_GS_BASE MSR;
//! `swapgs` swaps that with the (user's) GS base, so for a few instructions
//! `gs:[0]` is *this CPU's* kernel stack and `gs:[8]` a scratch slot. We
//! swap back right after switching stacks, so the thread can be moved to
//! another CPU later without any GS confusion.
//!
//! Why not `int 0x80`? It also works (and is simpler: the CPU switches stacks
//! via TSS.RSP0 for us), but `syscall` is what Linux and every modern x86_64
//! OS use, and it's faster.

use crate::errno::Errno;
use crate::fs::file::{File, InodeFile, LinuxStat};
use crate::fs::{self, InodeKind};
use crate::trap::TrapFrame;
use crate::{memory, process, signal, thread};
use alloc::{string::String, sync::Arc, vec::Vec};

/// Print one `--trace` line (and keep a copy if a test asked for one).
macro_rules! trace_out {
    ($($arg:tt)*) => {{
        let line = alloc::format!($($arg)*);
        crate::kprintln!("{}", line);
        trace_keep(&line);
    }};
}

use core::arch::global_asm;
use core::sync::atomic::Ordering;

pub const SYS_READ: u64 = 0;
pub const SYS_WRITE: u64 = 1;
pub const SYS_OPEN: u64 = 2;
pub const SYS_CLOSE: u64 = 3;
pub const SYS_STAT: u64 = 4;
pub const SYS_FSTAT: u64 = 5;
pub const SYS_LSEEK: u64 = 8;
pub const SYS_IOCTL: u64 = 16;
pub const SYS_PIPE: u64 = 22;
pub const SYS_SCHED_YIELD: u64 = 24;
pub const SYS_NANOSLEEP: u64 = 35;
pub const SYS_GETPID: u64 = 39;
pub const SYS_EXIT: u64 = 60;
pub const SYS_UNAME: u64 = 63;
pub const SYS_GETCWD: u64 = 79;
pub const SYS_CHDIR: u64 = 80;
pub const SYS_MKDIR: u64 = 83;
pub const SYS_RMDIR: u64 = 84;
pub const SYS_UNLINK: u64 = 87;
pub const SYS_READLINK: u64 = 89;
pub const SYS_SOCKET: u64 = 41;
pub const SYS_CONNECT: u64 = 42;
pub const SYS_ACCEPT: u64 = 43;
pub const SYS_ACCEPT4: u64 = 288;
pub const SYS_SENDTO: u64 = 44;
pub const SYS_RECVFROM: u64 = 45;
pub const SYS_BIND: u64 = 49;
pub const SYS_LISTEN: u64 = 50;
pub const SYS_SETSOCKOPT: u64 = 54;
pub const SYS_GETDENTS64: u64 = 217;
pub const SYS_GETRANDOM: u64 = 318;
pub const SYS_CLOCK_GETTIME: u64 = 228;
pub const SYS_SPAWN: u64 = 500;
pub const SYS_MOUNTS: u64 = 501;
pub const SYS_RB_SYSINFO: u64 = 502;
pub const SYS_RESOLVE: u64 = 503;
pub const SYS_NETINFO: u64 = 504;
pub const SYS_MMAP: u64 = 9;
pub const SYS_MPROTECT: u64 = 10;
pub const SYS_MUNMAP: u64 = 11;
pub const SYS_BRK: u64 = 12;
pub const SYS_RT_SIGACTION: u64 = 13;
pub const SYS_RT_SIGPROCMASK: u64 = 14;
pub const SYS_RT_SIGRETURN: u64 = 15;
pub const SYS_DUP: u64 = 32;
pub const SYS_DUP2: u64 = 33;
pub const SYS_PAUSE: u64 = 34;
pub const SYS_CLONE: u64 = 56;
pub const SYS_FORK: u64 = 57;
pub const SYS_VFORK: u64 = 58;
pub const SYS_EXECVE: u64 = 59;
pub const SYS_WAIT4: u64 = 61;
pub const SYS_KILL: u64 = 62;
pub const SYS_FCNTL: u64 = 72;
pub const SYS_SETPGID: u64 = 109;
pub const SYS_GETPPID: u64 = 110;
pub const SYS_GETPGRP: u64 = 111;
pub const SYS_GETPGID: u64 = 121;
pub const SYS_EXIT_GROUP: u64 = 231;
pub const SYS_DUP3: u64 = 292;
pub const SYS_PIPE2: u64 = 293;
pub const SYS_TRACE_ME: u64 = 505;
// stage 15
pub const SYS_POLL: u64 = 7;
pub const SYS_PREAD64: u64 = 17;
pub const SYS_PWRITE64: u64 = 18;
pub const SYS_READV: u64 = 19;
pub const SYS_WRITEV: u64 = 20;
pub const SYS_SELECT: u64 = 23;
pub const SYS_SETSID: u64 = 112;
pub const SYS_GETSID: u64 = 124;
pub const SYS_ARCH_PRCTL: u64 = 158;
pub const SYS_GETTID: u64 = 186;
pub const SYS_TKILL: u64 = 200;
pub const SYS_FUTEX: u64 = 202;
pub const SYS_EPOLL_CREATE: u64 = 213;
pub const SYS_SET_TID_ADDRESS: u64 = 218;
pub const SYS_EPOLL_WAIT: u64 = 232;
pub const SYS_EPOLL_CTL: u64 = 233;
pub const SYS_TGKILL: u64 = 234;
pub const SYS_PSELECT6: u64 = 270;
pub const SYS_PPOLL: u64 = 271;
pub const SYS_EPOLL_PWAIT: u64 = 281;
pub const SYS_EPOLL_CREATE1: u64 = 291;
// Stage 15.1: the rest of Linux's path calls, and a few musl needs early.
pub const SYS_LSTAT: u64 = 6;
pub const SYS_ACCESS: u64 = 21;
pub const SYS_RENAME: u64 = 82;
pub const SYS_FCHDIR: u64 = 81;
pub const SYS_GETUID: u64 = 102;
pub const SYS_GETGID: u64 = 104;
pub const SYS_GETEUID: u64 = 107;
pub const SYS_GETEGID: u64 = 108;
pub const SYS_CLOCK_NANOSLEEP: u64 = 230;
pub const SYS_OPENAT: u64 = 257;
pub const SYS_MKDIRAT: u64 = 258;
pub const SYS_NEWFSTATAT: u64 = 262;
pub const SYS_UNLINKAT: u64 = 263;
pub const SYS_RENAMEAT: u64 = 264;
pub const SYS_READLINKAT: u64 = 267;
pub const SYS_FACCESSAT: u64 = 269;
pub const SYS_RENAMEAT2: u64 = 316;
pub const SYS_FACCESSAT2: u64 = 439;
// Stage 16: what a C library (musl) calls besides open/read/write.
pub const SYS_MADVISE: u64 = 28;
pub const SYS_FSYNC: u64 = 74;
pub const SYS_FDATASYNC: u64 = 75;
pub const SYS_TRUNCATE: u64 = 76;
pub const SYS_FTRUNCATE: u64 = 77;
pub const SYS_CHMOD: u64 = 90;
pub const SYS_FCHMOD: u64 = 91;
pub const SYS_UMASK: u64 = 95;
pub const SYS_GETRLIMIT: u64 = 97;
pub const SYS_RT_SIGPENDING: u64 = 127;
pub const SYS_RT_SIGTIMEDWAIT: u64 = 128;
pub const SYS_RT_SIGQUEUEINFO: u64 = 129;
pub const SYS_RT_SIGSUSPEND: u64 = 130;
pub const SYS_SIGALTSTACK: u64 = 131;
pub const SYS_PRCTL: u64 = 157;
pub const SYS_SETRLIMIT: u64 = 160;
pub const SYS_FCHMODAT: u64 = 268;
pub const SYS_SET_ROBUST_LIST: u64 = 273;
pub const SYS_GET_ROBUST_LIST: u64 = 274;
pub const SYS_UTIMENSAT: u64 = 280;
pub const SYS_RT_TGSIGQUEUEINFO: u64 = 297;
pub const SYS_PRLIMIT64: u64 = 302;
pub const SYS_FCHMODAT2: u64 = 452;
pub const SYS_SCHED_SETAFFINITY: u64 = 203;
pub const SYS_SCHED_GETAFFINITY: u64 = 204;

// Socket constants (Linux's values).
pub const AF_INET: u64 = 2;
pub const SOCK_STREAM: u64 = 1;
pub const SOCK_DGRAM: u64 = 2;
pub const IPPROTO_ICMP: u64 = 1;
pub const SOL_SOCKET: u64 = 1;
pub const SO_RCVTIMEO: u64 = 20;

pub const O_ACCMODE: u64 = 3; // O_RDONLY 0, O_WRONLY 1, O_RDWR 2
pub const O_CREAT: u64 = 0o100;
pub const O_EXCL: u64 = 0o200;
pub const O_DIRECTORY: u64 = 0o200000;
pub const O_NOFOLLOW: u64 = 0o400000;
/// The *at() calls: "relative to the cwd" instead of a directory fd, and flags.
pub const AT_FDCWD: i64 = -100;
pub const AT_SYMLINK_NOFOLLOW: u64 = 0x100;
pub const AT_REMOVEDIR: u64 = 0x200;
pub const AT_EMPTY_PATH: u64 = 0x1000;
/// Longest path, counting the NUL (Linux's PATH_MAX).
pub const PATH_MAX: usize = 4096;
pub const O_NOCTTY: u64 = 0o400;
pub const O_TRUNC: u64 = 0o1000;
pub const O_APPEND: u64 = 0o2000;
pub const O_NONBLOCK: u64 = 0o4000;
pub const O_CLOEXEC: u64 = 0o2000000;
/// fcntl commands.
pub const F_DUPFD: u64 = 0;
pub const F_GETFD: u64 = 1;
pub const F_SETFD: u64 = 2;
pub const F_GETFL: u64 = 3;
pub const F_SETFL: u64 = 4;
pub const F_GETLK: u64 = 5;
pub const F_SETLK: u64 = 6;
pub const F_SETLKW: u64 = 7;
pub const F_DUPFD_CLOEXEC: u64 = 1030;
/// struct flock's l_type.
pub const F_RDLCK: i16 = 0;
pub const F_WRLCK: i16 = 1;
pub const F_UNLCK: i16 = 2;
/// utimensat: "set this time to now" / "leave this time alone" (tv_nsec).
pub const UTIME_NOW: i64 = (1 << 30) - 1;
pub const UTIME_OMIT: i64 = (1 << 30) - 2;
/// prctl options we know.
pub const PR_SET_PDEATHSIG: u64 = 1;
pub const PR_GET_PDEATHSIG: u64 = 2;
pub const PR_GET_DUMPABLE: u64 = 3;
pub const PR_SET_DUMPABLE: u64 = 4;
pub const PR_SET_NAME: u64 = 15;
pub const PR_GET_NAME: u64 = 16;
pub const PR_SET_CHILD_SUBREAPER: u64 = 36;
pub const PR_GET_CHILD_SUBREAPER: u64 = 37;
pub const PR_SET_NO_NEW_PRIVS: u64 = 38;
pub const PR_GET_NO_NEW_PRIVS: u64 = 39;
/// madvise advice values.
pub const MADV_DONTNEED: u64 = 4;
pub const MADV_FREE: u64 = 8;
pub const FD_CLOEXEC: u64 = 1;
/// ioctl(FIONBIO, *int): O_NONBLOCK on/off, for any file.
pub const FIONBIO: u64 = 0x5421;
/// mmap flags.
pub const MAP_SHARED: u64 = 0x01;
pub const MAP_PRIVATE: u64 = 0x02;
pub const MAP_FIXED: u64 = 0x10;
pub const MAP_ANONYMOUS: u64 = 0x20;
/// clone flags (Linux's values).
pub const CLONE_VM: u64 = 0x100; // share the address space
pub const CLONE_FS: u64 = 0x200; // share cwd/umask
pub const CLONE_FILES: u64 = 0x400; // share the fd table
pub const CLONE_SIGHAND: u64 = 0x800; // share signal handlers
pub const CLONE_VFORK: u64 = 0x4000; // parent waits until the child execs/exits
pub const CLONE_THREAD: u64 = 0x10000; // same process: a thread
pub const CLONE_SYSVSEM: u64 = 0x40000;
pub const CLONE_SETTLS: u64 = 0x80000; // new thread's FS base = tls argument
pub const CLONE_PARENT_SETTID: u64 = 0x100000; // write the tid to *ptid
pub const CLONE_CHILD_CLEARTID: u64 = 0x200000; // at exit: *ctid = 0 + futex wake
pub const CLONE_DETACHED: u64 = 0x400000;
pub const CLONE_CHILD_SETTID: u64 = 0x1000000; // write the tid to *ctid
/// arch_prctl codes.
pub const ARCH_SET_FS: u64 = 0x1002;
pub const ARCH_GET_FS: u64 = 0x1003;
pub const SPAWN_TRACE: u64 = 1;
/// Framebuffer ioctls (see the table at the top).
pub const FB_GET_INFO: u64 = 0x4600;
pub const FB_FLUSH: u64 = 0x46f0;
pub const FB_FILL: u64 = 0x46f1;
pub const FB_SET_CONSOLE: u64 = 0x46f2;

/// Name of a syscall number (for tracing).
pub fn name(nr: u64) -> &'static str {
    match nr {
        SYS_READ => "read",
        SYS_WRITE => "write",
        SYS_OPEN => "open",
        SYS_CLOSE => "close",
        SYS_STAT => "stat",
        SYS_FSTAT => "fstat",
        SYS_LSTAT => "lstat",
        SYS_ACCESS => "access",
        SYS_RENAME => "rename",
        SYS_FCHDIR => "fchdir",
        SYS_GETUID => "getuid",
        SYS_GETGID => "getgid",
        SYS_GETEUID => "geteuid",
        SYS_GETEGID => "getegid",
        SYS_CLOCK_NANOSLEEP => "clock_nanosleep",
        SYS_OPENAT => "openat",
        SYS_MKDIRAT => "mkdirat",
        SYS_NEWFSTATAT => "newfstatat",
        SYS_UNLINKAT => "unlinkat",
        SYS_RENAMEAT => "renameat",
        SYS_READLINKAT => "readlinkat",
        SYS_FACCESSAT => "faccessat",
        SYS_RENAMEAT2 => "renameat2",
        SYS_FACCESSAT2 => "faccessat2",
        SYS_MADVISE => "madvise",
        SYS_FSYNC => "fsync",
        SYS_FDATASYNC => "fdatasync",
        SYS_TRUNCATE => "truncate",
        SYS_FTRUNCATE => "ftruncate",
        SYS_CHMOD => "chmod",
        SYS_FCHMOD => "fchmod",
        SYS_UMASK => "umask",
        SYS_GETRLIMIT => "getrlimit",
        SYS_RT_SIGPENDING => "rt_sigpending",
        SYS_RT_SIGTIMEDWAIT => "rt_sigtimedwait",
        SYS_RT_SIGQUEUEINFO => "rt_sigqueueinfo",
        SYS_RT_SIGSUSPEND => "rt_sigsuspend",
        SYS_SIGALTSTACK => "sigaltstack",
        SYS_PRCTL => "prctl",
        SYS_SETRLIMIT => "setrlimit",
        SYS_FCHMODAT => "fchmodat",
        SYS_SET_ROBUST_LIST => "set_robust_list",
        SYS_GET_ROBUST_LIST => "get_robust_list",
        SYS_UTIMENSAT => "utimensat",
        SYS_RT_TGSIGQUEUEINFO => "rt_tgsigqueueinfo",
        SYS_PRLIMIT64 => "prlimit64",
        SYS_FCHMODAT2 => "fchmodat2",
        SYS_SCHED_SETAFFINITY => "sched_setaffinity",
        SYS_SCHED_GETAFFINITY => "sched_getaffinity",
        SYS_LSEEK => "lseek",
        SYS_IOCTL => "ioctl",
        SYS_PIPE => "pipe",
        SYS_SCHED_YIELD => "sched_yield",
        SYS_NANOSLEEP => "nanosleep",
        SYS_POLL => "poll",
        SYS_PREAD64 => "pread64",
        SYS_PWRITE64 => "pwrite64",
        SYS_READV => "readv",
        SYS_WRITEV => "writev",
        SYS_SELECT => "select",
        SYS_SETSID => "setsid",
        SYS_GETSID => "getsid",
        SYS_ARCH_PRCTL => "arch_prctl",
        SYS_GETTID => "gettid",
        SYS_TKILL => "tkill",
        SYS_FUTEX => "futex",
        SYS_EPOLL_CREATE => "epoll_create",
        SYS_SET_TID_ADDRESS => "set_tid_address",
        SYS_EPOLL_WAIT => "epoll_wait",
        SYS_EPOLL_CTL => "epoll_ctl",
        SYS_TGKILL => "tgkill",
        SYS_PSELECT6 => "pselect6",
        SYS_PPOLL => "ppoll",
        SYS_EPOLL_PWAIT => "epoll_pwait",
        SYS_EPOLL_CREATE1 => "epoll_create1",
        SYS_GETPID => "getpid",
        SYS_EXIT => "exit",
        SYS_WAIT4 => "wait4",
        SYS_MMAP => "mmap",
        SYS_MPROTECT => "mprotect",
        SYS_MUNMAP => "munmap",
        SYS_BRK => "brk",
        SYS_RT_SIGACTION => "rt_sigaction",
        SYS_RT_SIGPROCMASK => "rt_sigprocmask",
        SYS_RT_SIGRETURN => "rt_sigreturn",
        SYS_DUP => "dup",
        SYS_DUP2 => "dup2",
        SYS_DUP3 => "dup3",
        SYS_PAUSE => "pause",
        SYS_CLONE => "clone",
        SYS_FORK => "fork",
        SYS_VFORK => "vfork",
        SYS_EXECVE => "execve",
        SYS_KILL => "kill",
        SYS_FCNTL => "fcntl",
        SYS_SETPGID => "setpgid",
        SYS_GETPPID => "getppid",
        SYS_GETPGRP => "getpgrp",
        SYS_GETPGID => "getpgid",
        SYS_EXIT_GROUP => "exit_group",
        SYS_PIPE2 => "pipe2",
        SYS_TRACE_ME => "trace_me",
        SYS_UNAME => "uname",
        SYS_GETCWD => "getcwd",
        SYS_CHDIR => "chdir",
        SYS_MKDIR => "mkdir",
        SYS_RMDIR => "rmdir",
        SYS_UNLINK => "unlink",
        SYS_READLINK => "readlink",
        SYS_GETDENTS64 => "getdents64",
        SYS_GETRANDOM => "getrandom",
        SYS_SPAWN => "spawn",
        SYS_MOUNTS => "mounts",
        SYS_RB_SYSINFO => "rb_sysinfo",
        SYS_SOCKET => "socket",
        SYS_CONNECT => "connect",
        SYS_ACCEPT => "accept",
        SYS_ACCEPT4 => "accept4",
        SYS_SENDTO => "sendto",
        SYS_RECVFROM => "recvfrom",
        46 => "sendmsg",
        47 => "recvmsg",
        SYS_BIND => "bind",
        SYS_LISTEN => "listen",
        SYS_SETSOCKOPT => "setsockopt",
        SYS_CLOCK_GETTIME => "clock_gettime",
        SYS_RESOLVE => "resolve",
        SYS_NETINFO => "netinfo",
        _ => crate::syscall_ext::name(nr).unwrap_or("?"),
    }
}

/// spawn's options, as laid out in user memory.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct SpawnOptions {
    pub stdin: i64, // caller's fd to use as the child's fd 0; -1 = caller's fd 0
    pub stdout: i64,
    pub stderr: i64,
    pub flags: u64,
}

/// Called by the scheduler when switching threads: the kernel stack that
/// `syscall` on this CPU should use from now on.
pub fn set_kernel_stack(stack_top: u64) {
    crate::smp::this_cpu().syscall_kernel_rsp.store(stack_top, Ordering::Relaxed);
}

/// Program the MSRs that make `syscall` work (on the calling CPU; every CPU
/// runs this once).
pub fn init() {
    use x86_64::VirtAddr;
    use x86_64::registers::model_specific::{Efer, EferFlags, KernelGsBase, LStar, SFMask, Star};
    use x86_64::registers::rflags::RFlags;

    let s = crate::gdt::selectors();
    unsafe {
        // EFER.SCE: enable the syscall/sysret instructions at all.
        Efer::update(|flags| flags.insert(EferFlags::SYSTEM_CALL_EXTENSIONS));
    }
    // STAR: which segments syscall/sysret load (see gdt.rs for the layout rule).
    Star::write(s.user_code, s.user_data, s.kernel_code, s.kernel_data)
        .expect("GDT layout doesn't fit syscall/sysret");
    // LSTAR: where `syscall` jumps to.
    LStar::write(VirtAddr::new(syscall_entry as *const () as u64));
    // SFMASK: RFLAGS bits to clear on entry. IF: no interrupts until we're on
    // the kernel stack. DF: Rust expects the direction flag clear. TF: no
    // single-stepping into the kernel.
    SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::DIRECTION_FLAG | RFlags::TRAP_FLAG);
    // KERNEL_GS_BASE: this CPU's PerCpu block, for `swapgs` in the entry.
    KernelGsBase::write(VirtAddr::from_ptr(crate::smp::this_cpu()));
}

unsafe extern "C" {
    fn syscall_entry();
}

global_asm!(
    ".global syscall_entry",
    "syscall_entry:",
    // Interrupts are off (SFMASK). Switch to this thread's kernel stack,
    // found in this CPU's PerCpu block (gs:[0]; gs:[8] is scratch).
    "    swapgs",
    "    mov gs:[8], rsp",
    "    mov rsp, gs:[0]",
    // Build a TrapFrame (trap.rs), the same layout an interrupt makes:
    "    push 0",                 // ss (filled in by syscall_dispatch)
    "    push qword ptr gs:[8]",  // user rsp
    "    swapgs",                 // GS back to normal right away
    "    push r11",               // user rflags (saved there by `syscall`)
    "    push 0",                 // cs (filled in by syscall_dispatch)
    "    push rcx",               // user rip (saved there by `syscall`)
    "    push rax",               // `error` slot: the syscall number
    "    push 0x100",             // `vector`: TRAP_SYSCALL
    "    push rax",
    "    push rbx",
    "    push rcx",
    "    push rdx",
    "    push rsi",
    "    push rdi",
    "    push rbp",
    "    push r8",
    "    push r9",
    "    push r10",
    "    push r11",
    "    push r12",
    "    push r13",
    "    push r14",
    "    push r15",
    "    mov rdi, rsp",   // syscall_dispatch(&mut TrapFrame)
    "    sti",            // we're on our own stack now: preemption is fine
    "    call {dispatch}",
    // Back to user mode with `iretq` through the common exit (which
    // restores every register from the frame). Not `sysretq`: fork, execve,
    // signals and rt_sigreturn may have changed *any* register, and
    // sysret can't set rcx and r11.
    "    jmp {ret}",
    dispatch = sym syscall_dispatch,
    ret = sym crate::trap::trap_return,
);

/// Rust side of every system call. Arguments are in the saved registers
/// (rdi, rsi, rdx, r10, r8, r9 = a1..a6), the number in `error` (= rax at
/// entry); the result goes into the saved rax.
extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {
    let sel = crate::gdt::selectors();
    frame.cs = sel.user_code.0 as u64;
    frame.ss = sel.user_data.0 as u64;
    let nr = frame.error;
    let (a1, a2, a3, a4, a5) = (frame.rdi, frame.rsi, frame.rdx, frame.r10, frame.r8);
    let process = thread::current_process();
    let trace = process.as_ref().is_some_and(|p| p.tracing());
    if trace && (nr == SYS_EXIT || nr == SYS_EXIT_GROUP) {
        trace_out!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64);
    }
    drop(process); // don't hold an Arc while blocking/exiting
    // A successful execve throws away the memory its path lived in, so the
    // trace line for it is made before the call.
    let exec_line = (trace && nr == SYS_EXECVE).then(|| trace_line(nr, [a1, a2, a3, a4], 0));
    if nr == SYS_RT_SIGRETURN {
        // Restores *all* registers (rax included) from the signal frame.
        signal::sys_rt_sigreturn(frame);
        signal::deliver(frame); // signals unblocked by the restored mask
        return;
    }
    // Stage 18: AF_UNIX sockets (and sendmsg/recvmsg) first; None = not one.
    let mut unix = crate::net::unix::dispatch(nr, [a1, a2, a3, a4, a5, frame.r9]);
    let result = match nr {
        _ if unix.is_some() => unix.take().unwrap(),
        SYS_READ => sys_read(a1, a2, a3),
        SYS_WRITE => sys_write(a1, a2, a3),
        SYS_OPEN => sys_openat(AT_FDCWD as u64, a1, a2, a3),
        SYS_OPENAT => sys_openat(a1, a2, a3, a4),
        SYS_CLOSE => current().and_then(|p| p.close(a1)).map(|_| 0),
        SYS_STAT => sys_newfstatat(AT_FDCWD as u64, a1, a2, 0),
        SYS_LSTAT => sys_newfstatat(AT_FDCWD as u64, a1, a2, AT_SYMLINK_NOFOLLOW),
        SYS_FSTAT => sys_fstat(a1, a2),
        SYS_NEWFSTATAT => sys_newfstatat(a1, a2, a3, a4),
        SYS_ACCESS => sys_faccessat(AT_FDCWD as u64, a1),
        SYS_FACCESSAT | SYS_FACCESSAT2 => sys_faccessat(a1, a2),
        SYS_LSEEK => current().and_then(|p| p.file(a1)).and_then(|f| f.seek(a2 as i64, a3)),
        SYS_MMAP => current().and_then(|p| sys_mmap(&p, a1, a2, a3, a4, a5, frame.r9)),
        SYS_MPROTECT => current().and_then(|p| sys_mprotect(&p, a1, a2, a3)),
        SYS_MUNMAP => current().and_then(|p| sys_munmap(&p, a1, a2)),
        SYS_BRK => current().map(|p| p.mm().set_brk(a1)),
        SYS_RT_SIGACTION => sys_rt_sigaction(a1, a2, a3, a4),
        SYS_RT_SIGPROCMASK => sys_rt_sigprocmask(a1, a2, a3, a4),
        SYS_IOCTL => sys_ioctl(a1, a2, a3),
        SYS_PIPE => sys_pipe2(a1, 0),
        SYS_PIPE2 => sys_pipe2(a1, a2),
        SYS_SCHED_YIELD => {
            thread::yield_now();
            Ok(0)
        }
        SYS_DUP => current().and_then(|p| p.file(a1).and_then(|f| p.add_file(f))),
        SYS_DUP2 => current().and_then(|p| if a1 == a2 { p.file(a1).map(|_| a1) } else { p.dup_to(a1, a2, false) }),
        SYS_DUP3 => sys_dup3(a1, a2, a3),
        SYS_PAUSE => Err(thread::pause()),
        SYS_NANOSLEEP => sys_nanosleep(a1, a2),
        SYS_GETPID => current().map(|p| p.pid),
        SYS_CLONE => sys_clone(frame, a1, a2, a3, a4, a5),
        SYS_FORK => current().and_then(|p| process::fork(&p, frame, 0, false)),
        SYS_VFORK => current().and_then(|p| process::fork(&p, frame, 0, true)),
        SYS_EXECVE => sys_execve(frame, a1, a2, a3),
        // exit ends only the calling thread (the process ends with its
        // last thread); exit_group ends them all. libc's exit() is the latter.
        SYS_EXIT => process::exit_thread(process::ExitStatus::Exited(a1 as i32 as i64)),
        SYS_EXIT_GROUP => process::exit_group(process::ExitStatus::Exited(a1 as i32 as i64)),
        SYS_WAIT4 => sys_wait4(a1 as i32 as i64, a2, a3, a4),
        SYS_KILL => current().and_then(|p| signal::sys_kill(&p, a1 as i32 as i64, a2)),
        SYS_UNAME => sys_uname(a1),
        SYS_FCNTL => sys_fcntl(a1, a2, a3),
        SYS_GETCWD => sys_getcwd(a1, a2),
        SYS_CHDIR => user_path(a1).and_then(|p| sys_chdir(p)),
        SYS_FCHDIR => sys_fchdir(a1),
        SYS_MKDIR => sys_mkdirat(AT_FDCWD as u64, a1, a2),
        SYS_MKDIRAT => sys_mkdirat(a1, a2, a3),
        SYS_RMDIR => sys_unlinkat(AT_FDCWD as u64, a1, AT_REMOVEDIR),
        SYS_UNLINK => sys_unlinkat(AT_FDCWD as u64, a1, 0),
        SYS_UNLINKAT => sys_unlinkat(a1, a2, a3),
        SYS_RENAME => sys_renameat(AT_FDCWD as u64, a1, AT_FDCWD as u64, a2, 0),
        SYS_RENAMEAT => sys_renameat(a1, a2, a3, a4, 0),
        SYS_RENAMEAT2 => sys_renameat(a1, a2, a3, a4, a5),
        SYS_READLINK => sys_readlinkat(AT_FDCWD as u64, a1, a2, a3),
        SYS_READLINKAT => sys_readlinkat(a1, a2, a3, a4),
        SYS_SETPGID => current().and_then(|p| process::setpgid(&p, a1, a2)).map(|_| 0),
        SYS_GETPPID => current().map(|p| if p.parent() == process::ORPHAN { 0 } else { p.parent() }),
        SYS_GETPGRP => current().map(|p| p.pgid()),
        SYS_GETPGID => current().and_then(|p| if a1 == 0 { Ok(p.pgid()) } else { process::get(a1).map(|t| t.pgid()).ok_or(Errno::ESRCH) }),
        SYS_GETDENTS64 => sys_getdents64(a1, a2, a3),
        SYS_SPAWN => sys_spawn(a1, a2, a3, a4),
        SYS_MOUNTS => sys_mounts(a1, a2),
        SYS_RB_SYSINFO => sys_sysinfo(a1),
        SYS_SOCKET => sys_socket(a1, a2, a3),
        SYS_CONNECT => with_socket(a1, |s| s.connect(read_sockaddr(a2)?)).map(|_| 0),
        SYS_ACCEPT => sys_accept(a1, a2, a3, 0),
        SYS_ACCEPT4 => sys_accept(a1, a2, a3, a4),
        SYS_SENDTO => sys_sendto(a1, a2, a3, a5),
        SYS_RECVFROM => sys_recvfrom(a1, a2, a3, a5, frame.r9),
        SYS_BIND => with_socket(a1, |s| s.bind(read_sockaddr(a2)?.1)).map(|_| 0),
        SYS_LISTEN => with_socket(a1, |s| s.listen(a2 as usize)).map(|_| 0),
        SYS_CLOCK_GETTIME => sys_clock_gettime(a1, a2),
        SYS_CLOCK_NANOSLEEP => sys_clock_nanosleep(a1, a2, a3, a4),
        SYS_RESOLVE => sys_resolve(a1, a2, a3, a4),
        SYS_NETINFO => copy_to_user(a1, &crate::net::info()).map(|_| 0),
        SYS_GETRANDOM => sys_getrandom(a1, a2),
        SYS_TRACE_ME => current().map(|p| p.trace.store(a1 != 0, Ordering::Relaxed)).map(|_| 0),
        // --- stage 15 ---
        SYS_PREAD64 => sys_pread(a1, a2, a3, a4),
        SYS_PWRITE64 => sys_pwrite(a1, a2, a3, a4),
        SYS_READV => sys_readv(a1, a2, a3),
        SYS_WRITEV => sys_writev(a1, a2, a3),
        SYS_POLL => crate::poll::sys_poll_ms(a1, a2, a3 as i32 as i64),
        SYS_PPOLL => crate::poll::sys_ppoll(a1, a2, a3, a4, a5),
        SYS_SELECT => crate::poll::sys_select_tv(a1, a2, a3, a4, a5),
        SYS_PSELECT6 => crate::poll::sys_pselect6(a1, a2, a3, a4, a5, frame.r9),
        SYS_EPOLL_CREATE => if (a1 as i32) <= 0 { Err(Errno::EINVAL) } else { crate::poll::sys_epoll_create1(0) },
        SYS_EPOLL_CREATE1 => crate::poll::sys_epoll_create1(a1),
        SYS_EPOLL_CTL => crate::poll::sys_epoll_ctl(a1, a2, a3, a4),
        SYS_EPOLL_WAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, 0, 8),
        SYS_EPOLL_PWAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, a5, frame.r9),
        SYS_FUTEX => current().and_then(|p| crate::futex::sys_futex(&p, a1, a2, a3, a4, a5, frame.r9)),
        SYS_GETTID => task().map(|t| t.tid()),
        SYS_SET_TID_ADDRESS => task().map(|t| {
            t.clear_child_tid.store(a1, Ordering::Relaxed);
            t.tid()
        }),
        SYS_ARCH_PRCTL => sys_arch_prctl(a1, a2),
        SYS_TGKILL => sys_tgkill(Some(a1), a2, a3),
        SYS_TKILL => sys_tgkill(None, a1, a2),
        SYS_SETSID => current().and_then(|p| process::setsid(&p)),
        SYS_GETSID => current().and_then(|p| if a1 == 0 { Ok(p.sid()) } else { process::get(a1).map(|t| t.sid()).ok_or(Errno::ESRCH) }),
        // --- stage 16 ---
        SYS_MADVISE => sys_madvise(a1, a2, a3),
        SYS_UMASK => current().map(|p| p.umask.swap(a1 as u32 & 0o777, Ordering::Relaxed) as u64),
        SYS_FSYNC | SYS_FDATASYNC => sys_fsync(a1),
        SYS_TRUNCATE => sys_truncate(a1, a2),
        SYS_FTRUNCATE => sys_ftruncate(a1, a2),
        SYS_CHMOD => sys_fchmodat(AT_FDCWD as u64, a1, a2, 0),
        SYS_FCHMOD => sys_fchmod(a1, a2),
        SYS_FCHMODAT => sys_fchmodat(a1, a2, a3, 0),
        SYS_FCHMODAT2 => sys_fchmodat(a1, a2, a3, a4),
        SYS_UTIMENSAT => sys_utimensat(a1, a2, a3, a4),
        SYS_GETRLIMIT => sys_prlimit64(0, a1, 0, a2),
        SYS_SETRLIMIT => sys_prlimit64(0, a1, a2, 0),
        SYS_PRLIMIT64 => sys_prlimit64(a1, a2, a3, a4),
        SYS_PRCTL => sys_prctl(a1, a2, a3, a4, a5),
        SYS_SET_ROBUST_LIST => sys_set_robust_list(a1, a2),
        SYS_GET_ROBUST_LIST => sys_get_robust_list(a1, a2, a3),
        SYS_SIGALTSTACK => sys_sigaltstack(a1, a2, frame.rsp),
        SYS_RT_SIGPENDING => sys_rt_sigpending(a1, a2),
        SYS_RT_SIGSUSPEND => sys_rt_sigsuspend(a1, a2),
        SYS_RT_SIGTIMEDWAIT => sys_rt_sigtimedwait(a1, a2, a3, a4),
        SYS_RT_SIGQUEUEINFO => sys_rt_sigqueueinfo(a1, None, a2, a3),
        SYS_RT_TGSIGQUEUEINFO => sys_rt_sigqueueinfo(a1, Some(a2), a3, a4),
        SYS_SCHED_GETAFFINITY => sys_sched_affinity(a1, a2, a3, false),
        SYS_SCHED_SETAFFINITY => sys_sched_affinity(a1, a2, a3, true),
        _ => crate::syscall_ext::dispatch(nr, [a1, a2, a3, a4, a5, frame.r9]).unwrap_or(Err(Errno::ENOSYS)),
    };
    let ret = match result {
        Ok(value) => value as i64,
        Err(errno) => -(errno as i64),
    };
    // execve: success means "the new program starts with these registers"
    // (rax = 0 there anyway); the frame was replaced, so don't touch it.
    if !(nr == SYS_EXECVE && ret == 0) {
        frame.rax = ret as u64;
    }
    let interrupted = matches!(Errno::from_ret(ret), Some(Errno::ERESTARTSYS | Errno::ERESTARTNOHAND)) && ret < 0;
    if trace && !interrupted {
        let pid = current().map(|p| p.pid).unwrap_or(0);
        match exec_line {
            Some(line) if ret == 0 => trace_out!("[trace {}] {}", pid, line),
            _ => trace_out!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret)),
        }
    }
    // An interrupted call (-512/-514) is traced *after* signal delivery,
    // with what the program really gets (stage 17): EINTR, or "restarted".
    let line = (trace && interrupted).then(|| trace_line(nr, [a1, a2, a3, a4], -(Errno::EINTR as i64)));
    // A signal may have arrived meanwhile (or the call was interrupted by
    // one): act on it before returning to the program.
    let outcome = signal::deliver_outcome(frame);
    if let Some(line) = line {
        let pid = current().map(|p| p.pid).unwrap_or(0);
        match outcome {
            Some(true) => trace_out!("[trace {}] {} = ? (interrupted, restarted)", pid, line.rsplit_once(" = ").map(|l| l.0).unwrap_or(&line)),
            _ => trace_out!("[trace {}] {}", pid, line),
        }
    }
}

/// Stage 19, for tests: collect the `--trace` lines while this is Some
/// (tests/vdso.rs checks that clock_gettime makes no system call).
static TRACE_COPY: spin::Mutex<Option<String>> = spin::Mutex::new(None);

fn trace_keep(line: &str) {
    x86_64::instructions::interrupts::without_interrupts(|| {
        if let Some(copy) = TRACE_COPY.lock().as_mut() {
            copy.push_str(line);
            copy.push('\n');
        }
    });
}

/// Start (true) or stop (false) collecting `--trace` lines; returns what
/// was collected since the last start.
pub fn collect_trace(on: bool) -> String {
    x86_64::instructions::interrupts::without_interrupts(|| {
        let mut copy = TRACE_COPY.lock();
        let old = copy.take().unwrap_or_default();
        if on {
            *copy = Some(String::new());
        }
        old
    })
}

/// One syscall in strace style: `open("/mnt/disk0/readme.txt", 0) = 3`.
/// Strings and buffers are read back from user memory (after the call, so
/// `read` shows what it got), shortened to 40 bytes.
fn trace_line(nr: u64, a: [u64; 4], ret: i64) -> String {
    use alloc::format;
    let text = |ptr: u64, len: u64| -> String {
        let Ok(bytes) = user_slice(ptr, len.min(40)) else { return String::from("<bad pointer>") };
        let mut s = String::from("\"");
        for &b in bytes {
            match b {
                b'\n' => s.push_str("\\n"),
                b'"' => s.push_str("\\\""),
                0x20..=0x7e => s.push(b as char),
                _ => s.push_str(&format!("\\x{:02x}", b)),
            }
        }
        s.push('"');
        if len > 40 {
            s.push_str("...");
        }
        s
    };
    let cstr = |ptr: u64| -> String {
        match user_cstr(ptr, 4096) {
            Ok(bytes) => format!("\"{}\"", String::from_utf8_lossy(&bytes)),
            Err(_) => String::from("<bad pointer>"),
        }
    };
    let args = match nr {
        SYS_READ => format!("{}, {}, {}", a[0], if ret > 0 { text(a[1], ret as u64) } else { String::from("\"\"") }, a[2]),
        SYS_WRITE => format!("{}, {}, {}", a[0], text(a[1], a[2]), a[2]),
        // Paths are C strings (stage 15.1); the *at() calls show the dirfd
        // (-100 = AT_FDCWD) first, like strace.
        SYS_OPEN => format!("{}, {:#o}", cstr(a[0]), a[1]),
        SYS_OPENAT => format!("{}, {}, {:#o}", a[0] as i32, cstr(a[1]), a[2]),
        SYS_STAT | SYS_LSTAT => format!("{}, {:#x}", cstr(a[0]), a[1]),
        SYS_NEWFSTATAT => format!("{}, {}, {:#x}, {:#x}", a[0] as i32, cstr(a[1]), a[2], a[3]),
        SYS_READLINK => format!("{}, {:#x}, {}", cstr(a[0]), a[1], a[2]),
        SYS_CHDIR | SYS_MKDIR | SYS_RMDIR | SYS_UNLINK | SYS_ACCESS => cstr(a[0]),
        SYS_MKDIRAT | SYS_UNLINKAT | SYS_FACCESSAT | SYS_READLINKAT => format!("{}, {}", a[0] as i32, cstr(a[1])),
        SYS_RENAME => format!("{}, {}", cstr(a[0]), cstr(a[1])),
        SYS_SPAWN => format!("{}, {}", cstr(a[0]), text(a[1], a[2])),
        SYS_SCHED_YIELD | SYS_GETPID => String::new(),
        SYS_CLOSE | SYS_EXIT | SYS_EXIT_GROUP | SYS_DUP => format!("{}", a[0] as i64),
        SYS_PIPE | SYS_UNAME => format!("{:#x}", a[0]),
        SYS_FSTAT | SYS_GETCWD | SYS_MOUNTS => format!("{}, {:#x}", a[0], a[1]),
        SYS_WAIT4 => format!("{}, {:#x}, {}", a[0] as i32, a[1], a[2]),
        SYS_KILL => format!("{}, {}", a[0] as i32, a[1]),
        SYS_FORK | SYS_VFORK | SYS_GETPPID | SYS_GETPGRP | SYS_PAUSE => String::new(),
        SYS_EXECVE => format!("{}, ...", cstr(a[0])),
        SYS_MMAP => format!("{:#x}, {}, {}, {:#x}", a[0], a[1], a[2], a[3]),
        _ => format!("{}, {:#x}, {:#x}", a[0], a[1], a[2]),
    };
    match Errno::from_ret(ret) {
        Some(e) if ret < 0 => format!("{}({}) = {} {:?}", trace_name(nr), args, ret, e),
        _ => format!("{}({}) = {}", trace_name(nr), args, ret),
    }
}

/// A syscall's name for the trace; unknown ones by number (`syscall_130`),
/// so a missing call is easy to look up in Linux's table.
pub fn trace_name(nr: u64) -> String {
    match name(nr) {
        "?" => alloc::format!("syscall_{}", nr),
        known => String::from(known),
    }
}

pub(crate) fn current() -> Result<Arc<process::Process>, Errno> {
    thread::current_process().ok_or(Errno::EINVAL)
}

/// The calling thread.
pub(crate) fn task() -> Result<Arc<process::Task>, Errno> {
    thread::current_task().ok_or(Errno::EINVAL)
}

/// Read a `struct timespec {i64 seconds, i64 nanoseconds}` as microseconds
/// (rounded up, so a timeout never ends early).
pub fn read_timespec_us(ptr: u64) -> Result<u64, Errno> {
    let b = user_slice(ptr, 16)?;
    let secs = i64::from_le_bytes(b[..8].try_into().unwrap());
    let nsecs = i64::from_le_bytes(b[8..].try_into().unwrap());
    if secs < 0 || !(0..1_000_000_000).contains(&nsecs) {
        return Err(Errno::EINVAL);
    }
    Ok((secs as u64).saturating_mul(1_000_000).saturating_add((nsecs as u64).div_ceil(1000)))
}

/// Turn a user pointer + length into a slice, after checking that the user
/// may really access that memory. Never trust pointers from user space!
pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> {
    if len == 0 {
        return Ok(&[]); // nothing to access, so any pointer is fine (like Linux)
    }
    if !crate::mm::user_access_ok(ptr, len, false) {
        return Err(Errno::EFAULT);
    }
    Ok(unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) })
}

/// Like `user_slice`, but the user must be allowed to *write* the memory.
pub fn user_slice_mut(ptr: u64, len: u64) -> Result<&'static mut [u8], Errno> {
    if len == 0 {
        return Ok(&mut []);
    }
    if !crate::mm::user_access_ok(ptr, len, true) {
        return Err(Errno::EFAULT);
    }
    Ok(unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len as usize) })
}

fn sys_read(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?; // clone the Arc: no lock held while we block
    let buf = user_slice_mut(buf, len)?;
    file.read(buf).map(|n| n as u64)
}

fn sys_write(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let process = current()?;
    let file = process.file(fd)?;
    let buf = user_slice(buf, len)?;
    let result = file.write(buf).map(|n| n as u64);
    if result == Err(Errno::EPIPE) {
        // Nobody will ever read this: SIGPIPE, which by default ends the
        // writer. That's how `yes | head -1` stops `yes`.
        signal::send(&process, signal::SIGPIPE, signal::SigInfo::from_pid(signal::SIGPIPE, 0));
    }
    result
}

// --- Paths (stage 15.1: Linux's C-string ABI) --------------------------------------
//
// A path argument is a pointer to a NUL-terminated string, like in C (up
// to stage 15 we took a pointer and a length, which no libc does). We copy
// it in page by page with `user_cstr`, so a bad pointer, or a string that
// runs into an unmapped page, is EFAULT rather than a kernel crash, and at
// most PATH_MAX bytes (with the NUL): longer is ENAMETOOLONG.
//
// The *at() variants take a directory fd first: a relative path starts
// there instead of at the cwd (AT_FDCWD = "the cwd", what the old calls
// mean). musl uses them for stat (newfstatat), opendir is open with
// O_DIRECTORY, and so on.

/// The directory a relative path in an *at() call starts from.
pub(crate) fn at_base(dirfd: u64) -> Result<String, Errno> {
    let p = current()?;
    if dirfd as i32 as i64 == AT_FDCWD {
        return Ok(p.cwd.lock().clone());
    }
    let file = p.file(dirfd)?; // EBADF
    match file.path() {
        Some(path) if file.stat()?.kind == crate::fs::file::KIND_DIR => Ok(path),
        _ => Err(Errno::ENOTDIR),
    }
}

/// A path string (already copied in) made absolute, relative to `dirfd`.
pub(crate) fn path_at(dirfd: u64, raw: &[u8]) -> Result<String, Errno> {
    // Our file systems use UTF-8 names (Linux allows any bytes but '/' and NUL).
    let path = core::str::from_utf8(raw).map_err(|_| Errno::EINVAL)?;
    if path.is_empty() {
        return Err(Errno::ENOENT); // like Linux: "" is not a path
    }
    if path.starts_with('/') {
        return fs::normalize("/", path); // absolute: dirfd doesn't matter
    }
    fs::normalize(&at_base(dirfd)?, path)
}

/// Copy in a path argument and make it absolute, relative to `dirfd`.
pub(crate) fn user_path_at(dirfd: u64, ptr: u64) -> Result<String, Errno> {
    path_at(dirfd, &user_cstr(ptr, PATH_MAX - 1)?)
}

/// Copy in a path argument, relative to the cwd.
pub(crate) fn user_path(ptr: u64) -> Result<String, Errno> {
    user_path_at(AT_FDCWD as u64, ptr)
}

/// open / openat(dirfd, path, flags, mode). Everything is readable and
/// writable whatever O_RDONLY/O_WRONLY say, and `mode` is ignored (new
/// files get the file system's default permissions).
fn sys_openat(dirfd: u64, path_ptr: u64, flags: u64, mode: u64) -> Result<u64, Errno> {
    let path = user_path_at(dirfd, path_ptr)?;
    if let (0, Some(fd)) = (flags & O_NOFOLLOW, fs::procfs::fd_number(&path)) {
        return open_proc_fd(fd, flags);
    }
    let found = if flags & O_NOFOLLOW != 0 { fs::lookup_nofollow(&path) } else { fs::lookup(&path) };
    let inode = match found {
        Err(Errno::ENOENT) if flags & O_CREAT != 0 => {
            let new = fs::create(&path, InodeKind::File)?;
            apply_create_mode(&*new, mode);
            new
        }
        Ok(_) if flags & O_CREAT != 0 && flags & O_EXCL != 0 => return Err(Errno::EEXIST),
        Ok(i) if flags & O_NOFOLLOW != 0 && i.kind() == InodeKind::Symlink => return Err(Errno::ELOOP),
        other => other?,
    };
    let is_dir = inode.kind() == InodeKind::Directory;
    if flags & O_DIRECTORY != 0 && !is_dir {
        return Err(Errno::ENOTDIR); // opendir() on a file
    }
    if is_dir && flags & O_ACCMODE != 0 {
        return Err(Errno::EISDIR); // directories open read-only
    }
    // Device files (/dev/...) come with their own File.
    let cloexec = flags & O_CLOEXEC != 0;
    if let Some(device) = inode.open_device()? {
        let p = current()?;
        if flags & O_NONBLOCK != 0 {
            device.set_nonblocking(true);
        }
        if flags & O_NOCTTY == 0 {
            crate::tty::opened(&p, &*device); // maybe our controlling terminal now
        }
        return p.add_file_with(device, 0, cloexec);
    }
    if inode.kind() == InodeKind::Socket {
        return Err(Errno::ENXIO); // sockets are connect()ed to, not opened
    }
    if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File {
        inode.truncate()?;
    }
    let file = InodeFile::open_flags(inode, path, flags)?;
    current()?.add_file_with(Arc::new(file), 0, cloexec)
}

/// open("/proc/self/fd/N") (stage 18), as on Linux: a file with a path is
/// opened again (its own offset, the new flags); a pipe, terminal or
/// device gives the same open file again, like dup(N); sockets can't be
/// opened (ENXIO).
fn open_proc_fd(fd: u64, flags: u64) -> Result<u64, Errno> {
    let p = current()?;
    let file = p.file(fd).map_err(|_| Errno::ENOENT)?;
    let cloexec = flags & O_CLOEXEC != 0;
    if file.as_unix().is_some() || file.as_socket().is_some() {
        return Err(Errno::ENXIO);
    }
    if let (Some(path), Some(inode)) = (file.path(), file.vfs_inode()) {
        let is_dir = inode.kind() == InodeKind::Directory;
        if flags & O_DIRECTORY != 0 && !is_dir {
            return Err(Errno::ENOTDIR);
        }
        if is_dir && flags & O_ACCMODE != 0 {
            return Err(Errno::EISDIR);
        }
        if inode.open_device()?.is_none() {
            if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File {
                inode.truncate()?;
            }
            let file = InodeFile::open_flags(inode, path, flags)?;
            return p.add_file_with(Arc::new(file), 0, cloexec);
        }
    }
    if flags & O_DIRECTORY != 0 {
        return Err(Errno::ENOTDIR);
    }
    p.add_file_with(file, 0, cloexec)
}

/// newfstatat(dirfd, path, *stat, flags): stat, lstat (AT_SYMLINK_NOFOLLOW:
/// a symlink itself, not what it points to) and, with AT_EMPTY_PATH and
/// path "", fstat(dirfd). Writes Linux's `struct stat`.
fn sys_newfstatat(dirfd: u64, path_ptr: u64, stat_ptr: u64, flags: u64) -> Result<u64, Errno> {
    copy_to_user(stat_ptr, &stat_at(dirfd, path_ptr, flags)?)?;
    Ok(0)
}

/// The `struct stat` newfstatat (and statx, stage 17) report.
pub(crate) fn stat_at(dirfd: u64, path_ptr: u64, flags: u64) -> Result<LinuxStat, Errno> {
    let raw = user_cstr(path_ptr, PATH_MAX - 1)?;
    if raw.is_empty() && flags & AT_EMPTY_PATH != 0 && dirfd as i32 as i64 != AT_FDCWD {
        return fd_stat(dirfd);
    }
    let path = if raw.is_empty() && flags & AT_EMPTY_PATH != 0 { at_base(dirfd)? } else { path_at(dirfd, &raw)? };
    let follow = flags & AT_SYMLINK_NOFOLLOW == 0;
    if let (true, Some(fd)) = (follow, fs::procfs::fd_number(&path)) {
        // /proc/self/fd/N: what fd N is, even a pipe or socket (stage 18).
        return fd_stat(fd).map_err(|_| Errno::ENOENT);
    }
    // (The resolved path: st_dev is the file system the file is *on*,
    // even when a symlink on another one led to it.)
    let (inode, real) = fs::lookup_resolved(&path, follow)?;
    Ok(fs::file::linux_stat(&*inode, &real))
}

fn sys_fstat(fd: u64, stat_ptr: u64) -> Result<u64, Errno> {
    copy_to_user(stat_ptr, &fd_stat(fd)?)?;
    Ok(0)
}

/// fstat's `struct stat`: files opened by path report what stat() of the
/// path does. Others (pipes, sockets, consoles): the open file's address
/// as the inode number, unique while it's open.
pub(crate) fn fd_stat(fd: u64) -> Result<LinuxStat, Errno> {
    let file = current()?.file(fd)?;
    if let (Some(inode), Some(path)) = (file.vfs_inode(), file.path()) {
        return Ok(fs::file::linux_stat(&*inode, &path));
    }
    // Device files opened some other way than by path (the console fds
    // the kernel hands init, a pty from /dev/ptmx, /dev/tty): report the
    // /dev node they are, so fstat(fd) and stat("/dev/...") agree. That's
    // how ttyname() finds a terminal's name (stage 19).
    let kind = file.stat()?.kind;
    if kind == fs::file::KIND_DEVICE || kind == fs::file::KIND_CONSOLE {
        if let Some(link) = file.proc_link().filter(|l| l.starts_with("/dev/")) {
            if let Ok((inode, real)) = fs::lookup_resolved(&link, true) {
                return Ok(fs::file::linux_stat(&*inode, &real));
            }
        }
    }
    let ino = match file.path() {
        Some(path) => fs::file::ino_hash(path.as_bytes()),
        None => fs::file::anon_ino(&file),
    };
    Ok(LinuxStat::new(&file.stat()?, ino).with_times(file.times()))
}

/// access / faccessat(dirfd, path, mode): "does it exist?" There are no
/// permissions to check yet (we're root), so existing is enough.
fn sys_faccessat(dirfd: u64, path_ptr: u64) -> Result<u64, Errno> {
    fs::lookup(&user_path_at(dirfd, path_ptr)?)?;
    Ok(0)
}

/// mkdir / mkdirat(dirfd, path, mode); `mode` is ignored.
fn sys_mkdirat(dirfd: u64, path_ptr: u64, mode: u64) -> Result<u64, Errno> {
    let dir = fs::create(&user_path_at(dirfd, path_ptr)?, InodeKind::Directory)?;
    apply_create_mode(&*dir, mode);
    Ok(0)
}

/// open(O_CREAT) / mkdir (stage 16): the new file gets `mode` minus the
/// umask bits, where the filesystem keeps modes (ramfs, ext2). Elsewhere
/// (FAT, read-only btrfs) it's quietly skipped -- the file exists, which
/// is what the caller asked for.
pub(crate) fn apply_create_mode(inode: &dyn fs::Inode, mode: u64) {
    let umask = current().map(|p| p.umask.load(Ordering::Relaxed)).unwrap_or(0o022);
    let _ = inode.set_mode(mode as u32 & 0o7777 & !umask);
    crate::syscall_ext::set_creator(inode);
}

/// unlink / rmdir / unlinkat(dirfd, path, flags): AT_REMOVEDIR = rmdir.
fn sys_unlinkat(dirfd: u64, path_ptr: u64, flags: u64) -> Result<u64, Errno> {
    if flags & !AT_REMOVEDIR != 0 {
        return Err(Errno::EINVAL);
    }
    fs::remove(&user_path_at(dirfd, path_ptr)?, flags & AT_REMOVEDIR != 0)?;
    Ok(0)
}

/// rename / renameat / renameat2. Stage 17: RENAME_NOREPLACE (1) works
/// (EEXIST if the new name is taken); RENAME_EXCHANGE (2) and
/// RENAME_WHITEOUT (4) are EINVAL, as on file systems that lack them.
/// See `fs::rename` for what we can and can't move.
fn sys_renameat(olddir: u64, old_ptr: u64, newdir: u64, new_ptr: u64, flags: u64) -> Result<u64, Errno> {
    const RENAME_NOREPLACE: u64 = 1;
    let old = user_path_at(olddir, old_ptr)?;
    let new = user_path_at(newdir, new_ptr)?;
    if flags & !RENAME_NOREPLACE != 0 {
        return Err(Errno::EINVAL);
    }
    fs::rename(&old, &new, flags & RENAME_NOREPLACE != 0)?;
    Ok(0)
}

fn sys_chdir(path: String) -> Result<u64, Errno> {
    if fs::lookup(&path)?.kind() != InodeKind::Directory {
        return Err(Errno::ENOTDIR);
    }
    *current()?.cwd.lock() = path;
    Ok(0)
}

/// fchdir(fd): cd to a directory we have open.
fn sys_fchdir(fd: u64) -> Result<u64, Errno> {
    sys_chdir(at_base(fd)?)
}

/// readlink / readlinkat(dirfd, path, buf, size): the link's target, *no*
/// NUL added (like Linux); returns its length (cut to `size`).
fn sys_readlinkat(dirfd: u64, path_ptr: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let path = user_path_at(dirfd, path_ptr)?;
    if len as i64 <= 0 {
        return Err(Errno::EINVAL);
    }
    // (/proc/self/exe and /proc/self/fd/N are symlinks in procfs.rs now.)
    let target = {
        let inode = fs::lookup_nofollow(&path)?;
        if inode.kind() != InodeKind::Symlink {
            return Err(Errno::EINVAL); // not a symlink
        }
        inode.readlink()?
    };
    let n = target.len().min(len as usize);
    user_slice_mut(buf, n as u64)?.copy_from_slice(&target.as_bytes()[..n]);
    Ok(n as u64)
}

/// Copy a plain-data struct out to user memory.
pub fn copy_to_user<T: Copy>(ptr: u64, value: &T) -> Result<(), Errno> {
    let size = core::mem::size_of::<T>();
    let dst = user_slice_mut(ptr, size as u64)?;
    let src = unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size) };
    dst.copy_from_slice(src);
    Ok(())
}

/// Copy `N` u32s in from user memory (ioctl arguments).
fn read_user_u32s<const N: usize>(ptr: u64) -> Result<[u32; N], Errno> {
    let src = user_slice(ptr, 4 * N as u64)?;
    let mut out = [0u32; N];
    for (i, word) in out.iter_mut().enumerate() {
        *word = u32::from_le_bytes(src[4 * i..4 * i + 4].try_into().unwrap());
    }
    Ok(out)
}

fn sys_pipe2(fds_ptr: u64, flags: u64) -> Result<u64, Errno> {
    if flags & !(O_CLOEXEC | O_NONBLOCK) != 0 {
        return Err(Errno::EINVAL); // O_DIRECT (packet mode): not supported
    }
    let cloexec = flags & O_CLOEXEC != 0;
    user_slice_mut(fds_ptr, 8)?; // check before creating anything
    let process = current()?;
    let (reader, writer) = fs::pipe::new();
    if flags & O_NONBLOCK != 0 {
        reader.set_nonblocking(true);
        writer.set_nonblocking(true);
    }
    let read_fd = process.add_file_with(Arc::new(reader), 0, cloexec)?;
    let write_fd = match process.add_file_with(Arc::new(writer), 0, cloexec) {
        Ok(fd) => fd,
        Err(e) => {
            process.close(read_fd)?;
            return Err(e);
        }
    };
    copy_to_user(fds_ptr, &[read_fd as u32, write_fd as u32])?;
    Ok(0)
}

fn sys_dup3(old: u64, new: u64, flags: u64) -> Result<u64, Errno> {
    if old == new || flags & !O_CLOEXEC != 0 {
        return Err(Errno::EINVAL);
    }
    current()?.dup_to(old, new, flags & O_CLOEXEC != 0)
}

/// fcntl: the fd flags (close-on-exec), duplication, and the one status
/// flag we have: O_NONBLOCK (it belongs to the open file, so it's shared
/// with dup'ed fds). Everything is opened read/write.
fn sys_fcntl(fd: u64, cmd: u64, arg: u64) -> Result<u64, Errno> {
    let p = current()?;
    let file = p.file(fd)?;
    match cmd {
        F_DUPFD | F_DUPFD_CLOEXEC if arg as usize >= p.fd_limit() => Err(Errno::EINVAL),
        F_DUPFD => p.add_file_with(file, arg as usize, false),
        F_DUPFD_CLOEXEC => p.add_file_with(file, arg as usize, true),
        F_GETFD => p.cloexec(fd).map(|on| if on { FD_CLOEXEC } else { 0 }),
        F_SETFD => p.set_cloexec(fd, arg & FD_CLOEXEC != 0).map(|_| 0),
        F_GETFL => Ok(file.status_flags() | if file.nonblocking() { O_NONBLOCK } else { 0 }),
        F_SETFL => {
            // Only O_NONBLOCK and O_APPEND can change; the access mode can't.
            file.set_nonblocking(arg & O_NONBLOCK != 0);
            file.set_append(arg & O_APPEND != 0);
            Ok(0)
        }
        F_GETLK | F_SETLK | F_SETLKW => sys_fcntl_lock(&p, &*file, cmd, arg),
        _ => Err(Errno::EINVAL),
    }
}

/// getcwd: like Linux, the path with a NUL; returns the length + 1.
fn sys_getcwd(buf: u64, len: u64) -> Result<u64, Errno> {
    let mut cwd = current()?.cwd.lock().clone();
    cwd.push('\0');
    if (len as usize) < cwd.len() {
        return Err(Errno::ERANGE);
    }
    user_slice_mut(buf, cwd.len() as u64)?.copy_from_slice(cwd.as_bytes());
    Ok(cwd.len() as u64)
}

fn sys_getdents64(fd: u64, buf: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    file.getdents(user_slice_mut(buf, len)?).map(|n| n as u64)
}

/// spawn(path, argv blob, blob length, *options): our own call (500).
fn sys_spawn(path_ptr: u64, argv_ptr: u64, argv_len: u64, opts_ptr: u64) -> Result<u64, Errno> {
    let parent = current()?;
    let path = user_path(path_ptr)?;
    if argv_len as usize > process::MAX_ARGS_SIZE {
        return Err(Errno::E2BIG);
    }
    let blob = core::str::from_utf8(user_slice(argv_ptr, argv_len)?).map_err(|_| Errno::EINVAL)?;
    let mut args: Vec<String> = blob.split('\0').filter(|a| !a.is_empty()).map(String::from).collect();
    if args.is_empty() {
        args.push(String::from(path.rsplit('/').next().unwrap_or("")));
    }
    let opts = if opts_ptr == 0 {
        SpawnOptions { stdin: -1, stdout: -1, stderr: -1, flags: 0 }
    } else {
        let bytes = user_slice(opts_ptr, core::mem::size_of::<SpawnOptions>() as u64)?;
        unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const SpawnOptions) }
    };
    // Child fd n = the caller's fd chosen in the options (-1: the caller's fd n).
    let pick = |n: u64, choice: i64| parent.file(if choice < 0 { n } else { choice as u64 });
    let stdio = [pick(0, opts.stdin)?, pick(1, opts.stdout)?, pick(2, opts.stderr)?];
    let spawn = process::SpawnArgs {
        args,
        env: parent.env(),
        stdio,
        cwd: parent.cwd.lock().clone(),
        parent: parent.pid,
        pgid: parent.pgid(), // same process group, like fork
        sid: parent.sid(),   // ...and session
        ctty: parent.ctty.lock().clone(),
        trace: opts.flags & SPAWN_TRACE != 0,
    };
    drop(parent);
    process::spawn_path(&path, spawn).map(|child| child.pid)
}

/// ioctl: FIONBIO for any file; the rest is up to the file (terminals:
/// tty.rs; the framebuffer: below).
fn sys_ioctl(fd: u64, request: u64, arg: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    if let Some(fb) = file.as_framebuffer() {
        return fb_ioctl(fb, request, arg);
    }
    if request == FIONBIO {
        file.set_nonblocking(read_user_u32s::<1>(arg)?[0] != 0);
        return Ok(0);
    }
    file.ioctl(request, arg)
}

/// The /dev/fb0 ioctls (table at the top of this file).
fn fb_ioctl(fb: &crate::dev::Framebuffer, request: u64, arg: u64) -> Result<u64, Errno> {
    let info = fb.info;
    match request {
        FB_GET_INFO => {
            copy_to_user(arg, &[info.width, info.height, info.stride * 4, 32u32])?;
            Ok(0)
        }
        FB_FLUSH if arg == 0 => fb.flush().then_some(0).ok_or(Errno::EIO),
        FB_FLUSH => {
            let [x, y, w, h] = read_user_u32s::<4>(arg)?;
            crate::virtio::gpu::flush(x, y, w, h).then_some(0).ok_or(Errno::EIO)
        }
        FB_FILL => {
            let [x, y, w, h, color] = read_user_u32s::<5>(arg)?;
            fb.fill(x, y, w, h, color);
            Ok(0)
        }
        FB_SET_CONSOLE => {
            let [x, y, w, h] = if arg == 0 { [0; 4] } else { read_user_u32s::<4>(arg)? };
            let (cols, rows) = crate::vga_buffer::set_viewport(crate::fbcon::Rect { x, y, w, h }).ok_or(Errno::ENODEV)?;
            Ok((cols as u64) << 16 | rows as u64)
        }
        _ => Err(Errno::EINVAL),
    }
}

/// getrandom(buf, len, flags): Linux 318. We always have "enough entropy"
/// (the host's), so the flags (GRND_NONBLOCK, GRND_RANDOM) change nothing.
fn sys_getrandom(buf: u64, len: u64) -> Result<u64, Errno> {
    let buf = user_slice_mut(buf, len.min(65536))?;
    crate::virtio::rng::fill(buf);
    Ok(buf.len() as u64)
}

/// Linux's `struct utsname`: six NUL-terminated strings of 65 bytes each.
fn sys_uname(buf: u64) -> Result<u64, Errno> {
    let (host, domain) = (crate::syscall_ext::hostname(), crate::syscall_ext::domainname());
    let fields = [
        "Rustbelt",                     // sysname (`uname -s`)
        host.as_str(),                  // nodename (hostname; sethostname changes it)
        env!("CARGO_PKG_VERSION"),      // release (`uname -r`)
        "stage 22: arm64 bring-up",     // version (`uname -v`)
        "x86_64",                       // machine
        domain.as_str(),                // domainname
    ];
    let out = user_slice_mut(buf, 6 * 65)?;
    out.fill(0);
    for (i, field) in fields.iter().enumerate() {
        out[i * 65..i * 65 + field.len()].copy_from_slice(field.as_bytes());
    }
    Ok(0)
}

/// What `sysinfo` fills in (ours; the user side is in user/ulib.rs).
#[repr(C)]
#[derive(Default)]
pub struct SysInfo {
    pub cpus: u64,
    pub this_cpu: u64,
    pub threads: u64,
    pub processes: u64,
    pub frames_in_use: u64,
    pub uptime_ticks: u64,
    /// Per CPU: local-timer ticks so far, and how many found it busy
    /// (running something other than its idle thread).
    pub ticks: [u64; 16],
    pub busy_ticks: [u64; 16],
}

fn sys_sysinfo(buf: u64) -> Result<u64, Errno> {
    let out = user_slice_mut(buf, core::mem::size_of::<SysInfo>() as u64)?;
    let mut info = SysInfo {
        cpus: crate::smp::cpu_count() as u64,
        this_cpu: crate::smp::cpu_id() as u64,
        threads: thread::thread_count() as u64,
        processes: process::count() as u64,
        frames_in_use: memory::frames_in_use() as u64,
        uptime_ticks: crate::interrupts::ticks(),
        ..Default::default()
    };
    for cpu in 0..info.cpus as usize {
        (info.ticks[cpu], info.busy_ticks[cpu]) = thread::cpu_ticks(cpu);
    }
    let bytes = unsafe {
        core::slice::from_raw_parts(&info as *const SysInfo as *const u8, core::mem::size_of::<SysInfo>())
    };
    out.copy_from_slice(bytes);
    Ok(0)
}

/// The mount table as text, one "path fstype" line per mount.
fn sys_mounts(buf: u64, len: u64) -> Result<u64, Errno> {
    let mut text = String::new();
    for (path, fs_type) in fs::mounts() {
        text.push_str(&path);
        text.push(' ');
        text.push_str(fs_type);
        text.push('\n');
    }
    if (len as usize) < text.len() {
        return Err(Errno::ERANGE);
    }
    user_slice_mut(buf, text.len() as u64)?.copy_from_slice(text.as_bytes());
    Ok(text.len() as u64)
}

// --- Networking (net/socket.rs does the real work) -----------------------------

/// Linux's `struct sockaddr_in`: family, port and address in network byte
/// order (big-endian), 8 bytes of padding.
#[repr(C)]
#[derive(Clone, Copy, Default)]
pub struct SockAddrIn {
    pub family: u16,
    pub port_be: u16,
    pub addr: [u8; 4],
    pub zero: [u8; 8],
}

pub(crate) fn read_sockaddr(ptr: u64) -> Result<crate::net::socket::Addr, Errno> {
    let bytes = user_slice(ptr, 16)?;
    let family = u16::from_ne_bytes([bytes[0], bytes[1]]);
    if family as u64 != AF_INET {
        return Err(Errno::EAFNOSUPPORT);
    }
    Ok(([bytes[4], bytes[5], bytes[6], bytes[7]], u16::from_be_bytes([bytes[2], bytes[3]])))
}

/// Write a peer's address like Linux: `*len_ptr` (a u32) says how much
/// room there is; we write at most that much and set it to the real size
/// (16). No `len_ptr` (only our own older programs do that): all 16 bytes.
pub(crate) fn write_sockaddr(ptr: u64, len_ptr: u64, (addr, port): crate::net::socket::Addr) -> Result<(), Errno> {
    if ptr == 0 {
        return Ok(()); // caller doesn't want it
    }
    let sa = SockAddrIn { family: AF_INET as u16, port_be: port.to_be(), addr, zero: [0; 8] };
    let bytes = unsafe { core::slice::from_raw_parts(&sa as *const SockAddrIn as *const u8, 16) };
    let room = if len_ptr == 0 {
        16
    } else {
        u32::from_le_bytes(user_slice(len_ptr, 4)?.try_into().unwrap()).min(16) as usize
    };
    user_slice_mut(ptr, room as u64)?.copy_from_slice(&bytes[..room]);
    if len_ptr != 0 {
        copy_to_user(len_ptr, &16u32)?;
    }
    Ok(())
}

/// Run `f` on the socket behind `fd` (ENOTSOCK if it's another kind of
/// file). We hold our own Arc to it, so the fd may be closed meanwhile.
pub(crate) fn with_socket<R>(fd: u64, f: impl FnOnce(&crate::net::socket::Socket) -> Result<R, Errno>) -> Result<R, Errno> {
    let file = current()?.file(fd)?;
    f(file.as_socket().ok_or(Errno::ENOTSOCK)?)
}

fn sys_socket(domain: u64, ty: u64, protocol: u64) -> Result<u64, Errno> {
    use crate::net::socket::{Kind, Socket};
    if domain != AF_INET {
        return Err(Errno::EAFNOSUPPORT);
    }
    let kind = match (ty & 0xf, protocol) {
        (SOCK_STREAM, 0 | 6) => Kind::Tcp,
        (SOCK_DGRAM, 0 | 17) => Kind::Udp,
        (SOCK_DGRAM, IPPROTO_ICMP) => Kind::Icmp,
        _ => return Err(Errno::EPROTONOSUPPORT),
    };
    // SOCK_NONBLOCK and SOCK_CLOEXEC have O_NONBLOCK's and O_CLOEXEC's values.
    let socket = Socket::new(kind)?;
    socket.set_nonblocking(ty & O_NONBLOCK != 0);
    current()?.add_file_with(Arc::new(socket), 0, ty & O_CLOEXEC != 0)
}

/// accept(fd, *addr, *len) and accept4(..., flags: SOCK_NONBLOCK|SOCK_CLOEXEC).
fn sys_accept(fd: u64, addr_ptr: u64, len_ptr: u64, flags: u64) -> Result<u64, Errno> {
    let (socket, peer) = with_socket(fd, |s| s.accept())?;
    socket.set_nonblocking(flags & O_NONBLOCK != 0);
    let new_fd = current()?.add_file_with(Arc::new(socket), 0, flags & O_CLOEXEC != 0)?;
    write_sockaddr(addr_ptr, len_ptr, peer)?;
    Ok(new_fd)
}

fn sys_sendto(fd: u64, buf: u64, len: u64, addr_ptr: u64) -> Result<u64, Errno> {
    let to = if addr_ptr == 0 { None } else { Some(read_sockaddr(addr_ptr)?) };
    with_socket(fd, |s| s.send(user_slice(buf, len)?, to)).map(|n| n as u64)
}

fn sys_recvfrom(fd: u64, buf: u64, len: u64, addr_ptr: u64, len_ptr: u64) -> Result<u64, Errno> {
    let (n, from) = with_socket(fd, |s| s.recv(user_slice_mut(buf, len)?))?;
    if let Some(from) = from {
        write_sockaddr(addr_ptr, len_ptr, from)?;
    }
    Ok(n as u64)
}

/// clock_gettime(clock, *timespec): CLOCK_REALTIME (0, wall-clock time
/// since 1970, from the CMOS clock at boot) or the MONOTONIC family (1,
/// 4 RAW, 6 COARSE, 7 BOOTTIME: time since boot). CPU-time clocks: EINVAL.
fn sys_clock_gettime(clock: u64, ts_ptr: u64) -> Result<u64, Errno> {
    // Stage 19: nanoseconds, computed exactly like the vDSO does.
    let ns = match clock {
        0 | 5 => crate::time::realtime_ns(), // REALTIME, REALTIME_COARSE
        1 | 4 | 6 | 7 => crate::time::now_ns(),
        // Stage 17: CPU time used so far (process / this thread).
        2 | 3 => crate::syscall_ext::cpu_clock_us(clock)? * 1000,
        _ => return Err(Errno::EINVAL),
    };
    copy_to_user(ts_ptr, &[ns / 1_000_000_000, ns % 1_000_000_000])?;
    Ok(0)
}

fn sys_resolve(name_ptr: u64, name_len: u64, out: u64, max: u64) -> Result<u64, Errno> {
    if name_len > 253 {
        return Err(Errno::ENAMETOOLONG);
    }
    let name = core::str::from_utf8(user_slice(name_ptr, name_len)?).map_err(|_| Errno::EINVAL)?;
    let addrs = crate::net::resolve(name, 5_000_000)?;
    let n = addrs.len().min(max as usize);
    let dst = user_slice_mut(out, 4 * n as u64)?;
    for (i, a) in addrs.iter().take(n).enumerate() {
        dst[4 * i..4 * i + 4].copy_from_slice(a);
    }
    Ok(n as u64)
}

// --- Processes, memory, signals (stage 14) --------------------------------------

/// A NUL-terminated string from user memory (at most `max` bytes).
pub fn user_cstr(ptr: u64, max: usize) -> Result<Vec<u8>, Errno> {
    let mut out = Vec::new();
    // Page by page: the string may end right before an unmapped page.
    let mut addr = ptr;
    loop {
        let chunk = (4096 - addr % 4096).min((max + 1 - out.len()) as u64);
        let bytes = user_slice(addr, chunk)?;
        if let Some(end) = bytes.iter().position(|&b| b == 0) {
            out.extend_from_slice(&bytes[..end]);
            return Ok(out);
        }
        out.extend_from_slice(bytes);
        if out.len() > max {
            return Err(Errno::ENAMETOOLONG);
        }
        addr += chunk;
    }
}

/// A NULL-terminated array of C strings (execve's argv and envp).
fn user_cstr_array(ptr: u64) -> Result<Vec<Vec<u8>>, Errno> {
    let mut out = Vec::new();
    if ptr == 0 {
        return Ok(out); // Linux allows NULL here (an empty list)
    }
    let mut total = 0;
    for i in 0.. {
        let p = u64::from_le_bytes(user_slice(ptr + 8 * i, 8)?.try_into().unwrap());
        if p == 0 {
            break;
        }
        let s = user_cstr(p, process::MAX_ARGS_SIZE)?;
        total += s.len() + 1;
        if total > process::MAX_ARGS_SIZE {
            return Err(Errno::E2BIG);
        }
        out.push(s);
    }
    Ok(out)
}

fn sys_execve(frame: &mut TrapFrame, path: u64, argv: u64, envp: u64) -> Result<u64, Errno> {
    let p = current()?;
    let path = user_path(path)?;
    let argv = user_cstr_array(argv)?;
    let envp = user_cstr_array(envp)?;
    process::execve(&p, frame, &path, &argv, &envp)?;
    Ok(0)
}

/// clone(flags, stack, *ptid, *ctid, tls): a new thread (CLONE_THREAD)
/// or a new process (fork-like flags).
///
/// A thread shares everything with its process -- memory, fds, signal
/// handlers -- so CLONE_THREAD needs CLONE_VM and CLONE_SIGHAND, like
/// Linux. (CLONE_FILES / CLONE_FS are accepted and implied: our threads
/// always share the fd table and cwd.) pthread_create passes them all plus
/// SETTLS (its thread block), PARENT_SETTID and CHILD_CLEARTID (so
/// pthread_join can sleep on a futex until the thread is gone).
fn sys_clone(frame: &TrapFrame, flags: u64, stack: u64, ptid: u64, ctid: u64, tls: u64) -> Result<u64, Errno> {
    let p = current()?;
    if flags & CLONE_THREAD != 0 {
        const ALLOWED: u64 = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM
            | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID | CLONE_DETACHED;
        if flags & (CLONE_VM | CLONE_SIGHAND) != CLONE_VM | CLONE_SIGHAND || flags & !(ALLOWED | 0xff) != 0 {
            return Err(Errno::EINVAL);
        }
        let mut set_tid = Vec::new();
        if flags & CLONE_PARENT_SETTID != 0 {
            set_tid.push(ptid);
        }
        if flags & CLONE_CHILD_SETTID != 0 {
            set_tid.push(ctid);
        }
        let clear = if flags & CLONE_CHILD_CLEARTID != 0 { ctid } else { 0 };
        let tls = (flags & CLONE_SETTLS != 0).then_some(tls);
        return process::clone_thread(&p, frame, stack, tls, clear, &set_tid);
    }
    // A process: exit signal SIGCHLD (or 0) in the low byte, optionally
    // CLONE_VFORK (+ CLONE_VM: posix_spawn's "vfork"; we copy-on-write
    // instead of sharing, which a vfork child can't tell apart).
    let exit_signal = flags & 0xff;
    let allowed = 0xff | CLONE_VFORK | CLONE_PARENT_SETTID | if flags & CLONE_VFORK != 0 { CLONE_VM } else { 0 };
    if flags & !allowed != 0 || (exit_signal != 0 && exit_signal != signal::SIGCHLD as u64) {
        return Err(Errno::EINVAL); // shared memory/files between processes: not supported
    }
    let pid = process::fork(&p, frame, stack, flags & CLONE_VFORK != 0)?;
    if flags & CLONE_PARENT_SETTID != 0 {
        copy_to_user(ptid, &(pid as u32))?;
    }
    Ok(pid)
}

/// arch_prctl(ARCH_SET_FS, addr) / (ARCH_GET_FS, *addr): the FS segment
/// base, which libc points at the thread's own data (thread-local storage:
/// `fs:[0]` is "this thread's block" on x86_64). Saved per thread and
/// reloaded by the scheduler on every switch.
fn sys_arch_prctl(code: u64, addr: u64) -> Result<u64, Errno> {
    let t = task()?;
    match code {
        ARCH_SET_FS => {
            if addr >= crate::memory::USER_END {
                return Err(Errno::EPERM); // must be a (canonical) user address
            }
            t.fs_base.store(addr, Ordering::Relaxed);
            x86_64::registers::model_specific::FsBase::write(x86_64::VirtAddr::new(addr));
            Ok(0)
        }
        ARCH_GET_FS => copy_to_user(addr, &t.fs_base.load(Ordering::Relaxed)).map(|_| 0),
        _ => Err(Errno::EINVAL),
    }
}

/// tgkill(tgid, tid, sig) / tkill(tid, sig): a signal for one thread
/// (pthread_kill, raise). sig 0 = just check it exists.
fn sys_tgkill(tgid: Option<u64>, tid: u64, sig: u64) -> Result<u64, Errno> {
    if sig >= signal::NSIG as u64 {
        return Err(Errno::EINVAL);
    }
    let me = current()?;
    let procs = match tgid {
        Some(pid) => alloc::vec![process::get(pid).ok_or(Errno::ESRCH)?],
        None => process::all(),
    };
    let t = procs.iter().find_map(|p| p.task(tid)).ok_or(Errno::ESRCH)?;
    if sig != 0 {
        signal::send_thread(&t, sig as u32, signal::SigInfo::from_pid(sig as u32, me.pid));
    }
    Ok(0)
}

/// nanosleep(*req, *rem). Our timer ticks every ~55 ms, so sleeps are
/// rounded up to that. Cut short by a signal: the time left goes to *rem.
/// clock_nanosleep(clock, flags, *req, *rem): nanosleep on a chosen clock;
/// with TIMER_ABSTIME (1), `req` is a time on that clock to sleep until.
fn sys_clock_nanosleep(clock: u64, flags: u64, req: u64, rem: u64) -> Result<u64, Errno> {
    let now = match clock {
        0 => crate::time::realtime_us(),
        1 | 7 => crate::time::now_us(),
        _ => return Err(Errno::EINVAL), // CPU-time clocks: can't sleep on those
    };
    if flags & 1 == 0 {
        return sys_nanosleep(req, rem);
    }
    // Absolute: sleep the difference (and never report time left, like Linux).
    sleep_for(read_timespec_us(req)?.saturating_sub(now), 0)
}

fn sys_nanosleep(req: u64, rem: u64) -> Result<u64, Errno> {
    sleep_for(read_timespec_us(req)?, rem)
}

/// Sleep `us` microseconds; if a signal cuts it short, the time left goes
/// to `*rem` (a timespec) unless that's 0.
fn sleep_for(us: u64, rem: u64) -> Result<u64, Errno> {
    if us == 0 {
        thread::yield_now();
        return Ok(0);
    }
    // The PIT ticks only every ~55 ms, so sleep whole ticks first and then
    // yield until the (TSC-precise) clock says the time is really up.
    let target = crate::time::now_us() + us;
    let deadline = crate::time::deadline_after_us(us).saturating_sub(1);
    let mut result = match deadline.checked_sub(crate::interrupts::ticks()) {
        Some(t) if t > 0 => thread::sleep_ticks_intr(t),
        _ => Ok(()),
    };
    while result.is_ok() && crate::time::now_us() < target {
        if crate::signal::interrupted() {
            result = Err(Errno::ERESTARTNOHAND);
        } else {
            thread::yield_now();
        }
    }
    if result.is_err() && rem != 0 {
        let left = target.saturating_sub(crate::time::now_us());
        copy_to_user(rem, &[left / 1_000_000, left % 1_000_000 * 1000])?;
    }
    result.map(|_| 0)
}

/// pread64 / pwrite64: at `offset`, without moving the file offset.
fn sys_pread(fd: u64, buf: u64, len: u64, offset: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    if (offset as i64) < 0 {
        return Err(Errno::EINVAL);
    }
    file.read_at(user_slice_mut(buf, len)?, offset).map(|n| n as u64)
}

fn sys_pwrite(fd: u64, buf: u64, len: u64, offset: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    if (offset as i64) < 0 {
        return Err(Errno::EINVAL);
    }
    file.write_at(user_slice(buf, len)?, offset).map(|n| n as u64)
}

/// The `struct iovec {void *base; size_t len;}` array of readv/writev.
pub(crate) fn read_iovecs(ptr: u64, count: u64) -> Result<Vec<(u64, u64)>, Errno> {
    if count > 1024 {
        return Err(Errno::EINVAL);
    }
    let raw = user_slice(ptr, count * 16)?;
    Ok(raw.chunks(16).map(|c| (u64::from_le_bytes(c[..8].try_into().unwrap()), u64::from_le_bytes(c[8..].try_into().unwrap()))).collect())
}

/// readv: one read into a kernel buffer, then spread over the buffers
/// (so a pipe or terminal sees a single read, like Linux).
fn sys_readv(fd: u64, iov: u64, count: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    let iovs = read_iovecs(iov, count)?;
    let total: u64 = iovs.iter().map(|v| v.1).sum::<u64>().min(1 << 20);
    for &(base, len) in &iovs {
        user_slice_mut(base, len)?; // check first
    }
    let mut buf = alloc::vec![0u8; total as usize];
    let n = file.read(&mut buf)?;
    let mut done = 0;
    for &(base, len) in &iovs {
        let k = (len as usize).min(n - done);
        user_slice_mut(base, k as u64)?.copy_from_slice(&buf[done..done + k]);
        done += k;
    }
    Ok(n as u64)
}

/// writev: gather the buffers and write them in one go (so the pieces of
/// one printf can't be interleaved with another thread's output).
fn sys_writev(fd: u64, iov: u64, count: u64) -> Result<u64, Errno> {
    let process = current()?;
    let file = process.file(fd)?;
    let mut buf = Vec::new();
    for (base, len) in read_iovecs(iov, count)? {
        buf.extend_from_slice(user_slice(base, len)?);
    }
    let result = file.write(&buf).map(|n| n as u64);
    if result == Err(Errno::EPIPE) {
        signal::send(&process, signal::SIGPIPE, signal::SigInfo::from_pid(signal::SIGPIPE, 0));
    }
    result
}

/// wait4(pid, *status, options, *rusage). We don't count CPU time per
/// process yet, so a `struct rusage` (144 bytes) comes back all zeros.
fn sys_wait4(pid: i64, status_ptr: u64, options: u64, rusage_ptr: u64) -> Result<u64, Errno> {
    if status_ptr != 0 {
        user_slice_mut(status_ptr, 4)?; // check before reaping anything
    }
    if rusage_ptr != 0 {
        user_slice_mut(rusage_ptr, 144)?;
    }
    let (pid, status) = process::wait4(&*current()?, pid, options)?;
    if status_ptr != 0 && pid != 0 {
        copy_to_user(status_ptr, &status)?;
    }
    if rusage_ptr != 0 && pid != 0 {
        user_slice_mut(rusage_ptr, 144)?.fill(0);
    }
    Ok(pid)
}

fn sys_rt_sigaction(sig: u64, new_ptr: u64, old_ptr: u64, size: u64) -> Result<u64, Errno> {
    let new = if new_ptr == 0 {
        None
    } else {
        let bytes = user_slice(new_ptr, core::mem::size_of::<signal::SigAction>() as u64)?;
        Some(unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const signal::SigAction) })
    };
    if old_ptr != 0 {
        user_slice_mut(old_ptr, 32)?;
    }
    let old = signal::sys_rt_sigaction(&*current()?, sig, new, size)?;
    if old_ptr != 0 {
        copy_to_user(old_ptr, &old)?;
    }
    Ok(0)
}

fn sys_rt_sigprocmask(how: u64, set_ptr: u64, old_ptr: u64, size: u64) -> Result<u64, Errno> {
    let set = if set_ptr == 0 { None } else { Some(u64::from_le_bytes(user_slice(set_ptr, 8)?.try_into().unwrap())) };
    let old = signal::sys_rt_sigprocmask(&*task()?, how, set, size)?;
    if old_ptr != 0 {
        copy_to_user(old_ptr, &old)?;
    }
    Ok(0)
}

/// mmap(addr, len, prot, flags, fd, offset). The new region is empty: pages
/// appear on first touch (demand paging, mm.rs).
fn sys_mmap(p: &process::Process, addr: u64, len: u64, prot: u64, flags: u64, fd: u64, offset: u64) -> Result<u64, Errno> {
    use crate::mm::{Kind, Mm, Vma};
    let len = len.checked_next_multiple_of(4096).ok_or(Errno::ENOMEM)?;
    let shared = match flags & (MAP_SHARED | MAP_PRIVATE) {
        MAP_SHARED => true,
        MAP_PRIVATE => false,
        _ => return Err(Errno::EINVAL), // exactly one of the two
    };
    if len == 0 || prot & !7 != 0 || offset % 4096 != 0 {
        return Err(Errno::EINVAL);
    }
    let file = if flags & MAP_ANONYMOUS != 0 {
        None
    } else {
        // A file: only read-only sharing of its bytes is supported -- a
        // private copy of the file's pages (writes stay in this process).
        // MAP_SHARED of a file would need writes to reach the file (a page
        // cache), which we don't have.
        if shared {
            return Err(Errno::ENODEV);
        }
        let inode = p.file(fd)?.inode().ok_or(Errno::ENODEV)?;
        Some((inode, offset))
    };
    let mut mm = p.mm();
    let start = if flags & MAP_FIXED != 0 {
        if addr % 4096 != 0 || !Mm::range_ok(addr, addr.saturating_add(len)) {
            return Err(Errno::EINVAL);
        }
        mm.unmap(addr, addr + len); // MAP_FIXED replaces whatever was there
        addr
    } else {
        mm.find_free(len).ok_or(Errno::ENOMEM)?
    };
    mm.insert(Vma { start, end: start + len, prot, shared, file, kind: Kind::Mmap });
    if shared {
        // Shared memory gets its frames now, so a fork before the first
        // touch still shares them (mm.rs).
        if mm.populate(start, start + len).is_err() {
            mm.unmap(start, start + len);
            return Err(Errno::ENOMEM);
        }
    }
    Ok(start)
}

/// The pages of `addr..addr+len` for munmap/mprotect: `addr` page aligned.
///
/// Stage 19: anywhere in user space, not just below MMAP_TOP (where mmap
/// puts things): the vvar page, the vDSO and the stack are above it, and
/// programs may mprotect/munmap those too.
fn page_range(addr: u64, len: u64) -> Result<(u64, u64), Errno> {
    let end = addr.checked_add(len).ok_or(Errno::EINVAL)?.next_multiple_of(4096);
    if addr % 4096 != 0 || addr < crate::memory::USER_START || end.max(addr + 1) > crate::memory::USER_END {
        return Err(Errno::EINVAL);
    }
    Ok((addr, end))
}

fn sys_munmap(p: &process::Process, addr: u64, len: u64) -> Result<u64, Errno> {
    let (start, end) = page_range(addr, len)?;
    if len == 0 {
        return Err(Errno::EINVAL);
    }
    p.mm().unmap(start, end);
    Ok(0)
}

fn sys_mprotect(p: &process::Process, addr: u64, len: u64, prot: u64) -> Result<u64, Errno> {
    if prot & !7 != 0 {
        return Err(Errno::EINVAL);
    }
    let (start, end) = page_range(addr, len)?;
    if start == end {
        return Ok(0);
    }
    p.mm().protect(start, end, prot).map(|_| 0)
}

// --- Stage 16: libc batch 1 -------------------------------------------------

/// madvise(addr, len, advice): hints about how memory will be used. Most
/// are just hints (we accept and ignore them); MADV_DONTNEED and MADV_FREE
/// really give private pages back (they read as zeros afterwards).
/// MADV_REMOVE / MADV_WIPEONFORK and unknown values: EINVAL (we can't do
/// them, and saying "done" would be a lie a program might rely on).
fn sys_madvise(addr: u64, len: u64, advice: u64) -> Result<u64, Errno> {
    let discard = match advice {
        MADV_DONTNEED | MADV_FREE => true,
        // NORMAL, RANDOM, SEQUENTIAL, WILLNEED; DONTFORK..NOHUGEPAGE,
        // DONTDUMP, DODUMP; KEEPONFORK, COLD, PAGEOUT, POPULATE_(READ|WRITE).
        0..=3 | 10..=17 | 19..=23 => false,
        _ => return Err(Errno::EINVAL),
    };
    if addr % 4096 != 0 {
        return Err(Errno::EINVAL);
    }
    if len == 0 {
        return Ok(0);
    }
    let end = addr.checked_add(len).ok_or(Errno::EINVAL)?.next_multiple_of(4096);
    if !crate::mm::Mm::range_ok(addr, end) {
        return Err(Errno::ENOMEM);
    }
    if discard {
        current()?.mm().discard(addr, end)?;
    }
    Ok(0)
}

/// fsync / fdatasync: our filesystems write through to the disk on every
/// write (there's no page cache yet), so there's nothing left to flush.
/// Pipes, sockets and ttys can't be synced: EINVAL, like Linux.
fn sys_fsync(fd: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    if file.vfs_inode().is_some() { Ok(0) } else { Err(Errno::EINVAL) }
}

/// Longest file truncate/ftruncate may make (bigger: EFBIG). 4 GiB - 1 is
/// FAT's limit, and far more than our RAM, so ramfs can't be asked to
/// allocate something silly either.
const MAX_TRUNCATE: u64 = u32::MAX as u64;

fn set_len_checked(inode: &dyn fs::Inode, len: u64) -> Result<u64, Errno> {
    match inode.kind() {
        InodeKind::Directory => return Err(Errno::EISDIR),
        InodeKind::File => {}
        _ => return Err(Errno::EINVAL),
    }
    if (len as i64) < 0 {
        return Err(Errno::EINVAL);
    }
    if len > MAX_TRUNCATE {
        return Err(Errno::EFBIG);
    }
    inode.set_len(len as usize).map(|_| 0)
}

/// truncate(path, len): cut a file to `len` bytes, or grow it with zeros.
fn sys_truncate(path_ptr: u64, len: u64) -> Result<u64, Errno> {
    let inode = fs::lookup(&user_path(path_ptr)?)?;
    set_len_checked(&*inode, len)
}

/// ftruncate(fd, len): the same through an fd, which must be open for writing.
fn sys_ftruncate(fd: u64, len: u64) -> Result<u64, Errno> {
    let file = current()?.file(fd)?;
    let inode = file.inode().ok_or(Errno::EINVAL)?; // regular files only
    if file.status_flags() & O_ACCMODE == 0 {
        return Err(Errno::EINVAL); // opened O_RDONLY (Linux says EINVAL too)
    }
    set_len_checked(&*inode, len)
}

/// fchmodat(dirfd, path, mode) and fchmodat2 (same + flags). chmod 90 is
/// fchmodat(AT_FDCWD, ...). FAT has no permissions (EPERM), btrfs is
/// read-only here (EROFS). Symlinks have no mode of their own: with
/// AT_SYMLINK_NOFOLLOW on one, EOPNOTSUPP (as Linux).
fn sys_fchmodat(dirfd: u64, path_ptr: u64, mode: u64, flags: u64) -> Result<u64, Errno> {
    if flags & !(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH) != 0 {
        return Err(Errno::EINVAL);
    }
    let raw = user_cstr(path_ptr, PATH_MAX - 1)?;
    if raw.is_empty() && flags & AT_EMPTY_PATH != 0 && dirfd as i32 as i64 != AT_FDCWD {
        return sys_fchmod(dirfd, mode);
    }
    let path = if raw.is_empty() && flags & AT_EMPTY_PATH != 0 { at_base(dirfd)? } else { path_at(dirfd, &raw)? };
    let inode = if flags & AT_SYMLINK_NOFOLLOW != 0 { fs::lookup_nofollow(&path)? } else { fs::lookup(&path)? };
    if inode.kind() == InodeKind::Symlink {
        return Err(Errno::EOPNOTSUPP);
    }
    inode.set_mode(mode as u32 & 0o7777).map(|_| 0)
}

/// fchmod(fd, mode): files and directories opened by path; others EINVAL.
fn sys_fchmod(fd: u64, mode: u64) -> Result<u64, Errno> {
    let inode = current()?.file(fd)?.vfs_inode().ok_or(Errno::EINVAL)?;
    inode.set_mode(mode as u32 & 0o7777).map(|_| 0)
}

/// utimensat(dirfd, path, *[timespec; 2], flags): set the access and
/// modification times. NULL times = both "now"; a tv_nsec of UTIME_NOW /
/// UTIME_OMIT = now / leave alone. A NULL path means the file `dirfd` is
/// open on (that's how musl does futimens). The change time becomes now.
/// FAT accepts and forgets them (it only has a modification date in a
/// format we don't write); btrfs: EROFS.
fn sys_utimensat(dirfd: u64, path_ptr: u64, times_ptr: u64, flags: u64) -> Result<u64, Errno> {
    if flags & !(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH) != 0 {
        return Err(Errno::EINVAL);
    }
    let now = fs::now_timespec();
    let (atime, mtime) = if times_ptr == 0 {
        (Some(now), Some(now))
    } else {
        let raw = user_slice(times_ptr, 32)?;
        let word = |i: usize| i64::from_le_bytes(raw[i * 8..i * 8 + 8].try_into().unwrap());
        let one = |sec: i64, nsec: i64| match nsec {
            UTIME_NOW => Ok(Some(now)),
            UTIME_OMIT => Ok(None),
            0..=999_999_999 => Ok(Some([sec, nsec])),
            _ => Err(Errno::EINVAL),
        };
        (one(word(0), word(1))?, one(word(2), word(3))?)
    };
    let inode = if path_ptr == 0 {
        if flags & AT_SYMLINK_NOFOLLOW != 0 {
            return Err(Errno::EINVAL);
        }
        if dirfd as i32 as i64 == AT_FDCWD {
            return Err(Errno::EFAULT); // Linux: a NULL path needs a real fd
        }
        current()?.file(dirfd)?.vfs_inode().ok_or(Errno::EINVAL)?
    } else {
        let raw = user_cstr(path_ptr, PATH_MAX - 1)?;
        if raw.is_empty() && flags & AT_EMPTY_PATH != 0 && dirfd as i32 as i64 != AT_FDCWD {
            current()?.file(dirfd)?.vfs_inode().ok_or(Errno::EINVAL)?
        } else {
            let path = if raw.is_empty() && flags & AT_EMPTY_PATH != 0 { at_base(dirfd)? } else { path_at(dirfd, &raw)? };
            if flags & AT_SYMLINK_NOFOLLOW != 0 { fs::lookup_nofollow(&path)? } else { fs::lookup(&path)? }
        }
    };
    inode.set_times(atime, mtime).map(|_| 0)
}

/// prlimit64(pid, resource, *new, *old), and getrlimit / setrlimit on
/// myself. `struct rlimit` is {u64 soft, u64 hard}; RLIM_INFINITY = !0.
/// What the kernel enforces: RLIMIT_NOFILE (the fd table size, at most 64).
/// The rest are remembered and inherited (fork), so programs that check
/// them see sensible, consistent numbers.
fn sys_prlimit64(pid: u64, resource: u64, new_ptr: u64, old_ptr: u64) -> Result<u64, Errno> {
    let me = current()?;
    let target = if pid == 0 || pid == me.pid {
        me
    } else {
        process::get(pid).filter(|p| p.status().is_none()).ok_or(Errno::ESRCH)?
    };
    let new = if new_ptr == 0 {
        None
    } else {
        let raw = user_slice(new_ptr, 16)?;
        Some([u64::from_le_bytes(raw[..8].try_into().unwrap()), u64::from_le_bytes(raw[8..].try_into().unwrap())])
    };
    if old_ptr != 0 {
        user_slice_mut(old_ptr, 16)?; // EFAULT before changing anything
    }
    let old = target.rlimit(resource)?;
    if let Some(new) = new {
        target.set_rlimit(resource, new)?;
    }
    if old_ptr != 0 {
        copy_to_user(old_ptr, &old)?;
    }
    Ok(0)
}

/// prctl(option, ...): a grab bag of per-process settings. We know:
/// PR_SET_NAME / PR_GET_NAME (the thread's 16-byte name, `comm`; for the
/// main thread also what `ps` shows), PR_SET/GET_PDEATHSIG (a signal to
/// get when my parent dies), PR_SET/GET_DUMPABLE, PR_SET/GET_NO_NEW_PRIVS,
/// PR_SET/GET_CHILD_SUBREAPER (adopt orphaned descendants, process.rs).
/// Everything else: EINVAL, which is what Linux says to unknown options.
fn sys_prctl(option: u64, a2: u64, a3: u64, a4: u64, a5: u64) -> Result<u64, Errno> {
    let p = current()?;
    match option {
        PR_SET_PDEATHSIG => {
            if a2 > signal::NSIG as u64 {
                return Err(Errno::EINVAL);
            }
            p.pdeathsig.store(a2 as u32, Ordering::Relaxed);
            Ok(0)
        }
        PR_GET_PDEATHSIG => copy_to_user(a2, &(p.pdeathsig.load(Ordering::Relaxed) as i32)).map(|_| 0),
        PR_GET_DUMPABLE => Ok(p.dumpable.load(Ordering::Relaxed) as u64),
        PR_SET_DUMPABLE if a2 <= 1 => {
            p.dumpable.store(a2 as u32, Ordering::Relaxed);
            Ok(0)
        }
        PR_SET_NAME => {
            // Like Linux: up to 15 bytes are taken, a longer name is cut.
            let mut name = [0u8; 16];
            for i in 0..15 {
                let b = user_slice(a2 + i as u64, 1)?[0];
                if b == 0 {
                    break;
                }
                name[i] = b;
            }
            let t = task()?;
            *t.comm.lock() = name;
            if t.tid() == p.pid {
                let len = name.iter().position(|&b| b == 0).unwrap_or(16);
                p.set_name(&String::from_utf8_lossy(&name[..len]));
            }
            Ok(0)
        }
        PR_GET_NAME => {
            let name = *task()?.comm.lock();
            copy_to_user(a2, &name).map(|_| 0)
        }
        PR_SET_CHILD_SUBREAPER => {
            p.child_subreaper.store(a2 != 0, Ordering::Relaxed);
            Ok(0)
        }
        PR_GET_CHILD_SUBREAPER => copy_to_user(a2, &(p.child_subreaper.load(Ordering::Relaxed) as i32)).map(|_| 0),
        PR_SET_NO_NEW_PRIVS if a2 == 1 && a3 == 0 && a4 == 0 && a5 == 0 => {
            p.no_new_privs.store(true, Ordering::Relaxed);
            Ok(0)
        }
        PR_GET_NO_NEW_PRIVS if a2 == 0 && a3 == 0 && a4 == 0 && a5 == 0 => Ok(p.no_new_privs.load(Ordering::Relaxed) as u64),
        _ => Err(Errno::EINVAL),
    }
}

/// Size of `struct robust_list_head` (list, futex_offset, list_op_pending).
pub const ROBUST_LIST_HEAD_SIZE: u64 = 24;

/// set_robust_list(head, len): where this thread's list of held robust
/// mutexes lives. When the thread dies, the kernel walks it and marks each
/// one FUTEX_OWNER_DIED (process.rs, `exit_robust_list`), so the next
/// locker learns the owner died instead of waiting forever.
fn sys_set_robust_list(head: u64, len: u64) -> Result<u64, Errno> {
    if len != ROBUST_LIST_HEAD_SIZE {
        return Err(Errno::EINVAL);
    }
    task()?.robust_list.store(head, Ordering::Relaxed);
    Ok(0)
}

/// get_robust_list(tid, *head, *len): 0 = myself.
fn sys_get_robust_list(tid: u64, head_ptr: u64, len_ptr: u64) -> Result<u64, Errno> {
    let t = if tid == 0 { task()? } else { process::all().iter().find_map(|p| p.task(tid)).ok_or(Errno::ESRCH)? };
    copy_to_user(head_ptr, &t.robust_list.load(Ordering::Relaxed))?;
    copy_to_user(len_ptr, &ROBUST_LIST_HEAD_SIZE)?;
    Ok(0)
}

/// sigaltstack(*new, *old): `stack_t` = {void *ss_sp; int ss_flags; size_t
/// ss_size} (24 bytes with padding). Handlers installed with SA_ONSTACK
/// run on this stack -- how a program can still handle SIGSEGV after
/// overflowing its normal stack. `user_sp`: the caller's stack pointer
/// (to tell whether it's running on the alternate stack right now).
fn sys_sigaltstack(new_ptr: u64, old_ptr: u64, user_sp: u64) -> Result<u64, Errno> {
    let new = if new_ptr == 0 {
        None
    } else {
        let raw = user_slice(new_ptr, 24)?;
        let word = |i: usize| u64::from_le_bytes(raw[i * 8..i * 8 + 8].try_into().unwrap());
        Some((word(0), word(1) as u32, word(2)))
    };
    if old_ptr != 0 {
        user_slice_mut(old_ptr, 24)?;
    }
    let (sp, flags, size) = signal::sys_sigaltstack(&*task()?, new, user_sp)?;
    if old_ptr != 0 {
        copy_to_user(old_ptr, &[sp, flags as u64, size])?;
    }
    Ok(0)
}

/// rt_sigpending(*set, size): which signals are waiting because they're
/// blocked.
fn sys_rt_sigpending(set_ptr: u64, size: u64) -> Result<u64, Errno> {
    if size > 8 {
        return Err(Errno::EINVAL);
    }
    let set = signal::pending_blocked(&*task()?);
    let out = user_slice_mut(set_ptr, size)?;
    out.copy_from_slice(&set.to_le_bytes()[..size as usize]);
    Ok(0)
}

fn read_sigset(ptr: u64, size: u64) -> Result<u64, Errno> {
    if size != 8 {
        return Err(Errno::EINVAL);
    }
    Ok(u64::from_le_bytes(user_slice(ptr, 8)?.try_into().unwrap()))
}

/// rt_sigsuspend(*mask, 8): swap in `mask` and sleep until a signal is
/// handled (always "fails" with EINTR); the old mask comes back after the
/// handler, so there's no window where a signal can slip by unnoticed.
fn sys_rt_sigsuspend(mask_ptr: u64, size: u64) -> Result<u64, Errno> {
    let mask = read_sigset(mask_ptr, size)?;
    signal::set_temporary_mask(mask);
    Err(thread::pause())
}

/// rt_sigtimedwait(*set, *siginfo, *timeout, 8): wait for one of `set`
/// (normally blocked) and *take* it -- no handler runs; the signal number
/// is returned and its siginfo copied out. Timeout NULL = forever, {0, 0}
/// = just look; expired: EAGAIN. Another signal with a handler: EINTR.
/// Timing is rounded up to timer ticks (~55 ms).
fn sys_rt_sigtimedwait(set_ptr: u64, info_ptr: u64, timeout_ptr: u64, size: u64) -> Result<u64, Errno> {
    static SIGWAIT: thread::WaitQueue = thread::WaitQueue::new();
    let set = signal::blockable(read_sigset(set_ptr, size)?);
    let timeout = if timeout_ptr == 0 { None } else { Some(read_timespec_us(timeout_ptr)?) };
    let t = task()?;
    let got = match signal::take_waited(&t, set) {
        Some(got) => Ok(got),
        None if timeout == Some(0) => Err(Errno::EAGAIN),
        None => {
            // Let the waited-for signals wake us (they're usually blocked).
            let blocked = without_interrupts_blocked(&t);
            signal::set_temporary_mask(blocked & !set);
            let r = match timeout {
                Some(us) => SIGWAIT
                    .wait_until_deadline_intr(crate::time::deadline_after_us(us), || signal::take_waited(&t, set))
                    .and_then(|got| got.ok_or(Errno::EAGAIN)),
                None => SIGWAIT.wait_until_intr(|| signal::take_waited(&t, set)),
            };
            signal::restore_temporary_mask();
            r.map_err(|e| if e == Errno::EAGAIN { e } else { Errno::EINTR })
        }
    }?;
    let (sig, info) = got;
    if info_ptr != 0 {
        copy_to_user(info_ptr, &info)?;
    }
    Ok(sig as u64)
}

/// The thread's blocked set right now.
fn without_interrupts_blocked(t: &process::Task) -> u64 {
    x86_64::instructions::interrupts::without_interrupts(|| t.sig.lock().blocked)
}

/// rt_sigqueueinfo(tgid, sig, *siginfo) / rt_tgsigqueueinfo(tgid, tid,
/// sig, *siginfo): send a signal with my own siginfo (sigqueue() puts a
/// value in it). Real-time signals (32..64) queue up: sending one 3 times
/// delivers it 3 times, each with its own value. Pretending to be the
/// kernel or kill() (si_code >= 0, or SI_TKILL) is only allowed toward
/// myself: EPERM. A full queue: EAGAIN.
fn sys_rt_sigqueueinfo(tgid: u64, tid: Option<u64>, sig: u64, info_ptr: u64) -> Result<u64, Errno> {
    if sig > signal::NSIG as u64 {
        return Err(Errno::EINVAL);
    }
    let raw = user_slice(info_ptr, core::mem::size_of::<signal::SigInfo>() as u64)?;
    let mut info = unsafe { core::ptr::read_unaligned(raw.as_ptr() as *const signal::SigInfo) };
    let me = task()?;
    let (target_id, my_id) = match tid {
        Some(tid) => (tid, me.tid()),
        None => (tgid, me.process.pid),
    };
    if (info.code >= 0 || info.code == signal::SI_TKILL) && target_id != my_id {
        return Err(Errno::EPERM);
    }
    let p = process::get(tgid).filter(|p| p.status().is_none()).ok_or(Errno::ESRCH)?;
    let thread = match tid {
        Some(tid) => Some(p.task(tid).ok_or(Errno::ESRCH)?),
        None => None,
    };
    if sig == 0 {
        return Ok(0);
    }
    if signal::queue_full(&p) {
        return Err(Errno::EAGAIN);
    }
    info.signo = sig as i32;
    match thread {
        Some(t) => signal::send_thread(&t, sig as u32, info),
        None => signal::send(&p, sig as u32, info),
    }
    Ok(0)
}

/// fcntl(F_GETLK / F_SETLK / F_SETLKW, *flock): POSIX record locks (see
/// fs/lock.rs). `struct flock` = {i16 l_type, i16 l_whence, i64 l_start,
/// i64 l_len, i32 l_pid} (32 bytes with padding).
fn sys_fcntl_lock(p: &Arc<process::Process>, file: &dyn File, cmd: u64, arg: u64) -> Result<u64, Errno> {
    let raw = user_slice(arg, 32)?;
    let l_type = i16::from_le_bytes([raw[0], raw[1]]);
    let l_whence = i16::from_le_bytes([raw[2], raw[3]]);
    let l_start = i64::from_le_bytes(raw[8..16].try_into().unwrap());
    let l_len = i64::from_le_bytes(raw[16..24].try_into().unwrap());
    // Where the range starts from: SEEK_SET, SEEK_CUR or SEEK_END.
    let base = match l_whence {
        0 => 0,
        1 => file.seek(0, 1).unwrap_or(0) as i64,
        2 => file.stat()?.size as i64,
        _ => return Err(Errno::EINVAL),
    };
    let start = base.checked_add(l_start).ok_or(Errno::EINVAL)?;
    let (start, end) = match l_len {
        0 => (start, u64::MAX as i128), // to the end of the file, and beyond
        n if n > 0 => (start, start as i128 + n as i128),
        n => (start.checked_add(n).ok_or(Errno::EINVAL)?, start as i128), // the bytes *before* start
    };
    if start < 0 || end > u64::MAX as i128 {
        return Err(Errno::EINVAL);
    }
    let (start, end) = (start as u64, end as u64);
    let access = file.status_flags() & O_ACCMODE;
    let want = match l_type {
        F_RDLCK if access == 1 => return Err(Errno::EBADF), // reading needs a readable fd
        F_WRLCK if access == 0 => return Err(Errno::EBADF), // writing needs a writable fd
        F_RDLCK => Some(false),
        F_WRLCK => Some(true),
        F_UNLCK => None,
        _ => return Err(Errno::EINVAL),
    };
    let key = fs::lock::key_of(file);
    match cmd {
        F_GETLK => {
            let write = want.ok_or(Errno::EINVAL)?;
            let out = user_slice_mut(arg, 32)?;
            match fs::lock::conflict(key, fs::lock::Lock { pid: p.pid, start, end, write }) {
                Some(c) => {
                    let len = if c.end == u64::MAX { 0 } else { (c.end - c.start) as i64 };
                    out[0..2].copy_from_slice(&(if c.write { F_WRLCK } else { F_RDLCK }).to_le_bytes());
                    out[2..4].copy_from_slice(&0i16.to_le_bytes());
                    out[8..16].copy_from_slice(&(c.start as i64).to_le_bytes());
                    out[16..24].copy_from_slice(&len.to_le_bytes());
                    out[24..28].copy_from_slice(&(c.pid as i32).to_le_bytes());
                }
                None => out[0..2].copy_from_slice(&F_UNLCK.to_le_bytes()),
            }
            Ok(0)
        }
        F_SETLKW if want.is_some() => fs::lock::set_wait(key, p.pid, start, end, want.unwrap()).map(|_| 0),
        _ => fs::lock::set(key, p.pid, start, end, want).map(|_| 0),
    }
}

/// sched_getaffinity(tid, len, *mask) / sched_setaffinity: which CPUs a
/// thread may run on. Ours run on any CPU, so "get" reports every online
/// CPU (that's how musl's sysconf(_SC_NPROCESSORS_ONLN) and `nproc` count
/// them) and "set" only checks the mask names at least one online CPU (we
/// don't pin threads). Like the raw Linux call, "get" returns the number
/// of bytes written (8: one 64-bit word of CPU bits), not 0.
fn sys_sched_affinity(tid: u64, len: u64, mask_ptr: u64, set: bool) -> Result<u64, Errno> {
    if tid != 0 && !process::all().iter().any(|p| p.task(tid).is_some()) {
        return Err(Errno::ESRCH);
    }
    let cpus = crate::smp::cpu_count().min(64);
    let online: u64 = if cpus == 64 { u64::MAX } else { (1u64 << cpus) - 1 };
    if set {
        let raw = user_slice(mask_ptr, len.min(8))?;
        let mut word = [0u8; 8];
        word[..raw.len()].copy_from_slice(raw);
        return if u64::from_le_bytes(word) & online != 0 { Ok(0) } else { Err(Errno::EINVAL) };
    }
    if len < 8 || len % 8 != 0 {
        return Err(Errno::EINVAL); // Linux: room for whole longs, all CPUs
    }
    copy_to_user(mask_ptr, &online)?;
    Ok(8)
}