//! System calls via the `syscall` instruction. //! //! ABI (like Linux x86_64): number in rax, arguments in rdi, rsi, rdx, r10, //! r8, r9; result in rax; negative results are `-errno`. The `syscall` //! instruction itself clobbers rcx (return address) and r11 (saved //! RFLAGS); every other register is preserved for the caller. //! //! Numbers are Linux's where Linux has the same call, so `strace` output and //! man pages mostly apply. The process calls added in stage 14 (fork, //! execve, wait4, mmap, signals, dup, pipe2, fcntl, process groups) follow //! Linux exactly -- C strings, structs, flags -- so a C library can use //! them as they are, and so do stage 15's (poll/epoll, futex, threads, //! ttys, nanosleep, readv/writev...). The older file calls still take //! paths as pointer + length (Linux: NUL-terminated); relative paths start //! at the cwd. Every blocking call honours O_NONBLOCK (-> EAGAIN). //! //! | nr | name | arguments | note | //! |-----|-------------|-------------------------------------|------| //! | 0 | read | fd, buf, len | blocks: tty until a line (canonical) or a byte (raw), pipe until data/EOF | //! | 1 | write | fd, buf, len | to a pipe without readers: EPIPE + SIGPIPE; a full pipe blocks | //! | 2 | open | path, path_len, flags | O_CREAT 0o100, O_NOCTTY 0o400, O_TRUNC 0o1000, O_APPEND, O_NONBLOCK 0o4000, O_CLOEXEC 0o2000000 | //! | 3 | close | fd | | //! | 4 | stat | path, path_len, *Stat | our own small `Stat {kind, size, mode}`; follows symlinks | //! | 5 | fstat | fd, *Stat | | //! | 7 | poll | *pollfd, n, timeout_ms | -1 = forever; pipes, sockets, ttys, files, epoll fds | //! | 8 | lseek | fd, offset, whence | SEEK_SET/CUR/END = 0/1/2 | //! | 9 | mmap | addr, len, prot, flags, fd, offset | MAP_PRIVATE/SHARED, MAP_ANONYMOUS, MAP_FIXED; files: private only | //! | 10 | mprotect | addr, len, prot | PROT_READ 1, PROT_WRITE 2, PROT_EXEC 4 | //! | 11 | munmap | addr, len | | //! | 12 | brk | addr | returns the (new) end of the heap; brk(0) = current | //! | 13 | rt_sigaction | sig, *new, *old, 8 | kernel `struct sigaction`; handlers need SA_RESTORER | //! | 14 | rt_sigprocmask | how, *set, *old, 8 | SIG_BLOCK 0 / SIG_UNBLOCK 1 / SIG_SETMASK 2 | //! | 15 | rt_sigreturn | | only from a signal handler's restorer | //! | 16 | ioctl | fd, request, arg | any fd: FIONBIO; ttys: TCGETS/TCSETS(W/F), TIOCGPGRP/SPGRP, TIOCSCTTY, TIOCNOTTY, TIOCGWINSZ/SWINSZ, TIOCGPTN, TIOCSPTLCK, TTY_SET_RAW 1 (ours); /dev/fb0: FB_* (below) | //! | 17 | pread64 | fd, buf, len, offset | regular files; leaves the file position alone | //! | 18 | pwrite64 | fd, buf, len, offset | | //! | 19 | readv | fd, *iovec, n | `struct iovec {base, len}`, up to 1024 | //! | 20 | writev | fd, *iovec, n | | //! | 22 | pipe | *[u32; 2] | [read end, write end] | //! | 23 | select | n, *rd, *wr, *ex, *timeval | fd bitmaps (n <= 1024) | //! | 24 | sched_yield | | | //! | 32 | dup | fd | lowest free fd | //! | 33 | dup2 | old, new | closes `new` first if open | //! | 34 | pause | | sleeps until a signal; returns EINTR | //! | 35 | nanosleep | *timespec, *remaining | EINTR on a signal (remaining time written back) | //! | 39 | getpid | | | //! | 41 | socket | domain, type, protocol | AF_INET (2) only; type may add SOCK_NONBLOCK/SOCK_CLOEXEC; SOCK_STREAM (1) = TCP, SOCK_DGRAM (2) = UDP, SOCK_DGRAM + IPPROTO_ICMP (1) = ping socket | //! | 42 | connect | fd, *sockaddr_in, len | TCP: handshake, blocks (10 s timeout; nonblocking: EINPROGRESS); UDP/ICMP: set default destination. 127.x = loopback | //! | 43 | accept | fd, *sockaddr_in or 0, *u32 len or 0 | blocks for a connection (SO_RCVTIMEO applies); returns the new fd | //! | 44 | sendto | fd, buf, len, flags, *sockaddr_in or 0 | Linux has a 6th arg (addrlen); we only take 5, the address is always 16 bytes | //! | 45 | recvfrom | fd, buf, len, flags, *sockaddr_in or 0 | blocks (SO_RCVTIMEO -> EAGAIN); 0 = TCP peer closed. No addrlen pointer | //! | 49 | bind | fd, *sockaddr_in, len | sets the local port (address ignored: we have one) | //! | 50 | listen | fd, backlog | TCP; backlog 1..4 | //! | 54 | setsockopt | fd, level, name, *value, len | only SOL_SOCKET (1) / SO_RCVTIMEO (20) with a `struct timeval` | //! | 56 | clone | flags, stack, *ptid, *ctid, tls | fork-like (SIGCHLD [+ CLONE_VFORK]) or a thread: CLONE_VM|FS|FILES|SIGHAND|THREAD|SYSVSEM [+ SETTLS, PARENT_SETTID, CHILD_CLEARTID] | //! | 57 | fork | | copy-on-write; parent gets the pid, child 0 | //! | 58 | vfork | | fork + the parent waits until the child execs or exits | //! | 59 | execve | path, argv, envp (C strings) | Linux's stack layout: argc, argv, envp, auxv | //! | 60 | exit | code | ends this *thread* only (exit_group ends the process) | //! | 61 | wait4 | pid, *status, options, *rusage | WNOHANG 1, WUNTRACED 2, WCONTINUED 8; rusage must be 0 | //! | 62 | kill | pid, sig | pid 0 = my group, -1 = all, < -1 = group -pid | //! | 63 | uname | *utsname | Linux's `struct utsname` (6 x 65-byte strings) | //! | 72 | fcntl | fd, cmd, arg | F_DUPFD 0, F_GETFD 1, F_SETFD 2, F_GETFL 3, F_SETFL 4 (O_NONBLOCK), F_DUPFD_CLOEXEC 1030 | //! | 79 | getcwd | buf, len | like Linux: writes a NUL, returns the length including it | //! | 80 | chdir | path, path_len | | //! | 83 | mkdir | path, path_len | | //! | 84 | rmdir | path, path_len | only empty directories | //! | 87 | unlink | path, path_len | files and symlinks (not what a symlink points to) | //! | 89 | readlink | path, path_len, buf, len | a symlink's target; returns its length (cut to `len`, no NUL); /proc/self/exe = my program | //! | 109 | setpgid | pid, pgid | 0 = myself / a new group | //! | 110 | getppid | | 0 if started by the kernel or orphaned | //! | 111 | getpgrp | | | //! | 112 | setsid | | new session + group, no controlling tty | //! | 121 | getpgid | pid | | //! | 124 | getsid | pid | | //! | 158 | arch_prctl | code, addr | ARCH_SET_FS 0x1002 / ARCH_GET_FS 0x1003: the TLS pointer | //! | 186 | gettid | | thread id (= pid for the first thread) | //! | 200 | tkill | tid, sig | | //! | 202 | futex | *u32, op, val, *timespec, *u32, val3 | FUTEX_WAIT 0, WAKE 1, WAIT_BITSET 9, WAKE_BITSET 10 (+ PRIVATE 128) | //! | 213 | epoll_create | size | = epoll_create1(0) | //! | 217 | getdents64 | fd, buf, len | Linux `struct linux_dirent64` records | //! | 218 | set_tid_address | *u32 | cleared + futex-woken when this thread exits; returns the tid | //! | 228 | clock_gettime | clock, *timespec | REALTIME 0 (CMOS clock at boot + uptime), MONOTONIC 1 (since boot) | //! | 231 | exit_group | code | ends every thread of the process | //! | 232 | epoll_wait | epfd, *epoll_event, max, timeout_ms | level-triggered; EPOLLONESHOT supported | //! | 233 | epoll_ctl | epfd, op, fd, *epoll_event | ADD 1, DEL 2, MOD 3 | //! | 234 | tgkill | tgid, tid, sig | a signal for one thread | //! | 270 | pselect6 | n, *rd, *wr, *ex, *timespec, *{*mask, 8} | | //! | 271 | ppoll | *pollfd, n, *timespec, *mask, 8 | sets the mask while waiting | //! | 281 | epoll_pwait | epfd, *events, max, timeout, *mask, 8 | | //! | 288 | accept4 | fd, *addr, *len, flags | flags: SOCK_NONBLOCK, SOCK_CLOEXEC | //! | 291 | epoll_create1 | flags | EPOLL_CLOEXEC | //! | 292 | dup3 | old, new, flags | flags: O_CLOEXEC | //! | 293 | pipe2 | *[u32; 2], flags | flags: O_CLOEXEC | //! | 318 | getrandom | buf, len, flags | random bytes (virtio-rng); up to 64 KiB per call; flags ignored | //! | 500 | spawn | path, path_len, argv, argv_len, *SpawnOptions | ours: new process running a program (fork+exec in one step, like posix_spawn) | //! | 501 | mounts | buf, len | ours: "path fstype\n" per mount (Linux: read /proc/mounts) | //! | 502 | sysinfo | *SysInfo | ours: CPUs, threads, memory, per-CPU busy time (Linux: /proc/stat etc.) | //! | 503 | resolve | name, name_len, *[[u8; 4]], max | ours: DNS lookup (A records), returns the count. Linux does DNS in libc | //! | 504 | netinfo | *NetInfo | ours: MAC, IP, gateway, DNS, packet counters (Linux: netlink/ioctls) | //! | 505 | trace_me | on | ours: print my syscalls from now on (survives execve; `run --trace`) | //! //! spawn's `argv` is all arguments separated by NUL bytes ("ls\0-l\0/bin"); //! empty means just [program name]. `SpawnOptions` (or a null pointer for //! the defaults) picks the child's stdin/stdout/stderr from the caller's fds //! (-1 = same as mine) and flags (SPAWN_TRACE = print its syscalls). //! //! The framebuffer ioctls (on an fd of /dev/fb0; numbers in Linux's 'F' //! range, but the structs are our own -- Linux's fb_var_screeninfo has 40 //! fields): //! //! FB_GET_INFO 0x4600 *FbInfo {width, height, stride_bytes, bpp} (u32s) //! FB_FLUSH 0x46f0 *[x, y, w, h] or 0 = what `write` changed: show it //! FB_FILL 0x46f1 *[x, y, w, h, 0x00RRGGBB]: fill (not flushed) //! FB_SET_CONSOLE 0x46f2 *[x, y, w, h] or 0 = full screen: move the text //! console there; returns columns << 16 | rows //! //! What happens on `syscall` (the CPU does steps 1-3 in one instruction): //! 1. rcx = user RIP (next instruction), r11 = user RFLAGS //! 2. RFLAGS &= !SFMASK (we mask IF -> interrupts off) //! 3. CS/SS = kernel selectors from STAR, RIP = LSTAR (`syscall_entry`), CPL = 0 //! ...but RSP is still the *user* stack! The CPU doesn't switch stacks for //! `syscall` (unlike interrupts, which use TSS.RSP0). So `syscall_entry` //! must do it first: stash the user RSP, load this thread's kernel stack, //! save all registers as a `TrapFrame` (trap.rs), re-enable interrupts and //! call `syscall_dispatch` in Rust. The way back is trap.rs's common exit, //! `iretq` (stage 13 used the faster `sysretq`, but that can't restore //! rcx/r11, which a signal handler's return needs). //! //! **Several CPUs.** The "kernel stack of the running thread" differs per //! CPU, but at the entry we have no free register to compute which CPU we //! are on. The trick every x86_64 kernel uses: the GS segment base. Each CPU //! keeps a pointer to its own `smp::PerCpu` block in the KERNEL_GS_BASE MSR; //! `swapgs` swaps that with the (user's) GS base, so for a few instructions //! `gs:[0]` is *this CPU's* kernel stack and `gs:[8]` a scratch slot. We //! swap back right after switching stacks, so the thread can be moved to //! another CPU later without any GS confusion. //! //! Why not `int 0x80`? It also works (and is simpler: the CPU switches stacks //! via TSS.RSP0 for us), but `syscall` is what Linux and every modern x86_64 //! OS use, and it's faster. use crate::errno::Errno; use crate::fs::file::{File, InodeFile, LinuxStat, Stat}; use crate::fs::{self, InodeKind}; use crate::trap::TrapFrame; use crate::{memory, process, signal, thread}; use alloc::{string::String, sync::Arc, vec::Vec}; use core::arch::global_asm; use core::sync::atomic::Ordering; pub const SYS_READ: u64 = 0; pub const SYS_WRITE: u64 = 1; pub const SYS_OPEN: u64 = 2; pub const SYS_CLOSE: u64 = 3; pub const SYS_STAT: u64 = 4; pub const SYS_FSTAT: u64 = 5; pub const SYS_LSEEK: u64 = 8; pub const SYS_IOCTL: u64 = 16; pub const SYS_PIPE: u64 = 22; pub const SYS_SCHED_YIELD: u64 = 24; pub const SYS_NANOSLEEP: u64 = 35; pub const SYS_GETPID: u64 = 39; pub const SYS_EXIT: u64 = 60; pub const SYS_UNAME: u64 = 63; pub const SYS_GETCWD: u64 = 79; pub const SYS_CHDIR: u64 = 80; pub const SYS_MKDIR: u64 = 83; pub const SYS_RMDIR: u64 = 84; pub const SYS_UNLINK: u64 = 87; pub const SYS_READLINK: u64 = 89; pub const SYS_SOCKET: u64 = 41; pub const SYS_CONNECT: u64 = 42; pub const SYS_ACCEPT: u64 = 43; pub const SYS_ACCEPT4: u64 = 288; pub const SYS_SENDTO: u64 = 44; pub const SYS_RECVFROM: u64 = 45; pub const SYS_BIND: u64 = 49; pub const SYS_LISTEN: u64 = 50; pub const SYS_SETSOCKOPT: u64 = 54; pub const SYS_GETDENTS64: u64 = 217; pub const SYS_GETRANDOM: u64 = 318; pub const SYS_CLOCK_GETTIME: u64 = 228; pub const SYS_SPAWN: u64 = 500; pub const SYS_MOUNTS: u64 = 501; pub const SYS_SYSINFO: u64 = 502; pub const SYS_RESOLVE: u64 = 503; pub const SYS_NETINFO: u64 = 504; pub const SYS_MMAP: u64 = 9; pub const SYS_MPROTECT: u64 = 10; pub const SYS_MUNMAP: u64 = 11; pub const SYS_BRK: u64 = 12; pub const SYS_RT_SIGACTION: u64 = 13; pub const SYS_RT_SIGPROCMASK: u64 = 14; pub const SYS_RT_SIGRETURN: u64 = 15; pub const SYS_DUP: u64 = 32; pub const SYS_DUP2: u64 = 33; pub const SYS_PAUSE: u64 = 34; pub const SYS_CLONE: u64 = 56; pub const SYS_FORK: u64 = 57; pub const SYS_VFORK: u64 = 58; pub const SYS_EXECVE: u64 = 59; pub const SYS_WAIT4: u64 = 61; pub const SYS_KILL: u64 = 62; pub const SYS_FCNTL: u64 = 72; pub const SYS_SETPGID: u64 = 109; pub const SYS_GETPPID: u64 = 110; pub const SYS_GETPGRP: u64 = 111; pub const SYS_GETPGID: u64 = 121; pub const SYS_EXIT_GROUP: u64 = 231; pub const SYS_DUP3: u64 = 292; pub const SYS_PIPE2: u64 = 293; pub const SYS_TRACE_ME: u64 = 505; // stage 15 pub const SYS_POLL: u64 = 7; pub const SYS_PREAD64: u64 = 17; pub const SYS_PWRITE64: u64 = 18; pub const SYS_READV: u64 = 19; pub const SYS_WRITEV: u64 = 20; pub const SYS_SELECT: u64 = 23; pub const SYS_SETSID: u64 = 112; pub const SYS_GETSID: u64 = 124; pub const SYS_ARCH_PRCTL: u64 = 158; pub const SYS_GETTID: u64 = 186; pub const SYS_TKILL: u64 = 200; pub const SYS_FUTEX: u64 = 202; pub const SYS_EPOLL_CREATE: u64 = 213; pub const SYS_SET_TID_ADDRESS: u64 = 218; pub const SYS_EPOLL_WAIT: u64 = 232; pub const SYS_EPOLL_CTL: u64 = 233; pub const SYS_TGKILL: u64 = 234; pub const SYS_PSELECT6: u64 = 270; pub const SYS_PPOLL: u64 = 271; pub const SYS_EPOLL_PWAIT: u64 = 281; pub const SYS_EPOLL_CREATE1: u64 = 291; // Stage 15.1: the rest of Linux's path calls, and a few musl needs early. pub const SYS_LSTAT: u64 = 6; pub const SYS_ACCESS: u64 = 21; pub const SYS_RENAME: u64 = 82; pub const SYS_FCHDIR: u64 = 81; pub const SYS_GETUID: u64 = 102; pub const SYS_GETGID: u64 = 104; pub const SYS_GETEUID: u64 = 107; pub const SYS_GETEGID: u64 = 108; pub const SYS_CLOCK_NANOSLEEP: u64 = 230; pub const SYS_OPENAT: u64 = 257; pub const SYS_MKDIRAT: u64 = 258; pub const SYS_NEWFSTATAT: u64 = 262; pub const SYS_UNLINKAT: u64 = 263; pub const SYS_RENAMEAT: u64 = 264; pub const SYS_READLINKAT: u64 = 267; pub const SYS_FACCESSAT: u64 = 269; pub const SYS_RENAMEAT2: u64 = 316; pub const SYS_FACCESSAT2: u64 = 439; // Socket constants (Linux's values). pub const AF_INET: u64 = 2; pub const SOCK_STREAM: u64 = 1; pub const SOCK_DGRAM: u64 = 2; pub const IPPROTO_ICMP: u64 = 1; pub const SOL_SOCKET: u64 = 1; pub const SO_RCVTIMEO: u64 = 20; pub const O_ACCMODE: u64 = 3; // O_RDONLY 0, O_WRONLY 1, O_RDWR 2 pub const O_CREAT: u64 = 0o100; pub const O_EXCL: u64 = 0o200; pub const O_DIRECTORY: u64 = 0o200000; pub const O_NOFOLLOW: u64 = 0o400000; /// The *at() calls: "relative to the cwd" instead of a directory fd, and flags. pub const AT_FDCWD: i64 = -100; pub const AT_SYMLINK_NOFOLLOW: u64 = 0x100; pub const AT_REMOVEDIR: u64 = 0x200; pub const AT_EMPTY_PATH: u64 = 0x1000; /// Longest path, counting the NUL (Linux's PATH_MAX). pub const PATH_MAX: usize = 4096; pub const O_NOCTTY: u64 = 0o400; pub const O_TRUNC: u64 = 0o1000; pub const O_APPEND: u64 = 0o2000; pub const O_NONBLOCK: u64 = 0o4000; pub const O_CLOEXEC: u64 = 0o2000000; /// fcntl commands. pub const F_DUPFD: u64 = 0; pub const F_GETFD: u64 = 1; pub const F_SETFD: u64 = 2; pub const F_GETFL: u64 = 3; pub const F_SETFL: u64 = 4; pub const F_DUPFD_CLOEXEC: u64 = 1030; pub const FD_CLOEXEC: u64 = 1; /// ioctl(FIONBIO, *int): O_NONBLOCK on/off, for any file. pub const FIONBIO: u64 = 0x5421; /// mmap flags. pub const MAP_SHARED: u64 = 0x01; pub const MAP_PRIVATE: u64 = 0x02; pub const MAP_FIXED: u64 = 0x10; pub const MAP_ANONYMOUS: u64 = 0x20; /// clone flags (Linux's values). pub const CLONE_VM: u64 = 0x100; // share the address space pub const CLONE_FS: u64 = 0x200; // share cwd/umask pub const CLONE_FILES: u64 = 0x400; // share the fd table pub const CLONE_SIGHAND: u64 = 0x800; // share signal handlers pub const CLONE_VFORK: u64 = 0x4000; // parent waits until the child execs/exits pub const CLONE_THREAD: u64 = 0x10000; // same process: a thread pub const CLONE_SYSVSEM: u64 = 0x40000; pub const CLONE_SETTLS: u64 = 0x80000; // new thread's FS base = tls argument pub const CLONE_PARENT_SETTID: u64 = 0x100000; // write the tid to *ptid pub const CLONE_CHILD_CLEARTID: u64 = 0x200000; // at exit: *ctid = 0 + futex wake pub const CLONE_DETACHED: u64 = 0x400000; pub const CLONE_CHILD_SETTID: u64 = 0x1000000; // write the tid to *ctid /// arch_prctl codes. pub const ARCH_SET_FS: u64 = 0x1002; pub const ARCH_GET_FS: u64 = 0x1003; pub const SPAWN_TRACE: u64 = 1; /// Framebuffer ioctls (see the table at the top). pub const FB_GET_INFO: u64 = 0x4600; pub const FB_FLUSH: u64 = 0x46f0; pub const FB_FILL: u64 = 0x46f1; pub const FB_SET_CONSOLE: u64 = 0x46f2; /// Name of a syscall number (for tracing). pub fn name(nr: u64) -> &'static str { match nr { SYS_READ => "read", SYS_WRITE => "write", SYS_OPEN => "open", SYS_CLOSE => "close", SYS_STAT => "stat", SYS_FSTAT => "fstat", SYS_LSTAT => "lstat", SYS_ACCESS => "access", SYS_RENAME => "rename", SYS_FCHDIR => "fchdir", SYS_GETUID => "getuid", SYS_GETGID => "getgid", SYS_GETEUID => "geteuid", SYS_GETEGID => "getegid", SYS_CLOCK_NANOSLEEP => "clock_nanosleep", SYS_OPENAT => "openat", SYS_MKDIRAT => "mkdirat", SYS_NEWFSTATAT => "newfstatat", SYS_UNLINKAT => "unlinkat", SYS_RENAMEAT => "renameat", SYS_READLINKAT => "readlinkat", SYS_FACCESSAT => "faccessat", SYS_RENAMEAT2 => "renameat2", SYS_FACCESSAT2 => "faccessat2", SYS_LSEEK => "lseek", SYS_IOCTL => "ioctl", SYS_PIPE => "pipe", SYS_SCHED_YIELD => "sched_yield", SYS_NANOSLEEP => "nanosleep", SYS_POLL => "poll", SYS_PREAD64 => "pread64", SYS_PWRITE64 => "pwrite64", SYS_READV => "readv", SYS_WRITEV => "writev", SYS_SELECT => "select", SYS_SETSID => "setsid", SYS_GETSID => "getsid", SYS_ARCH_PRCTL => "arch_prctl", SYS_GETTID => "gettid", SYS_TKILL => "tkill", SYS_FUTEX => "futex", SYS_EPOLL_CREATE => "epoll_create", SYS_SET_TID_ADDRESS => "set_tid_address", SYS_EPOLL_WAIT => "epoll_wait", SYS_EPOLL_CTL => "epoll_ctl", SYS_TGKILL => "tgkill", SYS_PSELECT6 => "pselect6", SYS_PPOLL => "ppoll", SYS_EPOLL_PWAIT => "epoll_pwait", SYS_EPOLL_CREATE1 => "epoll_create1", SYS_GETPID => "getpid", SYS_EXIT => "exit", SYS_WAIT4 => "wait4", SYS_MMAP => "mmap", SYS_MPROTECT => "mprotect", SYS_MUNMAP => "munmap", SYS_BRK => "brk", SYS_RT_SIGACTION => "rt_sigaction", SYS_RT_SIGPROCMASK => "rt_sigprocmask", SYS_RT_SIGRETURN => "rt_sigreturn", SYS_DUP => "dup", SYS_DUP2 => "dup2", SYS_DUP3 => "dup3", SYS_PAUSE => "pause", SYS_CLONE => "clone", SYS_FORK => "fork", SYS_VFORK => "vfork", SYS_EXECVE => "execve", SYS_KILL => "kill", SYS_FCNTL => "fcntl", SYS_SETPGID => "setpgid", SYS_GETPPID => "getppid", SYS_GETPGRP => "getpgrp", SYS_GETPGID => "getpgid", SYS_EXIT_GROUP => "exit_group", SYS_PIPE2 => "pipe2", SYS_TRACE_ME => "trace_me", SYS_UNAME => "uname", SYS_GETCWD => "getcwd", SYS_CHDIR => "chdir", SYS_MKDIR => "mkdir", SYS_RMDIR => "rmdir", SYS_UNLINK => "unlink", SYS_READLINK => "readlink", SYS_GETDENTS64 => "getdents64", SYS_GETRANDOM => "getrandom", SYS_SPAWN => "spawn", SYS_MOUNTS => "mounts", SYS_SYSINFO => "sysinfo", SYS_SOCKET => "socket", SYS_CONNECT => "connect", SYS_ACCEPT => "accept", SYS_ACCEPT4 => "accept4", SYS_SENDTO => "sendto", SYS_RECVFROM => "recvfrom", SYS_BIND => "bind", SYS_LISTEN => "listen", SYS_SETSOCKOPT => "setsockopt", SYS_CLOCK_GETTIME => "clock_gettime", SYS_RESOLVE => "resolve", SYS_NETINFO => "netinfo", _ => "?", } } /// spawn's options, as laid out in user memory. #[repr(C)] #[derive(Clone, Copy)] pub struct SpawnOptions { pub stdin: i64, // caller's fd to use as the child's fd 0; -1 = caller's fd 0 pub stdout: i64, pub stderr: i64, pub flags: u64, } /// Called by the scheduler when switching threads: the kernel stack that /// `syscall` on this CPU should use from now on. pub fn set_kernel_stack(stack_top: u64) { crate::smp::this_cpu().syscall_kernel_rsp.store(stack_top, Ordering::Relaxed); } /// Program the MSRs that make `syscall` work (on the calling CPU; every CPU /// runs this once). pub fn init() { use x86_64::VirtAddr; use x86_64::registers::model_specific::{Efer, EferFlags, KernelGsBase, LStar, SFMask, Star}; use x86_64::registers::rflags::RFlags; let s = crate::gdt::selectors(); unsafe { // EFER.SCE: enable the syscall/sysret instructions at all. Efer::update(|flags| flags.insert(EferFlags::SYSTEM_CALL_EXTENSIONS)); } // STAR: which segments syscall/sysret load (see gdt.rs for the layout rule). Star::write(s.user_code, s.user_data, s.kernel_code, s.kernel_data) .expect("GDT layout doesn't fit syscall/sysret"); // LSTAR: where `syscall` jumps to. LStar::write(VirtAddr::new(syscall_entry as *const () as u64)); // SFMASK: RFLAGS bits to clear on entry. IF: no interrupts until we're on // the kernel stack. DF: Rust expects the direction flag clear. TF: no // single-stepping into the kernel. SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::DIRECTION_FLAG | RFlags::TRAP_FLAG); // KERNEL_GS_BASE: this CPU's PerCpu block, for `swapgs` in the entry. KernelGsBase::write(VirtAddr::from_ptr(crate::smp::this_cpu())); } unsafe extern "C" { fn syscall_entry(); } global_asm!( ".global syscall_entry", "syscall_entry:", // Interrupts are off (SFMASK). Switch to this thread's kernel stack, // found in this CPU's PerCpu block (gs:[0]; gs:[8] is scratch). " swapgs", " mov gs:[8], rsp", " mov rsp, gs:[0]", // Build a TrapFrame (trap.rs), the same layout an interrupt makes: " push 0", // ss (filled in by syscall_dispatch) " push qword ptr gs:[8]", // user rsp " swapgs", // GS back to normal right away " push r11", // user rflags (saved there by `syscall`) " push 0", // cs (filled in by syscall_dispatch) " push rcx", // user rip (saved there by `syscall`) " push rax", // `error` slot: the syscall number " push 0x100", // `vector`: TRAP_SYSCALL " push rax", " push rbx", " push rcx", " push rdx", " push rsi", " push rdi", " push rbp", " push r8", " push r9", " push r10", " push r11", " push r12", " push r13", " push r14", " push r15", " mov rdi, rsp", // syscall_dispatch(&mut TrapFrame) " sti", // we're on our own stack now: preemption is fine " call {dispatch}", // Back to user mode with `iretq` through the common exit (which // restores every register from the frame). Not `sysretq`: fork, execve, // signals and rt_sigreturn may have changed *any* register, and // sysret can't set rcx and r11. " jmp {ret}", dispatch = sym syscall_dispatch, ret = sym crate::trap::trap_return, ); /// Rust side of every system call. Arguments are in the saved registers /// (rdi, rsi, rdx, r10, r8, r9 = a1..a6), the number in `error` (= rax at /// entry); the result goes into the saved rax. extern "C" fn syscall_dispatch(frame: &mut TrapFrame) { let sel = crate::gdt::selectors(); frame.cs = sel.user_code.0 as u64; frame.ss = sel.user_data.0 as u64; let nr = frame.error; let (a1, a2, a3, a4, a5) = (frame.rdi, frame.rsi, frame.rdx, frame.r10, frame.r8); let process = thread::current_process(); let trace = process.as_ref().is_some_and(|p| p.tracing()); if trace && (nr == SYS_EXIT || nr == SYS_EXIT_GROUP) { crate::kprintln!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64); } drop(process); // don't hold an Arc while blocking/exiting // A successful execve throws away the memory its path lived in, so the // trace line for it is made before the call. let exec_line = (trace && nr == SYS_EXECVE).then(|| trace_line(nr, [a1, a2, a3, a4], 0)); if nr == SYS_RT_SIGRETURN { // Restores *all* registers (rax included) from the signal frame. signal::sys_rt_sigreturn(frame); signal::deliver(frame); // signals unblocked by the restored mask return; } let result = match nr { SYS_READ => sys_read(a1, a2, a3), SYS_WRITE => sys_write(a1, a2, a3), SYS_OPEN => sys_openat(AT_FDCWD as u64, a1, a2), SYS_OPENAT => sys_openat(a1, a2, a3), SYS_CLOSE => current().and_then(|p| p.close(a1)).map(|_| 0), SYS_STAT => sys_newfstatat(AT_FDCWD as u64, a1, a2, 0), SYS_LSTAT => sys_newfstatat(AT_FDCWD as u64, a1, a2, AT_SYMLINK_NOFOLLOW), SYS_FSTAT => sys_fstat(a1, a2), SYS_NEWFSTATAT => sys_newfstatat(a1, a2, a3, a4), SYS_ACCESS => sys_faccessat(AT_FDCWD as u64, a1), SYS_FACCESSAT | SYS_FACCESSAT2 => sys_faccessat(a1, a2), SYS_LSEEK => current().and_then(|p| p.file(a1)).and_then(|f| f.seek(a2 as i64, a3)), SYS_MMAP => current().and_then(|p| sys_mmap(&p, a1, a2, a3, a4, a5, frame.r9)), SYS_MPROTECT => current().and_then(|p| sys_mprotect(&p, a1, a2, a3)), SYS_MUNMAP => current().and_then(|p| sys_munmap(&p, a1, a2)), SYS_BRK => current().map(|p| p.mm.lock().set_brk(a1)), SYS_RT_SIGACTION => sys_rt_sigaction(a1, a2, a3, a4), SYS_RT_SIGPROCMASK => sys_rt_sigprocmask(a1, a2, a3, a4), SYS_IOCTL => sys_ioctl(a1, a2, a3), SYS_PIPE => sys_pipe2(a1, 0), SYS_PIPE2 => sys_pipe2(a1, a2), SYS_SCHED_YIELD => { thread::yield_now(); Ok(0) } SYS_DUP => current().and_then(|p| p.file(a1).and_then(|f| p.add_file(f))), SYS_DUP2 => current().and_then(|p| if a1 == a2 { p.file(a1).map(|_| a1) } else { p.dup_to(a1, a2, false) }), SYS_DUP3 => sys_dup3(a1, a2, a3), SYS_PAUSE => Err(thread::pause()), SYS_NANOSLEEP => sys_nanosleep(a1, a2), SYS_GETPID => current().map(|p| p.pid), SYS_CLONE => sys_clone(frame, a1, a2, a3, a4, a5), SYS_FORK => current().and_then(|p| process::fork(&p, frame, 0, false)), SYS_VFORK => current().and_then(|p| process::fork(&p, frame, 0, true)), SYS_EXECVE => sys_execve(frame, a1, a2, a3), // exit ends only the calling thread (the process ends with its // last thread); exit_group ends them all. libc's exit() is the latter. SYS_EXIT => process::exit_thread(process::ExitStatus::Exited(a1 as i32 as i64)), SYS_EXIT_GROUP => process::exit_group(process::ExitStatus::Exited(a1 as i32 as i64)), SYS_WAIT4 => sys_wait4(a1 as i32 as i64, a2, a3, a4), SYS_KILL => current().and_then(|p| signal::sys_kill(&p, a1 as i32 as i64, a2)), SYS_UNAME => sys_uname(a1), SYS_FCNTL => sys_fcntl(a1, a2, a3), SYS_GETCWD => sys_getcwd(a1, a2), SYS_CHDIR => user_path(a1).and_then(|p| sys_chdir(p)), SYS_FCHDIR => sys_fchdir(a1), SYS_MKDIR => sys_mkdirat(AT_FDCWD as u64, a1), SYS_MKDIRAT => sys_mkdirat(a1, a2), SYS_RMDIR => sys_unlinkat(AT_FDCWD as u64, a1, AT_REMOVEDIR), SYS_UNLINK => sys_unlinkat(AT_FDCWD as u64, a1, 0), SYS_UNLINKAT => sys_unlinkat(a1, a2, a3), SYS_RENAME => sys_renameat(AT_FDCWD as u64, a1, AT_FDCWD as u64, a2, 0), SYS_RENAMEAT => sys_renameat(a1, a2, a3, a4, 0), SYS_RENAMEAT2 => sys_renameat(a1, a2, a3, a4, a5), SYS_READLINK => sys_readlinkat(AT_FDCWD as u64, a1, a2, a3), SYS_READLINKAT => sys_readlinkat(a1, a2, a3, a4), SYS_SETPGID => current().and_then(|p| process::setpgid(&p, a1, a2)).map(|_| 0), SYS_GETPPID => current().map(|p| if p.parent() == process::ORPHAN { 0 } else { p.parent() }), SYS_GETPGRP => current().map(|p| p.pgid()), SYS_GETPGID => current().and_then(|p| if a1 == 0 { Ok(p.pgid()) } else { process::get(a1).map(|t| t.pgid()).ok_or(Errno::ESRCH) }), SYS_GETDENTS64 => sys_getdents64(a1, a2, a3), SYS_SPAWN => sys_spawn(a1, a2, a3, a4), SYS_MOUNTS => sys_mounts(a1, a2), SYS_SYSINFO => sys_sysinfo(a1), SYS_SOCKET => sys_socket(a1, a2, a3), SYS_CONNECT => with_socket(a1, |s| s.connect(read_sockaddr(a2)?)).map(|_| 0), SYS_ACCEPT => sys_accept(a1, a2, a3, 0), SYS_ACCEPT4 => sys_accept(a1, a2, a3, a4), SYS_SENDTO => sys_sendto(a1, a2, a3, a5), SYS_RECVFROM => sys_recvfrom(a1, a2, a3, a5, frame.r9), SYS_BIND => with_socket(a1, |s| s.bind(read_sockaddr(a2)?.1)).map(|_| 0), SYS_LISTEN => with_socket(a1, |s| s.listen(a2 as usize)).map(|_| 0), SYS_SETSOCKOPT => sys_setsockopt(a1, a2, a3, a4, a5), SYS_CLOCK_GETTIME => sys_clock_gettime(a1, a2), SYS_CLOCK_NANOSLEEP => sys_clock_nanosleep(a1, a2, a3, a4), // One user, root: musl asks at startup (and `id`, `ls -l` ask). SYS_GETUID | SYS_GETGID | SYS_GETEUID | SYS_GETEGID => Ok(0), SYS_RESOLVE => sys_resolve(a1, a2, a3, a4), SYS_NETINFO => copy_to_user(a1, &crate::net::info()).map(|_| 0), SYS_GETRANDOM => sys_getrandom(a1, a2), SYS_TRACE_ME => current().map(|p| p.trace.store(a1 != 0, Ordering::Relaxed)).map(|_| 0), // --- stage 15 --- SYS_PREAD64 => sys_pread(a1, a2, a3, a4), SYS_PWRITE64 => sys_pwrite(a1, a2, a3, a4), SYS_READV => sys_readv(a1, a2, a3), SYS_WRITEV => sys_writev(a1, a2, a3), SYS_POLL => crate::poll::sys_poll_ms(a1, a2, a3 as i32 as i64), SYS_PPOLL => crate::poll::sys_ppoll(a1, a2, a3, a4), SYS_SELECT => crate::poll::sys_select_tv(a1, a2, a3, a4, a5), SYS_PSELECT6 => crate::poll::sys_pselect6(a1, a2, a3, a4, a5, frame.r9), SYS_EPOLL_CREATE => if (a1 as i32) <= 0 { Err(Errno::EINVAL) } else { crate::poll::sys_epoll_create1(0) }, SYS_EPOLL_CREATE1 => crate::poll::sys_epoll_create1(a1), SYS_EPOLL_CTL => crate::poll::sys_epoll_ctl(a1, a2, a3, a4), SYS_EPOLL_WAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, 0), SYS_EPOLL_PWAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, a5), SYS_FUTEX => current().and_then(|p| crate::futex::sys_futex(&p, a1, a2, a3, a4, a5, frame.r9)), SYS_GETTID => task().map(|t| t.tid()), SYS_SET_TID_ADDRESS => task().map(|t| { t.clear_child_tid.store(a1, Ordering::Relaxed); t.tid() }), SYS_ARCH_PRCTL => sys_arch_prctl(a1, a2), SYS_TGKILL => sys_tgkill(Some(a1), a2, a3), SYS_TKILL => sys_tgkill(None, a1, a2), SYS_SETSID => current().and_then(|p| process::setsid(&p)), SYS_GETSID => current().and_then(|p| if a1 == 0 { Ok(p.sid()) } else { process::get(a1).map(|t| t.sid()).ok_or(Errno::ESRCH) }), _ => Err(Errno::ENOSYS), }; let ret = match result { Ok(value) => value as i64, Err(errno) => -(errno as i64), }; // execve: success means "the new program starts with these registers" // (rax = 0 there anyway); the frame was replaced, so don't touch it. if !(nr == SYS_EXECVE && ret == 0) { frame.rax = ret as u64; } if trace { let pid = current().map(|p| p.pid).unwrap_or(0); match exec_line { Some(line) if ret == 0 => crate::kprintln!("[trace {}] {}", pid, line), _ => crate::kprintln!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret)), } } // A signal may have arrived meanwhile (or the call was interrupted by // one): act on it before returning to the program. signal::deliver(frame); } /// One syscall in strace style: `open("/mnt/disk0/readme.txt", 0) = 3`. /// Strings and buffers are read back from user memory (after the call, so /// `read` shows what it got), shortened to 40 bytes. fn trace_line(nr: u64, a: [u64; 4], ret: i64) -> String { use alloc::format; let text = |ptr: u64, len: u64| -> String { let Ok(bytes) = user_slice(ptr, len.min(40)) else { return String::from("") }; let mut s = String::from("\""); for &b in bytes { match b { b'\n' => s.push_str("\\n"), b'"' => s.push_str("\\\""), 0x20..=0x7e => s.push(b as char), _ => s.push_str(&format!("\\x{:02x}", b)), } } s.push('"'); if len > 40 { s.push_str("..."); } s }; let cstr = |ptr: u64| -> String { match user_cstr(ptr, 4096) { Ok(bytes) => format!("\"{}\"", String::from_utf8_lossy(&bytes)), Err(_) => String::from(""), } }; let args = match nr { SYS_READ => format!("{}, {}, {}", a[0], if ret > 0 { text(a[1], ret as u64) } else { String::from("\"\"") }, a[2]), SYS_WRITE => format!("{}, {}, {}", a[0], text(a[1], a[2]), a[2]), // Paths are C strings (stage 15.1); the *at() calls show the dirfd // (-100 = AT_FDCWD) first, like strace. SYS_OPEN => format!("{}, {:#o}", cstr(a[0]), a[1]), SYS_OPENAT => format!("{}, {}, {:#o}", a[0] as i32, cstr(a[1]), a[2]), SYS_STAT | SYS_LSTAT => format!("{}, {:#x}", cstr(a[0]), a[1]), SYS_NEWFSTATAT => format!("{}, {}, {:#x}, {:#x}", a[0] as i32, cstr(a[1]), a[2], a[3]), SYS_READLINK => format!("{}, {:#x}, {}", cstr(a[0]), a[1], a[2]), SYS_CHDIR | SYS_MKDIR | SYS_RMDIR | SYS_UNLINK | SYS_ACCESS => cstr(a[0]), SYS_MKDIRAT | SYS_UNLINKAT | SYS_FACCESSAT | SYS_READLINKAT => format!("{}, {}", a[0] as i32, cstr(a[1])), SYS_RENAME => format!("{}, {}", cstr(a[0]), cstr(a[1])), SYS_SPAWN => format!("{}, {}", cstr(a[0]), text(a[1], a[2])), SYS_SCHED_YIELD | SYS_GETPID => String::new(), SYS_CLOSE | SYS_EXIT | SYS_EXIT_GROUP | SYS_DUP => format!("{}", a[0] as i64), SYS_PIPE | SYS_UNAME => format!("{:#x}", a[0]), SYS_FSTAT | SYS_GETCWD | SYS_MOUNTS => format!("{}, {:#x}", a[0], a[1]), SYS_WAIT4 => format!("{}, {:#x}, {}", a[0] as i32, a[1], a[2]), SYS_KILL => format!("{}, {}", a[0] as i32, a[1]), SYS_FORK | SYS_VFORK | SYS_GETPPID | SYS_GETPGRP | SYS_PAUSE => String::new(), SYS_EXECVE => format!("{}, ...", cstr(a[0])), SYS_MMAP => format!("{:#x}, {}, {}, {:#x}", a[0], a[1], a[2], a[3]), _ => format!("{}, {:#x}, {:#x}", a[0], a[1], a[2]), }; match Errno::from_ret(ret) { Some(e) if ret < 0 => format!("{}({}) = {} {:?}", name(nr), args, ret, e), _ => format!("{}({}) = {}", name(nr), args, ret), } } fn current() -> Result, Errno> { thread::current_process().ok_or(Errno::EINVAL) } /// The calling thread. fn task() -> Result, Errno> { thread::current_task().ok_or(Errno::EINVAL) } /// Read a `struct timespec {i64 seconds, i64 nanoseconds}` as microseconds /// (rounded up, so a timeout never ends early). pub fn read_timespec_us(ptr: u64) -> Result { let b = user_slice(ptr, 16)?; let secs = i64::from_le_bytes(b[..8].try_into().unwrap()); let nsecs = i64::from_le_bytes(b[8..].try_into().unwrap()); if secs < 0 || !(0..1_000_000_000).contains(&nsecs) { return Err(Errno::EINVAL); } Ok((secs as u64).saturating_mul(1_000_000).saturating_add((nsecs as u64).div_ceil(1000))) } /// Turn a user pointer + length into a slice, after checking that the user /// may really access that memory. Never trust pointers from user space! pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> { if len == 0 { return Ok(&[]); // nothing to access, so any pointer is fine (like Linux) } if !crate::mm::user_access_ok(ptr, len, false) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) }) } /// Like `user_slice`, but the user must be allowed to *write* the memory. pub fn user_slice_mut(ptr: u64, len: u64) -> Result<&'static mut [u8], Errno> { if len == 0 { return Ok(&mut []); } if !crate::mm::user_access_ok(ptr, len, true) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len as usize) }) } fn sys_read(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; // clone the Arc: no lock held while we block let buf = user_slice_mut(buf, len)?; file.read(buf).map(|n| n as u64) } fn sys_write(fd: u64, buf: u64, len: u64) -> Result { let process = current()?; let file = process.file(fd)?; let buf = user_slice(buf, len)?; let result = file.write(buf).map(|n| n as u64); if result == Err(Errno::EPIPE) { // Nobody will ever read this: SIGPIPE, which by default ends the // writer. That's how `yes | head -1` stops `yes`. signal::send(&process, signal::SIGPIPE, signal::SigInfo::from_pid(signal::SIGPIPE, 0)); } result } // --- Paths (stage 15.1: Linux's C-string ABI) -------------------------------------- // // A path argument is a pointer to a NUL-terminated string, like in C (up // to stage 15 we took a pointer and a length, which no libc does). We copy // it in page by page with `user_cstr`, so a bad pointer, or a string that // runs into an unmapped page, is EFAULT rather than a kernel crash, and at // most PATH_MAX bytes (with the NUL): longer is ENAMETOOLONG. // // The *at() variants take a directory fd first: a relative path starts // there instead of at the cwd (AT_FDCWD = "the cwd", what the old calls // mean). musl uses them for stat (newfstatat), opendir is open with // O_DIRECTORY, and so on. /// The directory a relative path in an *at() call starts from. fn at_base(dirfd: u64) -> Result { let p = current()?; if dirfd as i32 as i64 == AT_FDCWD { return Ok(p.cwd.lock().clone()); } let file = p.file(dirfd)?; // EBADF match file.path() { Some(path) if file.stat()?.kind == crate::fs::file::KIND_DIR => Ok(path), _ => Err(Errno::ENOTDIR), } } /// A path string (already copied in) made absolute, relative to `dirfd`. fn path_at(dirfd: u64, raw: &[u8]) -> Result { // Our file systems use UTF-8 names (Linux allows any bytes but '/' and NUL). let path = core::str::from_utf8(raw).map_err(|_| Errno::EINVAL)?; if path.is_empty() { return Err(Errno::ENOENT); // like Linux: "" is not a path } if path.starts_with('/') { return fs::normalize("/", path); // absolute: dirfd doesn't matter } fs::normalize(&at_base(dirfd)?, path) } /// Copy in a path argument and make it absolute, relative to `dirfd`. fn user_path_at(dirfd: u64, ptr: u64) -> Result { path_at(dirfd, &user_cstr(ptr, PATH_MAX - 1)?) } /// Copy in a path argument, relative to the cwd. fn user_path(ptr: u64) -> Result { user_path_at(AT_FDCWD as u64, ptr) } /// open / openat(dirfd, path, flags, mode). Everything is readable and /// writable whatever O_RDONLY/O_WRONLY say, and `mode` is ignored (new /// files get the file system's default permissions). fn sys_openat(dirfd: u64, path_ptr: u64, flags: u64) -> Result { let path = user_path_at(dirfd, path_ptr)?; let found = if flags & O_NOFOLLOW != 0 { fs::lookup_nofollow(&path) } else { fs::lookup(&path) }; let inode = match found { Err(Errno::ENOENT) if flags & O_CREAT != 0 => fs::create(&path, InodeKind::File)?, Ok(_) if flags & O_CREAT != 0 && flags & O_EXCL != 0 => return Err(Errno::EEXIST), Ok(i) if flags & O_NOFOLLOW != 0 && i.kind() == InodeKind::Symlink => return Err(Errno::ELOOP), other => other?, }; let is_dir = inode.kind() == InodeKind::Directory; if flags & O_DIRECTORY != 0 && !is_dir { return Err(Errno::ENOTDIR); // opendir() on a file } if is_dir && flags & O_ACCMODE != 0 { return Err(Errno::EISDIR); // directories open read-only } // Device files (/dev/...) come with their own File. let cloexec = flags & O_CLOEXEC != 0; if let Some(device) = inode.open_device()? { let p = current()?; if flags & O_NONBLOCK != 0 { device.set_nonblocking(true); } if flags & O_NOCTTY == 0 { crate::tty::opened(&p, &*device); // maybe our controlling terminal now } return p.add_file_with(device, 0, cloexec); } if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File { inode.truncate()?; } let file = if flags & O_APPEND != 0 { InodeFile::open_append(inode, path)? } else { InodeFile::open(inode, path)? }; current()?.add_file_with(Arc::new(file), 0, cloexec) } /// newfstatat(dirfd, path, *stat, flags): stat, lstat (AT_SYMLINK_NOFOLLOW: /// a symlink itself, not what it points to) and, with AT_EMPTY_PATH and /// path "", fstat(dirfd). Writes Linux's `struct stat`. fn sys_newfstatat(dirfd: u64, path_ptr: u64, stat_ptr: u64, flags: u64) -> Result { let raw = user_cstr(path_ptr, PATH_MAX - 1)?; if raw.is_empty() && flags & AT_EMPTY_PATH != 0 && dirfd as i32 as i64 != AT_FDCWD { return sys_fstat(dirfd, stat_ptr); } let path = if raw.is_empty() && flags & AT_EMPTY_PATH != 0 { at_base(dirfd)? } else { path_at(dirfd, &raw)? }; let inode = if flags & AT_SYMLINK_NOFOLLOW != 0 { fs::lookup_nofollow(&path)? } else { fs::lookup(&path)? }; let stat = LinuxStat::new(&Stat::of(&*inode), fs::file::ino_hash(path.as_bytes())); copy_to_user(stat_ptr, &stat)?; Ok(0) } fn sys_fstat(fd: u64, stat_ptr: u64) -> Result { let file = current()?.file(fd)?; // Files opened by path: the same inode number stat() gives. Others // (pipes, sockets, consoles): the open file's address, unique while open. let ino = match file.path() { Some(path) => fs::file::ino_hash(path.as_bytes()), None => Arc::as_ptr(&file) as *const u8 as u64, }; copy_to_user(stat_ptr, &LinuxStat::new(&file.stat()?, ino))?; Ok(0) } /// access / faccessat(dirfd, path, mode): "does it exist?" There are no /// permissions to check yet (we're root), so existing is enough. fn sys_faccessat(dirfd: u64, path_ptr: u64) -> Result { fs::lookup(&user_path_at(dirfd, path_ptr)?)?; Ok(0) } /// mkdir / mkdirat(dirfd, path, mode); `mode` is ignored. fn sys_mkdirat(dirfd: u64, path_ptr: u64) -> Result { fs::create(&user_path_at(dirfd, path_ptr)?, InodeKind::Directory)?; Ok(0) } /// unlink / rmdir / unlinkat(dirfd, path, flags): AT_REMOVEDIR = rmdir. fn sys_unlinkat(dirfd: u64, path_ptr: u64, flags: u64) -> Result { if flags & !AT_REMOVEDIR != 0 { return Err(Errno::EINVAL); } fs::remove(&user_path_at(dirfd, path_ptr)?, flags & AT_REMOVEDIR != 0)?; Ok(0) } /// rename / renameat / renameat2 (no flags: RENAME_NOREPLACE & co. are /// EINVAL). See `fs::rename` for what we can and can't move. fn sys_renameat(olddir: u64, old_ptr: u64, newdir: u64, new_ptr: u64, flags: u64) -> Result { let old = user_path_at(olddir, old_ptr)?; let new = user_path_at(newdir, new_ptr)?; if flags != 0 { return Err(Errno::EINVAL); } fs::rename(&old, &new)?; Ok(0) } fn sys_chdir(path: String) -> Result { if fs::lookup(&path)?.kind() != InodeKind::Directory { return Err(Errno::ENOTDIR); } *current()?.cwd.lock() = path; Ok(0) } /// fchdir(fd): cd to a directory we have open. fn sys_fchdir(fd: u64) -> Result { sys_chdir(at_base(fd)?) } /// readlink / readlinkat(dirfd, path, buf, size): the link's target, *no* /// NUL added (like Linux); returns its length (cut to `size`). fn sys_readlinkat(dirfd: u64, path_ptr: u64, buf: u64, len: u64) -> Result { let path = user_path_at(dirfd, path_ptr)?; if len as i64 <= 0 { return Err(Errno::EINVAL); } // There's no /proc yet, but libcs and programs ask "where am I?" // with readlink("/proc/self/exe"): answer just that one. let target = if path == "/proc/self/exe" { current()?.exe.lock().clone() } else { let inode = fs::lookup_nofollow(&path)?; if inode.kind() != InodeKind::Symlink { return Err(Errno::EINVAL); // not a symlink } inode.readlink()? }; let n = target.len().min(len as usize); user_slice_mut(buf, n as u64)?.copy_from_slice(&target.as_bytes()[..n]); Ok(n as u64) } /// Copy a plain-data struct out to user memory. pub fn copy_to_user(ptr: u64, value: &T) -> Result<(), Errno> { let size = core::mem::size_of::(); let dst = user_slice_mut(ptr, size as u64)?; let src = unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size) }; dst.copy_from_slice(src); Ok(()) } /// Copy `N` u32s in from user memory (ioctl arguments). fn read_user_u32s(ptr: u64) -> Result<[u32; N], Errno> { let src = user_slice(ptr, 4 * N as u64)?; let mut out = [0u32; N]; for (i, word) in out.iter_mut().enumerate() { *word = u32::from_le_bytes(src[4 * i..4 * i + 4].try_into().unwrap()); } Ok(out) } fn sys_pipe2(fds_ptr: u64, flags: u64) -> Result { if flags & !(O_CLOEXEC | O_NONBLOCK) != 0 { return Err(Errno::EINVAL); // O_DIRECT (packet mode): not supported } let cloexec = flags & O_CLOEXEC != 0; user_slice_mut(fds_ptr, 8)?; // check before creating anything let process = current()?; let (reader, writer) = fs::pipe::new(); if flags & O_NONBLOCK != 0 { reader.set_nonblocking(true); writer.set_nonblocking(true); } let read_fd = process.add_file_with(Arc::new(reader), 0, cloexec)?; let write_fd = match process.add_file_with(Arc::new(writer), 0, cloexec) { Ok(fd) => fd, Err(e) => { process.close(read_fd)?; return Err(e); } }; copy_to_user(fds_ptr, &[read_fd as u32, write_fd as u32])?; Ok(0) } fn sys_dup3(old: u64, new: u64, flags: u64) -> Result { if old == new || flags & !O_CLOEXEC != 0 { return Err(Errno::EINVAL); } current()?.dup_to(old, new, flags & O_CLOEXEC != 0) } /// fcntl: the fd flags (close-on-exec), duplication, and the one status /// flag we have: O_NONBLOCK (it belongs to the open file, so it's shared /// with dup'ed fds). Everything is opened read/write. fn sys_fcntl(fd: u64, cmd: u64, arg: u64) -> Result { let p = current()?; let file = p.file(fd)?; match cmd { F_DUPFD | F_DUPFD_CLOEXEC if arg as usize >= process::MAX_FDS => Err(Errno::EINVAL), F_DUPFD => p.add_file_with(file, arg as usize, false), F_DUPFD_CLOEXEC => p.add_file_with(file, arg as usize, true), F_GETFD => p.cloexec(fd).map(|on| if on { FD_CLOEXEC } else { 0 }), F_SETFD => p.set_cloexec(fd, arg & FD_CLOEXEC != 0).map(|_| 0), F_GETFL => Ok(2 | if file.nonblocking() { O_NONBLOCK } else { 0 }), // O_RDWR F_SETFL => { file.set_nonblocking(arg & O_NONBLOCK != 0); Ok(0) } _ => Err(Errno::EINVAL), } } /// getcwd: like Linux, the path with a NUL; returns the length + 1. fn sys_getcwd(buf: u64, len: u64) -> Result { let mut cwd = current()?.cwd.lock().clone(); cwd.push('\0'); if (len as usize) < cwd.len() { return Err(Errno::ERANGE); } user_slice_mut(buf, cwd.len() as u64)?.copy_from_slice(cwd.as_bytes()); Ok(cwd.len() as u64) } fn sys_getdents64(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; file.getdents(user_slice_mut(buf, len)?).map(|n| n as u64) } /// spawn(path, argv blob, blob length, *options): our own call (500). fn sys_spawn(path_ptr: u64, argv_ptr: u64, argv_len: u64, opts_ptr: u64) -> Result { let parent = current()?; let path = user_path(path_ptr)?; if argv_len as usize > process::MAX_ARGS_SIZE { return Err(Errno::E2BIG); } let blob = core::str::from_utf8(user_slice(argv_ptr, argv_len)?).map_err(|_| Errno::EINVAL)?; let mut args: Vec = blob.split('\0').filter(|a| !a.is_empty()).map(String::from).collect(); if args.is_empty() { args.push(String::from(path.rsplit('/').next().unwrap_or(""))); } let opts = if opts_ptr == 0 { SpawnOptions { stdin: -1, stdout: -1, stderr: -1, flags: 0 } } else { let bytes = user_slice(opts_ptr, core::mem::size_of::() as u64)?; unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const SpawnOptions) } }; // Child fd n = the caller's fd chosen in the options (-1: the caller's fd n). let pick = |n: u64, choice: i64| parent.file(if choice < 0 { n } else { choice as u64 }); let stdio = [pick(0, opts.stdin)?, pick(1, opts.stdout)?, pick(2, opts.stderr)?]; let spawn = process::SpawnArgs { args, env: parent.env(), stdio, cwd: parent.cwd.lock().clone(), parent: parent.pid, pgid: parent.pgid(), // same process group, like fork sid: parent.sid(), // ...and session ctty: parent.ctty.lock().clone(), trace: opts.flags & SPAWN_TRACE != 0, }; drop(parent); process::spawn_path(&path, spawn).map(|child| child.pid) } /// ioctl: FIONBIO for any file; the rest is up to the file (terminals: /// tty.rs; the framebuffer: below). fn sys_ioctl(fd: u64, request: u64, arg: u64) -> Result { let file = current()?.file(fd)?; if let Some(fb) = file.as_framebuffer() { return fb_ioctl(fb, request, arg); } if request == FIONBIO { file.set_nonblocking(read_user_u32s::<1>(arg)?[0] != 0); return Ok(0); } file.ioctl(request, arg) } /// The /dev/fb0 ioctls (table at the top of this file). fn fb_ioctl(fb: &crate::dev::Framebuffer, request: u64, arg: u64) -> Result { let info = fb.info; match request { FB_GET_INFO => { copy_to_user(arg, &[info.width, info.height, info.stride * 4, 32u32])?; Ok(0) } FB_FLUSH if arg == 0 => fb.flush().then_some(0).ok_or(Errno::EIO), FB_FLUSH => { let [x, y, w, h] = read_user_u32s::<4>(arg)?; crate::virtio::gpu::flush(x, y, w, h).then_some(0).ok_or(Errno::EIO) } FB_FILL => { let [x, y, w, h, color] = read_user_u32s::<5>(arg)?; fb.fill(x, y, w, h, color); Ok(0) } FB_SET_CONSOLE => { let [x, y, w, h] = if arg == 0 { [0; 4] } else { read_user_u32s::<4>(arg)? }; let (cols, rows) = crate::vga_buffer::set_viewport(crate::fbcon::Rect { x, y, w, h }).ok_or(Errno::ENODEV)?; Ok((cols as u64) << 16 | rows as u64) } _ => Err(Errno::EINVAL), } } /// getrandom(buf, len, flags): Linux 318. We always have "enough entropy" /// (the host's), so the flags (GRND_NONBLOCK, GRND_RANDOM) change nothing. fn sys_getrandom(buf: u64, len: u64) -> Result { let buf = user_slice_mut(buf, len.min(65536))?; crate::virtio::rng::fill(buf); Ok(buf.len() as u64) } /// Linux's `struct utsname`: six NUL-terminated strings of 65 bytes each. fn sys_uname(buf: u64) -> Result { let fields = [ "Rustbelt", // sysname (`uname -s`) "qemu", // nodename (hostname) env!("CARGO_PKG_VERSION"), // release (`uname -r`) "stage 15: libc plumbing", // version "x86_64", // machine "(none)", // domainname ]; let out = user_slice_mut(buf, 6 * 65)?; out.fill(0); for (i, field) in fields.iter().enumerate() { out[i * 65..i * 65 + field.len()].copy_from_slice(field.as_bytes()); } Ok(0) } /// What `sysinfo` fills in (ours; the user side is in user/ulib.rs). #[repr(C)] #[derive(Default)] pub struct SysInfo { pub cpus: u64, pub this_cpu: u64, pub threads: u64, pub processes: u64, pub frames_in_use: u64, pub uptime_ticks: u64, /// Per CPU: local-timer ticks so far, and how many found it busy /// (running something other than its idle thread). pub ticks: [u64; 16], pub busy_ticks: [u64; 16], } fn sys_sysinfo(buf: u64) -> Result { let out = user_slice_mut(buf, core::mem::size_of::() as u64)?; let mut info = SysInfo { cpus: crate::smp::cpu_count() as u64, this_cpu: crate::smp::cpu_id() as u64, threads: thread::thread_count() as u64, processes: process::count() as u64, frames_in_use: memory::frames_in_use() as u64, uptime_ticks: crate::interrupts::ticks(), ..Default::default() }; for cpu in 0..info.cpus as usize { (info.ticks[cpu], info.busy_ticks[cpu]) = thread::cpu_ticks(cpu); } let bytes = unsafe { core::slice::from_raw_parts(&info as *const SysInfo as *const u8, core::mem::size_of::()) }; out.copy_from_slice(bytes); Ok(0) } /// The mount table as text, one "path fstype" line per mount. fn sys_mounts(buf: u64, len: u64) -> Result { let mut text = String::new(); for (path, fs_type) in fs::mounts() { text.push_str(&path); text.push(' '); text.push_str(fs_type); text.push('\n'); } if (len as usize) < text.len() { return Err(Errno::ERANGE); } user_slice_mut(buf, text.len() as u64)?.copy_from_slice(text.as_bytes()); Ok(text.len() as u64) } // --- Networking (net/socket.rs does the real work) ----------------------------- /// Linux's `struct sockaddr_in`: family, port and address in network byte /// order (big-endian), 8 bytes of padding. #[repr(C)] #[derive(Clone, Copy, Default)] pub struct SockAddrIn { pub family: u16, pub port_be: u16, pub addr: [u8; 4], pub zero: [u8; 8], } fn read_sockaddr(ptr: u64) -> Result { let bytes = user_slice(ptr, 16)?; let family = u16::from_ne_bytes([bytes[0], bytes[1]]); if family as u64 != AF_INET { return Err(Errno::EAFNOSUPPORT); } Ok(([bytes[4], bytes[5], bytes[6], bytes[7]], u16::from_be_bytes([bytes[2], bytes[3]]))) } /// Write a peer's address like Linux: `*len_ptr` (a u32) says how much /// room there is; we write at most that much and set it to the real size /// (16). No `len_ptr` (only our own older programs do that): all 16 bytes. fn write_sockaddr(ptr: u64, len_ptr: u64, (addr, port): crate::net::socket::Addr) -> Result<(), Errno> { if ptr == 0 { return Ok(()); // caller doesn't want it } let sa = SockAddrIn { family: AF_INET as u16, port_be: port.to_be(), addr, zero: [0; 8] }; let bytes = unsafe { core::slice::from_raw_parts(&sa as *const SockAddrIn as *const u8, 16) }; let room = if len_ptr == 0 { 16 } else { u32::from_le_bytes(user_slice(len_ptr, 4)?.try_into().unwrap()).min(16) as usize }; user_slice_mut(ptr, room as u64)?.copy_from_slice(&bytes[..room]); if len_ptr != 0 { copy_to_user(len_ptr, &16u32)?; } Ok(()) } /// Run `f` on the socket behind `fd` (ENOTSOCK if it's another kind of /// file). We hold our own Arc to it, so the fd may be closed meanwhile. fn with_socket(fd: u64, f: impl FnOnce(&crate::net::socket::Socket) -> Result) -> Result { let file = current()?.file(fd)?; f(file.as_socket().ok_or(Errno::ENOTSOCK)?) } fn sys_socket(domain: u64, ty: u64, protocol: u64) -> Result { use crate::net::socket::{Kind, Socket}; if domain != AF_INET { return Err(Errno::EAFNOSUPPORT); } let kind = match (ty & 0xf, protocol) { (SOCK_STREAM, 0 | 6) => Kind::Tcp, (SOCK_DGRAM, 0 | 17) => Kind::Udp, (SOCK_DGRAM, IPPROTO_ICMP) => Kind::Icmp, _ => return Err(Errno::EPROTONOSUPPORT), }; // SOCK_NONBLOCK and SOCK_CLOEXEC have O_NONBLOCK's and O_CLOEXEC's values. let socket = Socket::new(kind)?; socket.set_nonblocking(ty & O_NONBLOCK != 0); current()?.add_file_with(Arc::new(socket), 0, ty & O_CLOEXEC != 0) } /// accept(fd, *addr, *len) and accept4(..., flags: SOCK_NONBLOCK|SOCK_CLOEXEC). fn sys_accept(fd: u64, addr_ptr: u64, len_ptr: u64, flags: u64) -> Result { let (socket, peer) = with_socket(fd, |s| s.accept())?; socket.set_nonblocking(flags & O_NONBLOCK != 0); let new_fd = current()?.add_file_with(Arc::new(socket), 0, flags & O_CLOEXEC != 0)?; write_sockaddr(addr_ptr, len_ptr, peer)?; Ok(new_fd) } fn sys_sendto(fd: u64, buf: u64, len: u64, addr_ptr: u64) -> Result { let to = if addr_ptr == 0 { None } else { Some(read_sockaddr(addr_ptr)?) }; with_socket(fd, |s| s.send(user_slice(buf, len)?, to)).map(|n| n as u64) } fn sys_recvfrom(fd: u64, buf: u64, len: u64, addr_ptr: u64, len_ptr: u64) -> Result { let (n, from) = with_socket(fd, |s| s.recv(user_slice_mut(buf, len)?))?; if let Some(from) = from { write_sockaddr(addr_ptr, len_ptr, from)?; } Ok(n as u64) } /// Only SO_RCVTIMEO: `struct timeval {i64 seconds, i64 microseconds}`; /// zero means "block forever", like Linux. fn sys_setsockopt(fd: u64, level: u64, name: u64, value: u64, len: u64) -> Result { if level != SOL_SOCKET || name != SO_RCVTIMEO || len < 16 { return Err(Errno::EOPNOTSUPP); } let tv = user_slice(value, 16)?; let secs = u64::from_ne_bytes(tv[..8].try_into().unwrap()); let usecs = u64::from_ne_bytes(tv[8..16].try_into().unwrap()); let total = secs.saturating_mul(1_000_000).saturating_add(usecs); with_socket(fd, |s| Ok(s.set_recv_timeout(if total == 0 { None } else { Some(total) })))?; Ok(0) } /// clock_gettime(clock, *timespec): CLOCK_REALTIME (0, wall-clock time /// since 1970, from the CMOS clock at boot) or the MONOTONIC family (1, /// 4 RAW, 6 COARSE, 7 BOOTTIME: time since boot). CPU-time clocks: EINVAL. fn sys_clock_gettime(clock: u64, ts_ptr: u64) -> Result { let us = match clock { 0 | 5 => crate::time::realtime_us(), // REALTIME, REALTIME_COARSE 1 | 4 | 6 | 7 => crate::time::now_us(), _ => return Err(Errno::EINVAL), }; copy_to_user(ts_ptr, &[us / 1_000_000, (us % 1_000_000) * 1000])?; Ok(0) } fn sys_resolve(name_ptr: u64, name_len: u64, out: u64, max: u64) -> Result { if name_len > 253 { return Err(Errno::ENAMETOOLONG); } let name = core::str::from_utf8(user_slice(name_ptr, name_len)?).map_err(|_| Errno::EINVAL)?; let addrs = crate::net::resolve(name, 5_000_000)?; let n = addrs.len().min(max as usize); let dst = user_slice_mut(out, 4 * n as u64)?; for (i, a) in addrs.iter().take(n).enumerate() { dst[4 * i..4 * i + 4].copy_from_slice(a); } Ok(n as u64) } // --- Processes, memory, signals (stage 14) -------------------------------------- /// A NUL-terminated string from user memory (at most `max` bytes). pub fn user_cstr(ptr: u64, max: usize) -> Result, Errno> { let mut out = Vec::new(); // Page by page: the string may end right before an unmapped page. let mut addr = ptr; loop { let chunk = (4096 - addr % 4096).min((max + 1 - out.len()) as u64); let bytes = user_slice(addr, chunk)?; if let Some(end) = bytes.iter().position(|&b| b == 0) { out.extend_from_slice(&bytes[..end]); return Ok(out); } out.extend_from_slice(bytes); if out.len() > max { return Err(Errno::ENAMETOOLONG); } addr += chunk; } } /// A NULL-terminated array of C strings (execve's argv and envp). fn user_cstr_array(ptr: u64) -> Result>, Errno> { let mut out = Vec::new(); if ptr == 0 { return Ok(out); // Linux allows NULL here (an empty list) } let mut total = 0; for i in 0.. { let p = u64::from_le_bytes(user_slice(ptr + 8 * i, 8)?.try_into().unwrap()); if p == 0 { break; } let s = user_cstr(p, process::MAX_ARGS_SIZE)?; total += s.len() + 1; if total > process::MAX_ARGS_SIZE { return Err(Errno::E2BIG); } out.push(s); } Ok(out) } fn sys_execve(frame: &mut TrapFrame, path: u64, argv: u64, envp: u64) -> Result { let p = current()?; let path = user_path(path)?; let argv = user_cstr_array(argv)?; let envp = user_cstr_array(envp)?; process::execve(&p, frame, &path, &argv, &envp)?; Ok(0) } /// clone(flags, stack, *ptid, *ctid, tls): a new thread (CLONE_THREAD) /// or a new process (fork-like flags). /// /// A thread shares everything with its process -- memory, fds, signal /// handlers -- so CLONE_THREAD needs CLONE_VM and CLONE_SIGHAND, like /// Linux. (CLONE_FILES / CLONE_FS are accepted and implied: our threads /// always share the fd table and cwd.) pthread_create passes them all plus /// SETTLS (its thread block), PARENT_SETTID and CHILD_CLEARTID (so /// pthread_join can sleep on a futex until the thread is gone). fn sys_clone(frame: &TrapFrame, flags: u64, stack: u64, ptid: u64, ctid: u64, tls: u64) -> Result { let p = current()?; if flags & CLONE_THREAD != 0 { const ALLOWED: u64 = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID | CLONE_DETACHED; if flags & (CLONE_VM | CLONE_SIGHAND) != CLONE_VM | CLONE_SIGHAND || flags & !(ALLOWED | 0xff) != 0 { return Err(Errno::EINVAL); } let mut set_tid = Vec::new(); if flags & CLONE_PARENT_SETTID != 0 { set_tid.push(ptid); } if flags & CLONE_CHILD_SETTID != 0 { set_tid.push(ctid); } let clear = if flags & CLONE_CHILD_CLEARTID != 0 { ctid } else { 0 }; let tls = (flags & CLONE_SETTLS != 0).then_some(tls); return process::clone_thread(&p, frame, stack, tls, clear, &set_tid); } // A process: exit signal SIGCHLD (or 0) in the low byte, optionally // CLONE_VFORK (+ CLONE_VM: posix_spawn's "vfork"; we copy-on-write // instead of sharing, which a vfork child can't tell apart). let exit_signal = flags & 0xff; let allowed = 0xff | CLONE_VFORK | CLONE_PARENT_SETTID | if flags & CLONE_VFORK != 0 { CLONE_VM } else { 0 }; if flags & !allowed != 0 || (exit_signal != 0 && exit_signal != signal::SIGCHLD as u64) { return Err(Errno::EINVAL); // shared memory/files between processes: not supported } let pid = process::fork(&p, frame, stack, flags & CLONE_VFORK != 0)?; if flags & CLONE_PARENT_SETTID != 0 { copy_to_user(ptid, &(pid as u32))?; } Ok(pid) } /// arch_prctl(ARCH_SET_FS, addr) / (ARCH_GET_FS, *addr): the FS segment /// base, which libc points at the thread's own data (thread-local storage: /// `fs:[0]` is "this thread's block" on x86_64). Saved per thread and /// reloaded by the scheduler on every switch. fn sys_arch_prctl(code: u64, addr: u64) -> Result { let t = task()?; match code { ARCH_SET_FS => { if addr >= crate::memory::USER_END { return Err(Errno::EPERM); // must be a (canonical) user address } t.fs_base.store(addr, Ordering::Relaxed); x86_64::registers::model_specific::FsBase::write(x86_64::VirtAddr::new(addr)); Ok(0) } ARCH_GET_FS => copy_to_user(addr, &t.fs_base.load(Ordering::Relaxed)).map(|_| 0), _ => Err(Errno::EINVAL), } } /// tgkill(tgid, tid, sig) / tkill(tid, sig): a signal for one thread /// (pthread_kill, raise). sig 0 = just check it exists. fn sys_tgkill(tgid: Option, tid: u64, sig: u64) -> Result { if sig >= signal::NSIG as u64 { return Err(Errno::EINVAL); } let me = current()?; let procs = match tgid { Some(pid) => alloc::vec![process::get(pid).ok_or(Errno::ESRCH)?], None => process::all(), }; let t = procs.iter().find_map(|p| p.task(tid)).ok_or(Errno::ESRCH)?; if sig != 0 { signal::send_thread(&t, sig as u32, signal::SigInfo::from_pid(sig as u32, me.pid)); } Ok(0) } /// nanosleep(*req, *rem). Our timer ticks every ~55 ms, so sleeps are /// rounded up to that. Cut short by a signal: the time left goes to *rem. /// clock_nanosleep(clock, flags, *req, *rem): nanosleep on a chosen clock; /// with TIMER_ABSTIME (1), `req` is a time on that clock to sleep until. fn sys_clock_nanosleep(clock: u64, flags: u64, req: u64, rem: u64) -> Result { let now = match clock { 0 => crate::time::realtime_us(), 1 | 7 => crate::time::now_us(), _ => return Err(Errno::EINVAL), // CPU-time clocks: can't sleep on those }; if flags & 1 == 0 { return sys_nanosleep(req, rem); } // Absolute: sleep the difference (and never report time left, like Linux). sleep_for(read_timespec_us(req)?.saturating_sub(now), 0) } fn sys_nanosleep(req: u64, rem: u64) -> Result { sleep_for(read_timespec_us(req)?, rem) } /// Sleep `us` microseconds; if a signal cuts it short, the time left goes /// to `*rem` (a timespec) unless that's 0. fn sleep_for(us: u64, rem: u64) -> Result { if us == 0 { thread::yield_now(); return Ok(0); } // The PIT ticks only every ~55 ms, so sleep whole ticks first and then // yield until the (TSC-precise) clock says the time is really up. let target = crate::time::now_us() + us; let deadline = crate::time::deadline_after_us(us).saturating_sub(1); let mut result = match deadline.checked_sub(crate::interrupts::ticks()) { Some(t) if t > 0 => thread::sleep_ticks_intr(t), _ => Ok(()), }; while result.is_ok() && crate::time::now_us() < target { if crate::signal::interrupted() { result = Err(Errno::ERESTARTNOHAND); } else { thread::yield_now(); } } if result.is_err() && rem != 0 { let left = target.saturating_sub(crate::time::now_us()); copy_to_user(rem, &[left / 1_000_000, left % 1_000_000 * 1000])?; } result.map(|_| 0) } /// pread64 / pwrite64: at `offset`, without moving the file offset. fn sys_pread(fd: u64, buf: u64, len: u64, offset: u64) -> Result { let file = current()?.file(fd)?; if (offset as i64) < 0 { return Err(Errno::EINVAL); } file.read_at(user_slice_mut(buf, len)?, offset).map(|n| n as u64) } fn sys_pwrite(fd: u64, buf: u64, len: u64, offset: u64) -> Result { let file = current()?.file(fd)?; if (offset as i64) < 0 { return Err(Errno::EINVAL); } file.write_at(user_slice(buf, len)?, offset).map(|n| n as u64) } /// The `struct iovec {void *base; size_t len;}` array of readv/writev. fn read_iovecs(ptr: u64, count: u64) -> Result, Errno> { if count > 1024 { return Err(Errno::EINVAL); } let raw = user_slice(ptr, count * 16)?; Ok(raw.chunks(16).map(|c| (u64::from_le_bytes(c[..8].try_into().unwrap()), u64::from_le_bytes(c[8..].try_into().unwrap()))).collect()) } /// readv: one read into a kernel buffer, then spread over the buffers /// (so a pipe or terminal sees a single read, like Linux). fn sys_readv(fd: u64, iov: u64, count: u64) -> Result { let file = current()?.file(fd)?; let iovs = read_iovecs(iov, count)?; let total: u64 = iovs.iter().map(|v| v.1).sum::().min(1 << 20); for &(base, len) in &iovs { user_slice_mut(base, len)?; // check first } let mut buf = alloc::vec![0u8; total as usize]; let n = file.read(&mut buf)?; let mut done = 0; for &(base, len) in &iovs { let k = (len as usize).min(n - done); user_slice_mut(base, k as u64)?.copy_from_slice(&buf[done..done + k]); done += k; } Ok(n as u64) } /// writev: gather the buffers and write them in one go (so the pieces of /// one printf can't be interleaved with another thread's output). fn sys_writev(fd: u64, iov: u64, count: u64) -> Result { let process = current()?; let file = process.file(fd)?; let mut buf = Vec::new(); for (base, len) in read_iovecs(iov, count)? { buf.extend_from_slice(user_slice(base, len)?); } let result = file.write(&buf).map(|n| n as u64); if result == Err(Errno::EPIPE) { signal::send(&process, signal::SIGPIPE, signal::SigInfo::from_pid(signal::SIGPIPE, 0)); } result } /// wait4(pid, *status, options, *rusage). We don't count CPU time per /// process yet, so a `struct rusage` (144 bytes) comes back all zeros. fn sys_wait4(pid: i64, status_ptr: u64, options: u64, rusage_ptr: u64) -> Result { if status_ptr != 0 { user_slice_mut(status_ptr, 4)?; // check before reaping anything } if rusage_ptr != 0 { user_slice_mut(rusage_ptr, 144)?; } let (pid, status) = process::wait4(&*current()?, pid, options)?; if status_ptr != 0 && pid != 0 { copy_to_user(status_ptr, &status)?; } if rusage_ptr != 0 && pid != 0 { user_slice_mut(rusage_ptr, 144)?.fill(0); } Ok(pid) } fn sys_rt_sigaction(sig: u64, new_ptr: u64, old_ptr: u64, size: u64) -> Result { let new = if new_ptr == 0 { None } else { let bytes = user_slice(new_ptr, core::mem::size_of::() as u64)?; Some(unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const signal::SigAction) }) }; if old_ptr != 0 { user_slice_mut(old_ptr, 32)?; } let old = signal::sys_rt_sigaction(&*current()?, sig, new, size)?; if old_ptr != 0 { copy_to_user(old_ptr, &old)?; } Ok(0) } fn sys_rt_sigprocmask(how: u64, set_ptr: u64, old_ptr: u64, size: u64) -> Result { let set = if set_ptr == 0 { None } else { Some(u64::from_le_bytes(user_slice(set_ptr, 8)?.try_into().unwrap())) }; let old = signal::sys_rt_sigprocmask(&*task()?, how, set, size)?; if old_ptr != 0 { copy_to_user(old_ptr, &old)?; } Ok(0) } /// mmap(addr, len, prot, flags, fd, offset). The new region is empty: pages /// appear on first touch (demand paging, mm.rs). fn sys_mmap(p: &process::Process, addr: u64, len: u64, prot: u64, flags: u64, fd: u64, offset: u64) -> Result { use crate::mm::{Kind, Mm, Vma}; let len = len.checked_next_multiple_of(4096).ok_or(Errno::ENOMEM)?; let shared = match flags & (MAP_SHARED | MAP_PRIVATE) { MAP_SHARED => true, MAP_PRIVATE => false, _ => return Err(Errno::EINVAL), // exactly one of the two }; if len == 0 || prot & !7 != 0 || offset % 4096 != 0 { return Err(Errno::EINVAL); } let file = if flags & MAP_ANONYMOUS != 0 { None } else { // A file: only read-only sharing of its bytes is supported -- a // private copy of the file's pages (writes stay in this process). // MAP_SHARED of a file would need writes to reach the file (a page // cache), which we don't have. if shared { return Err(Errno::ENODEV); } let inode = p.file(fd)?.inode().ok_or(Errno::ENODEV)?; Some((inode, offset)) }; let mut mm = p.mm.lock(); let start = if flags & MAP_FIXED != 0 { if addr % 4096 != 0 || !Mm::range_ok(addr, addr.saturating_add(len)) { return Err(Errno::EINVAL); } mm.unmap(addr, addr + len); // MAP_FIXED replaces whatever was there addr } else { mm.find_free(len).ok_or(Errno::ENOMEM)? }; mm.insert(Vma { start, end: start + len, prot, shared, file, kind: Kind::Mmap }); if shared { // Shared memory gets its frames now, so a fork before the first // touch still shares them (mm.rs). if mm.populate(start, start + len).is_err() { mm.unmap(start, start + len); return Err(Errno::ENOMEM); } } Ok(start) } fn page_range(addr: u64, len: u64) -> Result<(u64, u64), Errno> { let end = addr.checked_add(len).ok_or(Errno::EINVAL)?.next_multiple_of(4096); if addr % 4096 != 0 || !crate::mm::Mm::range_ok(addr, end.max(addr + 1)) { return Err(Errno::EINVAL); } Ok((addr, end)) } fn sys_munmap(p: &process::Process, addr: u64, len: u64) -> Result { let (start, end) = page_range(addr, len)?; if len == 0 { return Err(Errno::EINVAL); } p.mm.lock().unmap(start, end); Ok(0) } fn sys_mprotect(p: &process::Process, addr: u64, len: u64, prot: u64) -> Result { if prot & !7 != 0 { return Err(Errno::EINVAL); } let (start, end) = page_range(addr, len)?; if start == end { return Ok(0); } p.mm.lock().protect(start, end, prot).map(|_| 0) }