//! System calls via the `syscall` / `sysret` instructions. //! //! ABI (like Linux x86_64): number in rax, arguments in rdi, rsi, rdx (r10, //! r8 for 4th/5th), result in rax; negative results are `-errno`. The //! `syscall` instruction itself clobbers rcx (return address) and r11 (saved //! RFLAGS); every other register is preserved for the caller. //! //! Numbers are Linux's where Linux has the same call, so `strace` output and //! man pages mostly apply. Paths are passed as pointer + length (Linux uses //! NUL-terminated C strings); relative paths start at the process's cwd. //! //! | nr | name | arguments | note | //! |-----|-------------|-------------------------------------|------| //! | 0 | read | fd, buf, len | blocks: console until Enter, pipe until data/EOF | //! | 1 | write | fd, buf, len | | //! | 2 | open | path, path_len, flags | O_CREAT 0o100, O_TRUNC 0o1000; directories too (for getdents) | //! | 3 | close | fd | | //! | 4 | stat | path, path_len, *Stat | our own small `Stat {kind, size, mode}`; follows symlinks | //! | 5 | fstat | fd, *Stat | | //! | 8 | lseek | fd, offset, whence | SEEK_SET/CUR/END = 0/1/2 | //! | 16 | ioctl | fd, request, arg | only TTY_SET_RAW (ours): console raw/cooked mode | //! | 22 | pipe | *[u32; 2] | [read end, write end] | //! | 24 | sched_yield | | | //! | 35 | sleep_ms | milliseconds | Linux's nr 35 is nanosleep(timespec); ours is simpler | //! | 39 | getpid | | | //! | 60 | exit | code | | //! | 61 | wait | pid, flags | returns the exit code (139 if killed); WNOHANG = 1 -> EAGAIN if running. Linux wait4 is fancier | //! | 63 | uname | *utsname | Linux's `struct utsname` (6 x 65-byte strings) | //! | 79 | getcwd | buf, len | returns the length (no NUL) | //! | 80 | chdir | path, path_len | | //! | 83 | mkdir | path, path_len | | //! | 84 | rmdir | path, path_len | only empty directories | //! | 87 | unlink | path, path_len | files and symlinks (not what a symlink points to) | //! | 89 | readlink | path, path_len, buf, len | a symlink's target; returns its length (cut to `len`, no NUL) | //! | 217 | getdents64 | fd, buf, len | Linux `struct linux_dirent64` records | //! | 500 | spawn | path, path_len, argv, argv_len, *SpawnOptions | ours: start a program, returns its pid (see process.rs for why not fork/exec) | //! | 501 | mounts | buf, len | ours: "path fstype\n" per mount (Linux: read /proc/mounts) | //! //! spawn's `argv` is all arguments separated by NUL bytes ("ls\0-l\0/bin"); //! empty means just [program name]. `SpawnOptions` (or a null pointer for //! the defaults) picks the child's stdin/stdout/stderr from the caller's fds //! (-1 = same as mine) and flags (SPAWN_TRACE = print its syscalls). //! //! What happens on `syscall` (the CPU does steps 1-3 in one instruction): //! 1. rcx = user RIP (next instruction), r11 = user RFLAGS //! 2. RFLAGS &= !SFMASK (we mask IF -> interrupts off) //! 3. CS/SS = kernel selectors from STAR, RIP = LSTAR (`syscall_entry`), CPL = 0 //! ...but RSP is still the *user* stack! The CPU doesn't switch stacks for //! `syscall` (unlike interrupts, which use TSS.RSP0). So `syscall_entry` //! must do it first: stash the user RSP, load this thread's kernel stack, //! save registers, re-enable interrupts and call `syscall_dispatch` in Rust. //! `sysretq` reverses it: RIP = rcx, RFLAGS = r11, CPL = 3. //! //! Single CPU only: the "current kernel stack" and the user-RSP scratch slot //! are plain globals. (Multi-core kernels use per-CPU data via `swapgs`.) //! //! Why not `int 0x80`? It also works (and is simpler: the CPU switches stacks //! via TSS.RSP0 for us), but `syscall` is what Linux and every modern x86_64 //! OS use, and it's faster. use crate::errno::Errno; use crate::fs::file::{InodeFile, Stat}; use crate::fs::{self, InodeKind}; use crate::{memory, process, thread}; use alloc::{string::String, sync::Arc, vec::Vec}; use core::arch::global_asm; use core::sync::atomic::{AtomicU64, Ordering}; pub const SYS_READ: u64 = 0; pub const SYS_WRITE: u64 = 1; pub const SYS_OPEN: u64 = 2; pub const SYS_CLOSE: u64 = 3; pub const SYS_STAT: u64 = 4; pub const SYS_FSTAT: u64 = 5; pub const SYS_LSEEK: u64 = 8; pub const SYS_IOCTL: u64 = 16; pub const SYS_PIPE: u64 = 22; pub const SYS_SCHED_YIELD: u64 = 24; pub const SYS_SLEEP_MS: u64 = 35; pub const SYS_GETPID: u64 = 39; pub const SYS_EXIT: u64 = 60; pub const SYS_WAIT: u64 = 61; pub const SYS_UNAME: u64 = 63; pub const SYS_GETCWD: u64 = 79; pub const SYS_CHDIR: u64 = 80; pub const SYS_MKDIR: u64 = 83; pub const SYS_RMDIR: u64 = 84; pub const SYS_UNLINK: u64 = 87; pub const SYS_READLINK: u64 = 89; pub const SYS_GETDENTS64: u64 = 217; pub const SYS_SPAWN: u64 = 500; pub const SYS_MOUNTS: u64 = 501; pub const O_CREAT: u64 = 0o100; pub const O_TRUNC: u64 = 0o1000; pub const O_APPEND: u64 = 0o2000; /// ioctl request (ours): arg 1 = raw mode, 0 = cooked; returns the old mode. pub const TTY_SET_RAW: u64 = 1; pub const SPAWN_TRACE: u64 = 1; /// Name of a syscall number (for tracing). pub fn name(nr: u64) -> &'static str { match nr { SYS_READ => "read", SYS_WRITE => "write", SYS_OPEN => "open", SYS_CLOSE => "close", SYS_STAT => "stat", SYS_FSTAT => "fstat", SYS_LSEEK => "lseek", SYS_IOCTL => "ioctl", SYS_PIPE => "pipe", SYS_SCHED_YIELD => "sched_yield", SYS_SLEEP_MS => "sleep_ms", SYS_GETPID => "getpid", SYS_EXIT => "exit", SYS_WAIT => "wait", SYS_UNAME => "uname", SYS_GETCWD => "getcwd", SYS_CHDIR => "chdir", SYS_MKDIR => "mkdir", SYS_RMDIR => "rmdir", SYS_UNLINK => "unlink", SYS_READLINK => "readlink", SYS_GETDENTS64 => "getdents64", SYS_SPAWN => "spawn", SYS_MOUNTS => "mounts", _ => "?", } } /// spawn's options, as laid out in user memory. #[repr(C)] #[derive(Clone, Copy)] pub struct SpawnOptions { pub stdin: i64, // caller's fd to use as the child's fd 0; -1 = caller's fd 0 pub stdout: i64, pub stderr: i64, pub flags: u64, } /// Kernel stack top of the running thread (set by the scheduler). #[unsafe(no_mangle)] static SYSCALL_KERNEL_RSP: AtomicU64 = AtomicU64::new(0); /// Scratch slot for the user RSP during the first instructions of the entry. #[unsafe(no_mangle)] static SYSCALL_USER_RSP: AtomicU64 = AtomicU64::new(0); /// Called by the scheduler when switching threads. pub fn set_kernel_stack(stack_top: u64) { SYSCALL_KERNEL_RSP.store(stack_top, Ordering::Relaxed); } /// Program the MSRs that make `syscall` work. pub fn init() { use x86_64::VirtAddr; use x86_64::registers::model_specific::{Efer, EferFlags, LStar, SFMask, Star}; use x86_64::registers::rflags::RFlags; let s = crate::gdt::selectors(); unsafe { // EFER.SCE: enable the syscall/sysret instructions at all. Efer::update(|flags| flags.insert(EferFlags::SYSTEM_CALL_EXTENSIONS)); } // STAR: which segments syscall/sysret load (see gdt.rs for the layout rule). Star::write(s.user_code, s.user_data, s.kernel_code, s.kernel_data) .expect("GDT layout doesn't fit syscall/sysret"); // LSTAR: where `syscall` jumps to. LStar::write(VirtAddr::new(syscall_entry as *const () as u64)); // SFMASK: RFLAGS bits to clear on entry. IF: no interrupts until we're on // the kernel stack. DF: Rust expects the direction flag clear. TF: no // single-stepping into the kernel. SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::DIRECTION_FLAG | RFlags::TRAP_FLAG); } unsafe extern "C" { fn syscall_entry(); } global_asm!( ".global syscall_entry", "syscall_entry:", // Interrupts are off (SFMASK). Switch to this thread's kernel stack. " mov [rip + SYSCALL_USER_RSP], rsp", " mov rsp, [rip + SYSCALL_KERNEL_RSP]", " push qword ptr [rip + SYSCALL_USER_RSP]", // user rsp " push rcx", // user rip " push r11", // user rflags // Preserve the argument registers for the user (Rust may clobber them). // rbx, rbp, r12-r15 are callee-saved: the Rust code keeps them intact. " push rdi", " push rsi", " push rdx", " push r8", " push r9", " push r10", " sub rsp, 8", // 10 pushes total -> stack 16-byte aligned for the call // syscall_dispatch(nr, a1, a2, a3, a4, a5) in the C calling convention: // shuffle rax, rdi, rsi, rdx, r10, r8 -> rdi, rsi, rdx, rcx, r8, r9. " mov r9, r8", " mov r8, r10", " mov rcx, rdx", " mov rdx, rsi", " mov rsi, rdi", " mov rdi, rax", " sti", // we're on our own stack now: preemption is fine " call {dispatch}", // result in rax " cli", // no interrupts while we're back on the user stack in ring 0 " add rsp, 8", " pop r10", " pop r9", " pop r8", " pop rdx", " pop rsi", " pop rdi", " pop r11", " pop rcx", " pop rsp", // (rcx came from the `syscall` itself, so it's a canonical address; // sysret with a non-canonical rcx would fault *in ring 0* -- a classic bug.) " sysretq", dispatch = sym syscall_dispatch, ); /// Rust side of every system call. Returns the value for rax. extern "C" fn syscall_dispatch(nr: u64, a1: u64, a2: u64, a3: u64, a4: u64, a5: u64) -> i64 { let process = thread::current_process(); let trace = process.as_ref().is_some_and(|p| p.trace); if trace && nr == SYS_EXIT { crate::kprintln!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64); } drop(process); // don't hold an Arc while blocking/exiting let result = match nr { SYS_READ => sys_read(a1, a2, a3), SYS_WRITE => sys_write(a1, a2, a3), SYS_OPEN => sys_open(a1, a2, a3), SYS_CLOSE => current().and_then(|p| p.close(a1)).map(|_| 0), SYS_STAT => sys_stat(a1, a2, a3), SYS_FSTAT => sys_fstat(a1, a2), SYS_LSEEK => current().and_then(|p| p.file(a1)).and_then(|f| f.seek(a2 as i64, a3)), SYS_IOCTL => sys_ioctl(a1, a2, a3), SYS_PIPE => sys_pipe(a1), SYS_SCHED_YIELD => { thread::yield_now(); Ok(0) } SYS_SLEEP_MS => { // The PIT fires ~18.2 times a second: 1 tick = ~55 ms. Round up. thread::sleep_ticks(a1.div_ceil(55)); Ok(0) } SYS_GETPID => current().map(|p| p.pid), SYS_EXIT => process::exit_current(process::ExitStatus::Exited(a1 as i64)), SYS_WAIT => current().and_then(|p| process::wait_child(p.pid, a1, a2)).map(|s| s.code() as u64), SYS_UNAME => sys_uname(a1), SYS_GETCWD => sys_getcwd(a1, a2), SYS_CHDIR => sys_chdir(a1, a2), SYS_MKDIR => user_path(a1, a2).and_then(|p| fs::create(&p, InodeKind::Directory)).map(|_| 0), SYS_RMDIR => user_path(a1, a2).and_then(|p| fs::remove(&p, true)).map(|_| 0), SYS_UNLINK => user_path(a1, a2).and_then(|p| fs::remove(&p, false)).map(|_| 0), SYS_READLINK => sys_readlink(a1, a2, a3, a4), SYS_GETDENTS64 => sys_getdents64(a1, a2, a3), SYS_SPAWN => sys_spawn(a1, a2, a3, a4, a5), SYS_MOUNTS => sys_mounts(a1, a2), _ => Err(Errno::ENOSYS), }; let ret = match result { Ok(value) => value as i64, Err(errno) => -(errno as i64), }; if trace { let pid = current().map(|p| p.pid).unwrap_or(0); crate::kprintln!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret)); } ret } /// One syscall in strace style: `open("/mnt/disk0/readme.txt", 0) = 3`. /// Strings and buffers are read back from user memory (after the call, so /// `read` shows what it got), shortened to 40 bytes. fn trace_line(nr: u64, a: [u64; 4], ret: i64) -> String { use alloc::format; let text = |ptr: u64, len: u64| -> String { let Ok(bytes) = user_slice(ptr, len.min(40)) else { return String::from("") }; let mut s = String::from("\""); for &b in bytes { match b { b'\n' => s.push_str("\\n"), b'"' => s.push_str("\\\""), 0x20..=0x7e => s.push(b as char), _ => s.push_str(&format!("\\x{:02x}", b)), } } s.push('"'); if len > 40 { s.push_str("..."); } s }; let args = match nr { SYS_READ => format!("{}, {}, {}", a[0], if ret > 0 { text(a[1], ret as u64) } else { String::from("\"\"") }, a[2]), SYS_WRITE => format!("{}, {}, {}", a[0], text(a[1], a[2]), a[2]), SYS_OPEN => format!("{}, {:#o}", text(a[0], a[1]), a[2]), SYS_STAT => format!("{}, {:#x}", text(a[0], a[1]), a[2]), SYS_READLINK => format!("{}, {:#x}, {}", text(a[0], a[1]), a[2], a[3]), SYS_CHDIR | SYS_MKDIR | SYS_RMDIR | SYS_UNLINK => text(a[0], a[1]), SYS_SPAWN => format!("{}, {}", text(a[0], a[1]), text(a[2], a[3])), SYS_SCHED_YIELD | SYS_GETPID => String::new(), SYS_CLOSE | SYS_SLEEP_MS | SYS_EXIT => format!("{}", a[0] as i64), SYS_PIPE | SYS_UNAME => format!("{:#x}", a[0]), SYS_FSTAT | SYS_WAIT | SYS_GETCWD | SYS_MOUNTS => format!("{}, {:#x}", a[0], a[1]), _ => format!("{}, {:#x}, {:#x}", a[0], a[1], a[2]), }; match Errno::from_ret(ret) { Some(e) if ret < 0 => format!("{}({}) = {} {:?}", name(nr), args, ret, e), _ => format!("{}({}) = {}", name(nr), args, ret), } } fn current() -> Result, Errno> { thread::current_process().ok_or(Errno::EINVAL) } /// Turn a user pointer + length into a slice, after checking that the user /// may really access that memory. Never trust pointers from user space! pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> { if len == 0 { return Ok(&[]); // nothing to access, so any pointer is fine (like Linux) } if !memory::user_range_ok(ptr, len, false) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) }) } /// Like `user_slice`, but the user must be allowed to *write* the memory. pub fn user_slice_mut(ptr: u64, len: u64) -> Result<&'static mut [u8], Errno> { if len == 0 { return Ok(&mut []); } if !memory::user_range_ok(ptr, len, true) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len as usize) }) } fn sys_read(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; // clone the Arc: no lock held while we block let buf = user_slice_mut(buf, len)?; file.read(buf).map(|n| n as u64) } fn sys_write(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; let buf = user_slice(buf, len)?; file.write(buf).map(|n| n as u64) } /// Read a path argument and make it absolute (relative to the cwd). fn user_path(ptr: u64, len: u64) -> Result { if len > 4096 { return Err(Errno::ENAMETOOLONG); } let path = core::str::from_utf8(user_slice(ptr, len)?).map_err(|_| Errno::EINVAL)?; current()?.resolve(path) } /// Copy a plain-data struct out to user memory. fn copy_to_user(ptr: u64, value: &T) -> Result<(), Errno> { let size = core::mem::size_of::(); let dst = user_slice_mut(ptr, size as u64)?; let src = unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size) }; dst.copy_from_slice(src); Ok(()) } fn sys_open(path_ptr: u64, path_len: u64, flags: u64) -> Result { let path = user_path(path_ptr, path_len)?; let inode = match fs::lookup(&path) { Err(Errno::ENOENT) if flags & O_CREAT != 0 => fs::create(&path, InodeKind::File)?, other => other?, }; if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File { inode.truncate()?; } let file = if flags & O_APPEND != 0 { InodeFile::open_append(inode)? } else { InodeFile::open(inode)? }; current()?.add_file(Arc::new(file)) } fn sys_stat(path_ptr: u64, path_len: u64, stat_ptr: u64) -> Result { let inode = fs::lookup(&user_path(path_ptr, path_len)?)?; copy_to_user(stat_ptr, &Stat::of(&*inode))?; Ok(0) } fn sys_fstat(fd: u64, stat_ptr: u64) -> Result { let stat = current()?.file(fd)?.stat()?; copy_to_user(stat_ptr, &stat)?; Ok(0) } fn sys_pipe(fds_ptr: u64) -> Result { user_slice_mut(fds_ptr, 8)?; // check before creating anything let process = current()?; let (reader, writer) = fs::pipe::new(); let read_fd = process.add_file(Arc::new(reader))?; let write_fd = match process.add_file(Arc::new(writer)) { Ok(fd) => fd, Err(e) => { process.close(read_fd)?; return Err(e); } }; copy_to_user(fds_ptr, &[read_fd as u32, write_fd as u32])?; Ok(0) } fn sys_getcwd(buf: u64, len: u64) -> Result { let cwd = current()?.cwd.lock().clone(); if (len as usize) < cwd.len() { return Err(Errno::ERANGE); } user_slice_mut(buf, cwd.len() as u64)?.copy_from_slice(cwd.as_bytes()); Ok(cwd.len() as u64) } fn sys_chdir(path_ptr: u64, path_len: u64) -> Result { let path = user_path(path_ptr, path_len)?; if fs::lookup(&path)?.kind() != InodeKind::Directory { return Err(Errno::ENOTDIR); } *current()?.cwd.lock() = path; Ok(0) } fn sys_readlink(path_ptr: u64, path_len: u64, buf: u64, len: u64) -> Result { let target = fs::readlink(&user_path(path_ptr, path_len)?)?; let n = target.len().min(len as usize); user_slice_mut(buf, n as u64)?.copy_from_slice(&target.as_bytes()[..n]); Ok(n as u64) } fn sys_getdents64(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; file.getdents(user_slice_mut(buf, len)?).map(|n| n as u64) } fn sys_spawn(path_ptr: u64, path_len: u64, argv_ptr: u64, argv_len: u64, opts_ptr: u64) -> Result { let parent = current()?; let path = user_path(path_ptr, path_len)?; if argv_len as usize > process::MAX_ARGS_SIZE { return Err(Errno::E2BIG); } let blob = core::str::from_utf8(user_slice(argv_ptr, argv_len)?).map_err(|_| Errno::EINVAL)?; let mut args: Vec = blob.split('\0').filter(|a| !a.is_empty()).map(String::from).collect(); if args.is_empty() { args.push(String::from(path.rsplit('/').next().unwrap_or(""))); } let opts = if opts_ptr == 0 { SpawnOptions { stdin: -1, stdout: -1, stderr: -1, flags: 0 } } else { let bytes = user_slice(opts_ptr, core::mem::size_of::() as u64)?; unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const SpawnOptions) } }; // Child fd n = the caller's fd chosen in the options (-1: the caller's fd n). let pick = |n: u64, choice: i64| parent.file(if choice < 0 { n } else { choice as u64 }); let stdio = [pick(0, opts.stdin)?, pick(1, opts.stdout)?, pick(2, opts.stderr)?]; let spawn = process::SpawnArgs { args, stdio, cwd: parent.cwd.lock().clone(), parent: parent.pid, trace: opts.flags & SPAWN_TRACE != 0, }; drop(parent); process::spawn_path(&path, spawn).map(|child| child.pid) } fn sys_ioctl(fd: u64, request: u64, arg: u64) -> Result { let process = current()?; if process.file(fd)?.stat()?.kind != crate::fs::file::KIND_CONSOLE { return Err(Errno::ENOTTY); } match request { TTY_SET_RAW => Ok(crate::tty::set_raw(arg != 0, process.pid) as u64), _ => Err(Errno::EINVAL), } } /// Linux's `struct utsname`: six NUL-terminated strings of 65 bytes each. fn sys_uname(buf: u64) -> Result { let fields = [ "Rustbelt", // sysname (`uname -s`) "qemu", // nodename (hostname) env!("CARGO_PKG_VERSION"), // release (`uname -r`) "stage 8: ext2", // version "x86_64", // machine "(none)", // domainname ]; let out = user_slice_mut(buf, 6 * 65)?; out.fill(0); for (i, field) in fields.iter().enumerate() { out[i * 65..i * 65 + field.len()].copy_from_slice(field.as_bytes()); } Ok(0) } /// The mount table as text, one "path fstype" line per mount. fn sys_mounts(buf: u64, len: u64) -> Result { let mut text = String::new(); for (path, fs_type) in fs::mounts() { text.push_str(&path); text.push(' '); text.push_str(fs_type); text.push('\n'); } if (len as usize) < text.len() { return Err(Errno::ERANGE); } user_slice_mut(buf, text.len() as u64)?.copy_from_slice(text.as_bytes()); Ok(text.len() as u64) }