//! System calls via the `syscall` / `sysret` instructions. //! //! ABI (like Linux x86_64): number in rax, arguments in rdi, rsi, rdx (r10, //! r8 for 4th/5th), result in rax; negative results are `-errno`. The //! `syscall` instruction itself clobbers rcx (return address) and r11 (saved //! RFLAGS); every other register is preserved for the caller. //! //! | nr | name | arguments | note | //! |----|-------------|--------------------------------|------| //! | 0 | read | fd, buf, len | fd 0 blocks until a line is typed | //! | 1 | write | fd, buf, len | fd 1/2 = console | //! | 2 | open | path_ptr, path_len, flags | Linux takes a C string; we take ptr+len. flags: O_CREAT = 0o100 | //! | 3 | close | fd | | //! | 24 | sched_yield | | | //! | 39 | getpid | | | //! | 60 | exit | code | | //! //! What happens on `syscall` (the CPU does steps 1-3 in one instruction): //! 1. rcx = user RIP (next instruction), r11 = user RFLAGS //! 2. RFLAGS &= !SFMASK (we mask IF -> interrupts off) //! 3. CS/SS = kernel selectors from STAR, RIP = LSTAR (`syscall_entry`), CPL = 0 //! ...but RSP is still the *user* stack! The CPU doesn't switch stacks for //! `syscall` (unlike interrupts, which use TSS.RSP0). So `syscall_entry` //! must do it first: stash the user RSP, load this thread's kernel stack, //! save registers, re-enable interrupts and call `syscall_dispatch` in Rust. //! `sysretq` reverses it: RIP = rcx, RFLAGS = r11, CPL = 3. //! //! Single CPU only: the "current kernel stack" and the user-RSP scratch slot //! are plain globals. (Multi-core kernels use per-CPU data via `swapgs`.) //! //! Why not `int 0x80`? It also works (and is simpler: the CPU switches stacks //! via TSS.RSP0 for us), but `syscall` is what Linux and every modern x86_64 //! OS use, and it's faster. use crate::errno::Errno; use crate::fs::{self, file::InodeFile}; use crate::{memory, process, thread}; use alloc::sync::Arc; use core::arch::global_asm; use core::sync::atomic::{AtomicU64, Ordering}; pub const SYS_READ: u64 = 0; pub const SYS_WRITE: u64 = 1; pub const SYS_OPEN: u64 = 2; pub const SYS_CLOSE: u64 = 3; pub const SYS_SCHED_YIELD: u64 = 24; pub const SYS_GETPID: u64 = 39; pub const SYS_EXIT: u64 = 60; pub const O_CREAT: u64 = 0o100; /// Kernel stack top of the running thread (set by the scheduler). #[unsafe(no_mangle)] static SYSCALL_KERNEL_RSP: AtomicU64 = AtomicU64::new(0); /// Scratch slot for the user RSP during the first instructions of the entry. #[unsafe(no_mangle)] static SYSCALL_USER_RSP: AtomicU64 = AtomicU64::new(0); /// Called by the scheduler when switching threads. pub fn set_kernel_stack(stack_top: u64) { SYSCALL_KERNEL_RSP.store(stack_top, Ordering::Relaxed); } /// Program the MSRs that make `syscall` work. pub fn init() { use x86_64::VirtAddr; use x86_64::registers::model_specific::{Efer, EferFlags, LStar, SFMask, Star}; use x86_64::registers::rflags::RFlags; let s = crate::gdt::selectors(); unsafe { // EFER.SCE: enable the syscall/sysret instructions at all. Efer::update(|flags| flags.insert(EferFlags::SYSTEM_CALL_EXTENSIONS)); } // STAR: which segments syscall/sysret load (see gdt.rs for the layout rule). Star::write(s.user_code, s.user_data, s.kernel_code, s.kernel_data) .expect("GDT layout doesn't fit syscall/sysret"); // LSTAR: where `syscall` jumps to. LStar::write(VirtAddr::new(syscall_entry as *const () as u64)); // SFMASK: RFLAGS bits to clear on entry. IF: no interrupts until we're on // the kernel stack. DF: Rust expects the direction flag clear. TF: no // single-stepping into the kernel. SFMask::write(RFlags::INTERRUPT_FLAG | RFlags::DIRECTION_FLAG | RFlags::TRAP_FLAG); } unsafe extern "C" { fn syscall_entry(); } global_asm!( ".global syscall_entry", "syscall_entry:", // Interrupts are off (SFMASK). Switch to this thread's kernel stack. " mov [rip + SYSCALL_USER_RSP], rsp", " mov rsp, [rip + SYSCALL_KERNEL_RSP]", " push qword ptr [rip + SYSCALL_USER_RSP]", // user rsp " push rcx", // user rip " push r11", // user rflags // Preserve the argument registers for the user (Rust may clobber them). // rbx, rbp, r12-r15 are callee-saved: the Rust code keeps them intact. " push rdi", " push rsi", " push rdx", " push r8", " push r9", " push r10", " sub rsp, 8", // 10 pushes total -> stack 16-byte aligned for the call // syscall_dispatch(nr, a1, a2, a3, a4, a5) in the C calling convention: // shuffle rax, rdi, rsi, rdx, r10, r8 -> rdi, rsi, rdx, rcx, r8, r9. " mov r9, r8", " mov r8, r10", " mov rcx, rdx", " mov rdx, rsi", " mov rsi, rdi", " mov rdi, rax", " sti", // we're on our own stack now: preemption is fine " call {dispatch}", // result in rax " cli", // no interrupts while we're back on the user stack in ring 0 " add rsp, 8", " pop r10", " pop r9", " pop r8", " pop rdx", " pop rsi", " pop rdi", " pop r11", " pop rcx", " pop rsp", // (rcx came from the `syscall` itself, so it's a canonical address; // sysret with a non-canonical rcx would fault *in ring 0* -- a classic bug.) " sysretq", dispatch = sym syscall_dispatch, ); /// Rust side of every system call. Returns the value for rax. extern "C" fn syscall_dispatch(nr: u64, a1: u64, a2: u64, a3: u64, _a4: u64, _a5: u64) -> i64 { let result = match nr { SYS_READ => sys_read(a1, a2, a3), SYS_WRITE => sys_write(a1, a2, a3), SYS_OPEN => sys_open(a1, a2, a3), SYS_CLOSE => sys_close(a1), SYS_SCHED_YIELD => { thread::yield_now(); Ok(0) } SYS_GETPID => current().map(|p| p.pid), SYS_EXIT => process::exit_current(process::ExitStatus::Exited(a1 as i64)), _ => Err(Errno::ENOSYS), }; match result { Ok(value) => value as i64, Err(errno) => -(errno as i64), } } fn current() -> Result, Errno> { thread::current_process().ok_or(Errno::EINVAL) } /// Turn a user pointer + length into a slice, after checking that the user /// may really access that memory. Never trust pointers from user space! pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> { if !memory::user_range_ok(ptr, len, false) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) }) } /// Like `user_slice`, but the user must be allowed to *write* the memory. pub fn user_slice_mut(ptr: u64, len: u64) -> Result<&'static mut [u8], Errno> { if !memory::user_range_ok(ptr, len, true) { return Err(Errno::EFAULT); } Ok(unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, len as usize) }) } fn sys_read(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; // clone the Arc: no lock held while we block let buf = user_slice_mut(buf, len)?; file.read(buf).map(|n| n as u64) } fn sys_write(fd: u64, buf: u64, len: u64) -> Result { let file = current()?.file(fd)?; let buf = user_slice(buf, len)?; file.write(buf).map(|n| n as u64) } fn sys_open(path_ptr: u64, path_len: u64, flags: u64) -> Result { if path_len > 255 { return Err(Errno::ENAMETOOLONG); } let path = core::str::from_utf8(user_slice(path_ptr, path_len)?).map_err(|_| Errno::EINVAL)?; let inode = match fs::lookup(path) { Err(Errno::ENOENT) if flags & O_CREAT != 0 => fs::create(path, fs::InodeKind::File)?, other => other?, }; let file = Arc::new(InodeFile::open(inode)?); current()?.add_file(file) } fn sys_close(fd: u64) -> Result { current()?.close(fd).map(|_| 0) }