rust-kernel

@@ -261,7 +261,7 @@ dependencies = [  [[package]] name = "rustbelt"-version = "0.14.0"+version = "0.15.0" dependencies = [  "bootloader",  "conquer-once",

@@ -1,6 +1,6 @@ [package] name = "rustbelt"-version = "0.14.0"+version = "0.15.0" edition = "2024"  [dependencies]

@@ -76,6 +76,7 @@ const PROGRAMS: &[&str] = &[     "ifconfig", "ping", "host", "fetch", "httpd", // stage 12: networking     "random", "getty", "tile", // stage 13: virtual drivers     "forktest", "cowtest", "mmaptest", "sigtest", // stage 14: processes+    "threads", "echoserver", "ptytest", "nbtest", // stage 15: libc plumbing ];  const DISK_SECTORS: u64 = 64 * 1024 * 1024 / 512;

@@ -100,6 +100,13 @@ start spin 1000 kill %1 wait kill -l+# Stage 15: libc plumbing. Threads summing on all 4 CPUs (futex mutex,+# TLS), an epoll + poll echo server over 127.0.0.1, ptys and the line+# discipline, nonblocking I/O; each prints "all checks passed".+threads+echoserver+ptytest+nbtest nosuchprogram cd Q: cd A:/tmp

@@ -68,7 +68,7 @@ VIRTIO_DEVICES=(-vga none -device virtio-vga -device virtio-rng-pci  LOG=target/headless-serial.log set +e-timeout 90 qemu-system-x86_64 -smp 4 \+timeout 120 qemu-system-x86_64 -smp 4 \     -drive format=raw,file="$IMG" \     -drive file=target/disk.img,format=raw,if=virtio,snapshot=on \     -drive "$EXT2_DRIVE" \@@ -85,7 +85,7 @@ set -e # Lines the shell self-test must have printed (only for a plain boot). if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     expect=(-        "Rustbelt 0.14.0 (stage 14: processes) x86_64" # ver+        "Rustbelt 0.15.0 (stage 15: libc plumbing) x86_64" # ver         "[smp] 4 CPUs online"                          # smp.rs started 3 more cores         "B:  /mnt/disk0   fat32"                       # drives         "C:  /mnt/disk1   ext2"@@ -149,6 +149,10 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         "[sigtest] all checks passed"         "(start spin 1000), exit code 143"             # kill %1 (SIGTERM)         "15) SIGTERM"                                  # kill -l+        "[threads] all checks passed"                  # stage 15: libc plumbing+        "[echoserver] all checks passed"+        "[ptytest] all checks passed"+        "[nbtest] all checks passed"         "sh: nosuchprogram: command not found"         "cd: there is no drive Q:"         "A:/tmp $ pwd"@@ -157,11 +161,11 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     for line in "${expect[@]}"; do         grep -qF -- "$line" "$LOG" || { echo "[run-headless] MISSING: $line"; missing=1; }     done-    # Pipe results: `... | grep -i EXT | wc -l` -> 1, `dir bin | wc -l` -> 29,+    # Pipe results: `... | grep -i EXT | wc -l` -> 1, `dir bin | wc -l` -> 33,     # `cat C:/chunks.txt | wc -l` (copied from E:) -> 3,     # `type D:/COW.TXT | wc -l` (copied from E:) -> 4, and     # `cat D:/seq.txt | wc -l` (downloaded by wget) -> 20000.-    for n in 1 29 3 4 20000; do+    for n in 1 33 3 4 20000; do         grep -qx -- "$n" "$LOG" || { echo "[run-headless] MISSING pipe result: $n"; missing=1; }     done     # No memory leaks: once the shell and all ~60 programs it ran have@@ -172,7 +176,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         echo "[run-headless] MISSING: frames back at the baseline after the shell"; missing=1     fi     # The other direction: the host fetched a page from httpd in the guest.-    if grep -qF "Hello from Rustbelt! Rustbelt 0.14.0" "$CURL_OUT" 2>/dev/null; then+    if grep -qF "Hello from Rustbelt! Rustbelt 0.15.0" "$CURL_OUT" 2>/dev/null; then         echo "[run-headless] host curl got httpd's page: $(head -1 "$CURL_OUT")"     else         echo "[run-headless] MISSING: host curl through hostfwd to httpd"; missing=1@@ -180,8 +184,11 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     # random -n 8: 16 hex digits; random -d 6 5: five numbers 1-6.     grep -qxE -- "[0-9a-f]{16}" "$LOG" || { echo "[run-headless] MISSING: random -n 8 output"; missing=1; }     grep -qxE -- "[1-6]( [1-6]){4}" "$LOG" || { echo "[run-headless] MISSING: random -d 6 5 output"; missing=1; }+    # hvc0 is a tty now: its output turns "\n" into "\r\n" (ONLCR), so+    # drop the '\r's before comparing whole lines.+    tr -d '\r' < "$HVC_OUT" > "$HVC_OUT.txt" 2>/dev/null && mv "$HVC_OUT.txt" "$HVC_OUT"     # The shell on hvc0 ran the host's commands and answered over hvc0.-    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.14.0 (stage 14" \+    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.15.0 (stage 15" \                 "$ echo hi over hvc0" "hi over hvc0" "fb0" "hvc0"; do         grep -qF -- "$line" "$HVC_OUT" 2>/dev/null || { echo "[run-headless] MISSING on hvc0: $line"; missing=1; }     done

@@ -4,7 +4,11 @@ //!   /dev/null      reads as empty, swallows writes //!   /dev/random    random bytes (virtio-rng; /dev/urandom is the same -- //!   /dev/urandom   on modern Linux they only differ before boot-time seeding)+//!   /dev/console   the console terminal (keyboard + screen) //!   /dev/hvc0      the virtio console: a second terminal+//!   /dev/tty       your controlling terminal, whichever it is (tty.rs)+//!   /dev/ptmx      opening it creates a pseudo-terminal: you get the+//!   /dev/pts/N     master end, and its slave end appears as /dev/pts/N //!   /dev/fb0       the virtio-gpu framebuffer: pixels at byte offsets, //!                  plus ioctls (see syscall.rs) for its size, flushing, //!                  filling rectangles and moving the text console@@ -17,9 +21,9 @@ use crate::errno::Errno; use crate::fs::file::{File, KIND_DEVICE, SEEK_CUR, SEEK_END, SEEK_SET, Stat}; use crate::fs::{DirEntry, FileSystem, Inode, InodeKind};+use crate::tty; use crate::virtio::{console, gpu, rng}; use alloc::{string::String, sync::Arc, vec::Vec};-use core::sync::atomic::Ordering; use spin::Mutex;  pub struct DevFs;@@ -35,7 +39,7 @@ impl FileSystem for DevFs {  /// The devices that exist on this machine. fn names() -> Vec<&'static str> {-    let mut names = alloc::vec!["null", "random", "urandom"];+    let mut names = alloc::vec!["console", "null", "ptmx", "pts", "random", "tty", "urandom"];     if console::present() {         names.push("hvc0");     }@@ -57,10 +61,53 @@ impl Inode for DevDir {     }     fn lookup(&self, name: &str) -> Result<Arc<dyn Inode>, Errno> {         let name = names().into_iter().find(|n| *n == name).ok_or(Errno::ENOENT)?;+        if name == "pts" {+            return Ok(Arc::new(PtsDir));+        }         Ok(Arc::new(DevNode(name)))     }     fn list(&self) -> Result<Vec<DirEntry>, Errno> {-        Ok(names().into_iter().map(|n| DirEntry { name: String::from(n), kind: InodeKind::Device }).collect())+        let kind = |n| if n == "pts" { InodeKind::Directory } else { InodeKind::Device };+        Ok(names().into_iter().map(|n| DirEntry { name: String::from(n), kind: kind(n) }).collect())+    }+}++/// /dev/pts: one entry per pseudo-terminal (named by number).+struct PtsDir;++impl Inode for PtsDir {+    fn kind(&self) -> InodeKind {+        InodeKind::Directory+    }+    fn size(&self) -> usize {+        tty::pty_count()+    }+    fn lookup(&self, name: &str) -> Result<Arc<dyn Inode>, Errno> {+        let n: usize = name.parse().map_err(|_| Errno::ENOENT)?;+        if !tty::pty_numbers().contains(&n) {+            return Err(Errno::ENOENT);+        }+        Ok(Arc::new(PtsNode(n)))+    }+    fn list(&self) -> Result<Vec<DirEntry>, Errno> {+        Ok(tty::pty_numbers().into_iter().map(|n| DirEntry { name: alloc::format!("{}", n), kind: InodeKind::Device }).collect())+    }+}++struct PtsNode(usize);++impl Inode for PtsNode {+    fn kind(&self) -> InodeKind {+        InodeKind::Device+    }+    fn size(&self) -> usize {+        0+    }+    fn mode(&self) -> u32 {+        0o620+    }+    fn open_device(&self) -> Result<Option<Arc<dyn File>>, Errno> {+        tty::open_pts(self.0).map(Some)     } } @@ -80,17 +127,17 @@ impl Inode for DevNode {     fn mode(&self) -> u32 {         0o666     }-    fn open_device(&self) -> Option<Arc<dyn File>> {-        Some(match self.0 {+    fn open_device(&self) -> Result<Option<Arc<dyn File>>, Errno> {+        Ok(Some(match self.0 {             "null" => Arc::new(Null),             "random" | "urandom" => Arc::new(Random),-            "hvc0" => {-                console::OPEN_COUNT.fetch_add(1, Ordering::Relaxed);-                Arc::new(Hvc)-            }-            "fb0" => Arc::new(Framebuffer::new(gpu::framebuffer()?)),-            _ => return None,-        })+            "console" => tty::console_file(),+            "hvc0" => tty::open_hvc(),+            "tty" => tty::open_ctty()?,+            "ptmx" => tty::open_ptmx()?,+            "fb0" => Arc::new(Framebuffer::new(gpu::framebuffer().ok_or(Errno::ENODEV)?)),+            _ => return Ok(None),+        }))     } } @@ -130,27 +177,6 @@ impl File for Random {     } } -struct Hvc;--impl File for Hvc {-    fn read(&self, buf: &mut [u8]) -> Result<usize, Errno> {-        console::read(buf)-    }-    fn write(&self, buf: &[u8]) -> Result<usize, Errno> {-        console::write(buf);-        Ok(buf.len()) // what didn't fit is dropped, like a serial line-    }-    fn stat(&self) -> Result<Stat, Errno> {-        device_stat(0)-    }-}--impl Drop for Hvc {-    fn drop(&mut self) {-        console::OPEN_COUNT.fetch_sub(1, Ordering::Relaxed);-    }-}- /// /dev/fb0: the framebuffer as a file of `stride * height * 4` bytes. /// Writing changes pixels in memory; they reach the screen on the FLUSH /// ioctl or when the file is closed (we remember which rows changed).

@@ -8,7 +8,8 @@ pub enum Errno {     ENOENT = 2,        // no such file or directory     ESRCH = 3,         // no such process (kill)     EINTR = 4,         // interrupted by a signal-    EIO = 5,           // input/output error (disk trouble)+    EIO = 5,           // input/output error (disk trouble; a pty whose other end closed)+    ENXIO = 6,         // no such device (/dev/tty without a controlling terminal)     E2BIG = 7,         // argument list too long     ENOEXEC = 8,       // not a valid executable     EBADF = 9,         // bad file descriptor@@ -49,6 +50,7 @@ pub enum Errno {     ENOTCONN = 107,       // not connected     ETIMEDOUT = 110,      // connect/DNS timed out     ECONNREFUSED = 111,   // nobody listens on that port (got a RST)+    EINPROGRESS = 115,    // nonblocking connect: started, not done yet     // Kernel-internal (never seen by programs): "a signal interrupted this     // call". On the way back to user space signal.rs turns them into     // EINTR or restarts the call (Linux uses the same numbers).@@ -60,12 +62,12 @@ impl Errno {     /// The errno for a negative syscall result like -2, if we know it.     pub fn from_ret(ret: i64) -> Option<Errno> {         use Errno::*;-        const ALL: [Errno; 46] = [+        const ALL: [Errno; 48] = [             EPERM, ESRCH, EINTR, ERESTARTSYS, ERESTARTNOHAND, ENOENT, EIO, E2BIG, ENOEXEC, EBADF, ECHILD, EAGAIN, ENOMEM, EFAULT, EBUSY, EEXIST, ENODEV,             ENOTDIR, EISDIR, EINVAL, EMFILE, ENOTTY, EFBIG, ENOSPC, ESPIPE, EROFS, EPIPE, ERANGE,             ENAMETOOLONG, ENOSYS, ENOTEMPTY, ELOOP, EOPNOTSUPP, ENOTSOCK, EDESTADDRREQ, EMSGSIZE,             EPROTONOSUPPORT, EAFNOSUPPORT, EADDRINUSE, ENETDOWN, ENETUNREACH, ECONNRESET, EISCONN,-            ENOTCONN, ETIMEDOUT, ECONNREFUSED,+            ENOTCONN, ETIMEDOUT, ECONNREFUSED, ENXIO, EINPROGRESS,         ];         ALL.into_iter().find(|&e| -(e as i64) == ret)     }

@@ -75,6 +75,41 @@ pub trait File: Send + Sync {     fn inode(&self) -> Option<Arc<dyn Inode>> {         None     }++    // --- stage 15: what event loops and a C library need -----------------++    /// Readiness right now, as poll(2) bits (crate::poll::POLLIN...): can a+    /// read / a write go ahead without blocking? Files that never block+    /// (regular files, /dev/null) are always ready.+    fn poll(&self) -> u16 {+        crate::poll::POLLIN | crate::poll::POLLOUT+    }+    /// O_NONBLOCK (fcntl F_SETFL, ioctl FIONBIO): instead of blocking,+    /// fail with EAGAIN. It belongs to the open file, so dup'ed and+    /// inherited fds share it. Only files that can block keep it.+    fn set_nonblocking(&self, _on: bool) {}+    fn nonblocking(&self) -> bool {+        false+    }+    /// Device-specific ioctls (ttys: termios, window size...).+    fn ioctl(&self, _cmd: u64, _arg: u64) -> Result<u64, Errno> {+        Err(Errno::ENOTTY)+    }+    /// pread/pwrite: at an explicit offset, leaving the file offset alone.+    fn read_at(&self, _buf: &mut [u8], _offset: u64) -> Result<usize, Errno> {+        Err(Errno::ESPIPE)+    }+    fn write_at(&self, _buf: &[u8], _offset: u64) -> Result<usize, Errno> {+        Err(Errno::ESPIPE)+    }+    /// epoll_ctl / epoll_wait need the epoll instance behind the fd.+    fn as_epoll(&self) -> Option<&crate::poll::Epoll> {+        None+    }+    /// The terminal behind the fd, if it is one (tty.rs).+    fn as_tty(&self) -> Option<&crate::tty::TtyFile> {+        None+    } }  /// A file or directory opened through the VFS: inode + current position.@@ -134,6 +169,20 @@ impl File for InodeFile {         Ok(Stat::of(&*self.inode))     } +    fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Errno> {+        match self.inode.kind() {+            InodeKind::Directory => Err(Errno::EISDIR),+            _ => self.inode.read_at(offset as usize, buf),+        }+    }++    fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Errno> {+        match self.inode.kind() {+            InodeKind::Directory => Err(Errno::EISDIR),+            _ => self.inode.write_at(offset as usize, buf),+        }+    }+     fn inode(&self) -> Option<Arc<dyn Inode>> {         (self.inode.kind() == InodeKind::File).then(|| self.inode.clone())     }

@@ -106,8 +106,8 @@ pub trait Inode: Send + Sync {     }     /// Device files: the `File` that `open` should return (None = an     /// ordinary inode, opened as an `InodeFile`).-    fn open_device(&self) -> Option<Arc<dyn file::File>> {-        None+    fn open_device(&self) -> Result<Option<Arc<dyn file::File>>, Errno> {+        Ok(None)     } } 

@@ -6,20 +6,31 @@ //! every write end is closed (then read returns 0 = EOF); writing when every //! read end is closed fails with EPIPE. Closing = dropping the last `Arc` //! of an end (fd closed, process exited), which our `Drop` impls notice.-//! Simplification: the buffer is unbounded, so writers never block.+//!+//! The buffer holds at most PIPE_CAPACITY bytes (64 KiB, like Linux): a+//! writer that runs ahead of its reader blocks until there's room. With+//! O_NONBLOCK, instead of blocking, a read of an empty pipe or a write to+//! a full one fails with EAGAIN ("try again later"); event loops use+//! poll/epoll (poll.rs) to learn when that is.  use super::file::{File, KIND_PIPE, Stat}; use crate::errno::Errno;+use crate::poll::{self, POLLERR, POLLHUP, POLLIN, POLLOUT}; use crate::thread::WaitQueue; use alloc::{collections::VecDeque, sync::Arc};+use core::sync::atomic::{AtomicBool, Ordering}; use spin::Mutex; use x86_64::instructions::interrupts::without_interrupts; +/// How many bytes a pipe buffers before writers have to wait.+pub const PIPE_CAPACITY: usize = 64 * 1024;+ struct Pipe {     // Locked with interrupts off only (it's touched in `wait_until`'s     // condition, see the locking rule in thread.rs).     state: Mutex<PipeState>,     readers_wait: WaitQueue,+    writers_wait: WaitQueue, }  struct PipeState {@@ -28,16 +39,17 @@ struct PipeState {     read_closed: bool, } -pub struct PipeReader(Arc<Pipe>);-pub struct PipeWriter(Arc<Pipe>);+pub struct PipeReader(Arc<Pipe>, AtomicBool);+pub struct PipeWriter(Arc<Pipe>, AtomicBool);  /// A new pipe: (read end, write end). pub fn new() -> (PipeReader, PipeWriter) {     let pipe = Arc::new(Pipe {         state: Mutex::new(PipeState { buffer: VecDeque::new(), write_closed: false, read_closed: false }),         readers_wait: WaitQueue::new(),+        writers_wait: WaitQueue::new(),     });-    (PipeReader(pipe.clone()), PipeWriter(pipe))+    (PipeReader(pipe.clone(), AtomicBool::new(false)), PipeWriter(pipe, AtomicBool::new(false))) }  impl File for PipeReader {@@ -45,18 +57,28 @@ impl File for PipeReader {         if buf.is_empty() {             return Ok(0);         }+        let nonblocking = self.nonblocking();         // Block until there's data or no writer is left (or a signal).-        self.0.readers_wait.wait_until_intr(|| {+        let n = self.0.readers_wait.wait_until_intr(|| {             let mut state = self.0.state.lock();             if state.buffer.is_empty() {-                return if state.write_closed { Some(0) } else { None };+                return match () {+                    _ if state.write_closed => Some(Ok(0)),+                    _ if nonblocking => Some(Err(Errno::EAGAIN)),+                    _ => None,+                };             }             let n = buf.len().min(state.buffer.len());             for (dst, src) in buf.iter_mut().zip(state.buffer.drain(..n)) {                 *dst = src;             }-            Some(n)-        })+            Some(Ok(n))+        })??;+        if n > 0 {+            self.0.writers_wait.wake_all(); // there's room now+            poll::notify();+        }+        Ok(n)     }     fn write(&self, _buf: &[u8]) -> Result<usize, Errno> {         Err(Errno::EBADF) // wrong end@@ -65,38 +87,99 @@ impl File for PipeReader {         let size = without_interrupts(|| self.0.state.lock().buffer.len());         Ok(Stat { kind: KIND_PIPE, size: size as u64, mode: 0 })     }+    fn poll(&self) -> u16 {+        without_interrupts(|| {+            let state = self.0.state.lock();+            let mut events = 0;+            if !state.buffer.is_empty() {+                events |= POLLIN;+            }+            if state.write_closed {+                events |= POLLHUP; // (a read returns 0 = EOF)+            }+            events+        })+    }+    fn set_nonblocking(&self, on: bool) {+        self.1.store(on, Ordering::Relaxed);+    }+    fn nonblocking(&self) -> bool {+        self.1.load(Ordering::Relaxed)+    } }  impl File for PipeWriter {     fn read(&self, _buf: &mut [u8]) -> Result<usize, Errno> {         Err(Errno::EBADF)     }+    /// Write all of `buf`, waiting for room as needed. Interrupted by a+    /// signal (or non-blocking) after some bytes: return that count.     fn write(&self, buf: &[u8]) -> Result<usize, Errno> {-        without_interrupts(|| {+        let nonblocking = self.nonblocking();+        let mut written = 0;+        let result = self.0.writers_wait.wait_until_intr(|| {             let mut state = self.0.state.lock();             if state.read_closed {-                return Err(Errno::EPIPE);+                return Some(Err(Errno::EPIPE));+            }+            let room = PIPE_CAPACITY - state.buffer.len();+            let n = room.min(buf.len() - written);+            state.buffer.extend(&buf[written..written + n]);+            written += n;+            if n > 0 {+                // Wake readers now, not after the loop: if the pipe is full+                // we're about to sleep until one of them makes room.+                self.0.readers_wait.wake_all();+                poll::notify();+            }+            match () {+                _ if written == buf.len() => Some(Ok(written)),+                _ if nonblocking && written > 0 => Some(Ok(written)),+                _ if nonblocking => Some(Err(Errno::EAGAIN)),+                _ => None, // full: wait for a reader to make room             }-            state.buffer.extend(buf);-            Ok(buf.len())-        })?;-        self.0.readers_wait.wake_all();-        Ok(buf.len())+        });+        match result {+            Ok(r) => r,+            Err(_) if written > 0 => Ok(written), // a signal, after some bytes+            Err(e) => Err(e),+        }     }     fn stat(&self) -> Result<Stat, Errno> {         Ok(Stat { kind: KIND_PIPE, size: 0, mode: 0 })     }+    fn poll(&self) -> u16 {+        without_interrupts(|| {+            let state = self.0.state.lock();+            if state.read_closed {+                POLLERR // (a write fails with EPIPE)+            } else if state.buffer.len() < PIPE_CAPACITY {+                POLLOUT+            } else {+                0+            }+        })+    }+    fn set_nonblocking(&self, on: bool) {+        self.1.store(on, Ordering::Relaxed);+    }+    fn nonblocking(&self) -> bool {+        self.1.load(Ordering::Relaxed)+    } }  impl Drop for PipeWriter {     fn drop(&mut self) {         without_interrupts(|| self.0.state.lock().write_closed = true);         self.0.readers_wait.wake_all(); // readers see EOF+        poll::notify();     } }  impl Drop for PipeReader {     fn drop(&mut self) {         without_interrupts(|| self.0.state.lock().read_closed = true);+        self.0.writers_wait.wake_all(); // writers get EPIPE+        poll::notify();     } }

@@ -0,0 +1,218 @@+//! Futexes ("fast userspace mutexes"): the one syscall threads need to+//! sleep and wake each other.+//!+//! A lock in a C library is just an integer in memory. Taking a free lock+//! is one atomic instruction in user space -- no syscall at all. Only when+//! a thread has to *wait* does it ask the kernel:+//!+//!   futex(&word, FUTEX_WAIT, expected): "put me to sleep, but only if+//!                                        word still == expected"+//!   futex(&word, FUTEX_WAKE, n):        "wake up to n sleepers on word"+//!+//! The "only if still equal" check is the whole trick: the kernel does it+//! while holding its own lock, the same lock FUTEX_WAKE takes. So if the+//! lock holder releases the lock (changes the word) and calls WAKE between+//! our user-space check and our WAIT, either WAIT sees the changed word+//! and returns EAGAIN at once, or we're already queued and WAKE finds us.+//! No wake-up can get lost.+//!+//! The kernel keeps no state per futex word: just a list of sleepers,+//! each labelled with the word it sleeps on (its *key*). For memory private+//! to one process the key is (address space, virtual address); for+//! MAP_SHARED memory, which different processes may map at different+//! addresses, it's the physical address.+//!+//! pthread_join uses it too: the kernel writes 0 to the exiting thread's+//! tid word and does a FUTEX_WAKE on it (CLONE_CHILD_CLEARTID).++use crate::errno::Errno;+use crate::memory;+use crate::process::Process;+use crate::thread::{self, Tid, WaitQueue};+use alloc::vec::Vec;+use core::sync::atomic::{AtomicU64, Ordering};+use spin::Mutex;+use x86_64::instructions::interrupts::without_interrupts;++pub const FUTEX_WAIT: u64 = 0;+pub const FUTEX_WAKE: u64 = 1;+pub const FUTEX_REQUEUE: u64 = 3;+pub const FUTEX_CMP_REQUEUE: u64 = 4;+pub const FUTEX_WAIT_BITSET: u64 = 9;+pub const FUTEX_WAKE_BITSET: u64 = 10;+pub const FUTEX_PRIVATE_FLAG: u64 = 128;+pub const FUTEX_CLOCK_REALTIME: u64 = 256;+pub const FUTEX_BITSET_MATCH_ANY: u32 = u32::MAX;++/// Which word a sleeper waits on.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+enum Key {+    /// (level-4 page table = the address space, virtual address)+    Private(u64, u64),+    /// Physical address (MAP_SHARED memory).+    Shared(u64),+}++struct Waiter {+    id: u64,+    key: Key,+    bitset: u32,+    kthread: Tid,+    woken: bool,+}++/// All sleepers. Locked with interrupts off (read in wait conditions).+static WAITERS: Mutex<Vec<Waiter>> = Mutex::new(Vec::new());+/// Only used for its sleeping logic: wakers wake threads one by one.+static QUEUE: WaitQueue = WaitQueue::new();+static NEXT_ID: AtomicU64 = AtomicU64::new(1);++/// Look up the key of `uaddr` in `p` and read the word there (through the+/// physical mapping, with the address space locked so it can't change+/// under us). The page must have been faulted in.+fn key_and_value(p: &Process, uaddr: u64, private: bool) -> Result<(Key, u32), Errno> {+    let mm = p.mm.lock();+    let (frame, _) = memory::user_pte(mm.l4, uaddr).ok_or(Errno::EFAULT)?;+    let phys = frame.start_address().as_u64() + (uaddr & 0xfff);+    let value = unsafe { core::ptr::read_volatile(memory::phys_to_virt(x86_64::PhysAddr::new(phys)).as_ptr::<u32>()) };+    let shared = !private && mm.vmas().iter().any(|v| v.shared && (v.start..v.end).contains(&uaddr));+    let key = if shared { Key::Shared(phys) } else { Key::Private(mm.l4.start_address().as_u64(), uaddr) };+    Ok((key, value))+}++/// Check a futex address and fault its page in.+fn prepare(uaddr: u64) -> Result<(), Errno> {+    if uaddr % 4 != 0 {+        return Err(Errno::EINVAL);+    }+    if !crate::mm::user_access_ok(uaddr, 4, false) {+        return Err(Errno::EFAULT);+    }+    Ok(())+}++/// FUTEX_WAIT: sleep while *uaddr == val, until woken, `deadline` (PIT+/// tick) or a signal.+pub fn wait(p: &Process, uaddr: u64, val: u32, deadline: Option<u64>, bitset: u32, private: bool) -> Result<u64, Errno> {+    if bitset == 0 {+        return Err(Errno::EINVAL);+    }+    prepare(uaddr)?;+    let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);+    // Compare and enqueue in one step under WAITERS (see the top).+    without_interrupts(|| {+        let (key, value) = key_and_value(p, uaddr, private)?;+        let mut waiters = WAITERS.lock();+        if value != val {+            return Err(Errno::EAGAIN);+        }+        waiters.push(Waiter { id, key, bitset, kthread: thread::current_id(), woken: false });+        Ok(())+    })?;+    let woken = || WAITERS.lock().iter().any(|w| w.id == id && w.woken).then_some(());+    let result = match deadline {+        Some(deadline) => QUEUE.wait_until_deadline_intr(deadline, woken),+        None => QUEUE.wait_until_intr(woken).map(Some),+    };+    // Leave the list; if a wake-up raced with the timeout/signal, it counts.+    let was_woken = without_interrupts(|| {+        let mut waiters = WAITERS.lock();+        let i = waiters.iter().position(|w| w.id == id).expect("futex waiter vanished");+        waiters.swap_remove(i).woken+    });+    match result {+        _ if was_woken => Ok(0),+        Ok(_) => Err(Errno::ETIMEDOUT),+        Err(_) => Err(Errno::EINTR),+    }+}++/// Wake up to `n` sleepers on `key` (matching `bitset`); move up to+/// `requeue` more to `to` instead. Returns how many were woken (+ moved).+fn wake_key(key: Key, n: u64, bitset: u32, requeue: Option<(Key, u64)>) -> u64 {+    let mut kthreads = Vec::new();+    let mut moved = 0;+    without_interrupts(|| {+        let mut waiters = WAITERS.lock();+        for w in waiters.iter_mut().filter(|w| w.key == key && !w.woken && w.bitset & bitset != 0) {+            if (kthreads.len() as u64) < n {+                w.woken = true;+                kthreads.push(w.kthread);+            } else if let Some((to, max)) = requeue {+                if moved < max {+                    w.key = to;+                    moved += 1;+                }+            }+        }+    });+    for &k in &kthreads {+        thread::wake(k);+    }+    kthreads.len() as u64 + moved+}++/// FUTEX_WAKE from the kernel (thread exit, CLONE_CHILD_CLEARTID).+pub fn wake(p: &Process, uaddr: u64, n: u64) -> u64 {+    match prepare(uaddr).and_then(|_| without_interrupts(|| key_and_value(p, uaddr, false))) {+        Ok((key, _)) => wake_key(key, n, FUTEX_BITSET_MATCH_ANY, None),+        Err(_) => 0,+    }+}++/// How many sleepers there are (tests: nothing left behind).+pub fn waiters() -> usize {+    without_interrupts(|| WAITERS.lock().len())+}++/// The futex syscall (202): futex(uaddr, op, val, timeout/val2, uaddr2, val3).+pub fn sys_futex(p: &Process, uaddr: u64, op: u64, val: u64, timeout: u64, uaddr2: u64, val3: u64) -> Result<u64, Errno> {+    let private = op & FUTEX_PRIVATE_FLAG != 0;+    let realtime = op & FUTEX_CLOCK_REALTIME != 0;+    let cmd = op & !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME);+    match cmd {+        FUTEX_WAIT | FUTEX_WAIT_BITSET => {+            let bitset = if cmd == FUTEX_WAIT { FUTEX_BITSET_MATCH_ANY } else { val3 as u32 };+            let deadline = match timeout {+                0 => None,+                ptr => {+                    let us = crate::syscall::read_timespec_us(ptr)?;+                    // FUTEX_WAIT: relative. FUTEX_WAIT_BITSET: an absolute+                    // time on CLOCK_MONOTONIC (or REALTIME with the flag).+                    let us = if cmd == FUTEX_WAIT {+                        us+                    } else {+                        let now = if realtime { crate::time::realtime_us() } else { crate::time::now_us() };+                        us.saturating_sub(now)+                    };+                    if us == 0 {+                        return Err(Errno::ETIMEDOUT);+                    }+                    Some(crate::time::deadline_after_us(us))+                }+            };+            wait(p, uaddr, val as u32, deadline, bitset, private)+        }+        FUTEX_WAKE | FUTEX_WAKE_BITSET => {+            let bitset = if cmd == FUTEX_WAKE { FUTEX_BITSET_MATCH_ANY } else { val3 as u32 };+            if bitset == 0 {+                return Err(Errno::EINVAL);+            }+            prepare(uaddr)?;+            let (key, _) = without_interrupts(|| key_and_value(p, uaddr, private))?;+            Ok(wake_key(key, val, bitset, None))+        }+        FUTEX_REQUEUE | FUTEX_CMP_REQUEUE => {+            prepare(uaddr)?;+            prepare(uaddr2)?;+            let (key, value) = without_interrupts(|| key_and_value(p, uaddr, private))?;+            if cmd == FUTEX_CMP_REQUEUE && value != val3 as u32 {+                return Err(Errno::EAGAIN);+            }+            let (to, _) = without_interrupts(|| key_and_value(p, uaddr2, private))?;+            // (val2 travels in the timeout argument's register.)+            Ok(wake_key(key, val, FUTEX_BITSET_MATCH_ANY, Some((to, timeout))))+        }+        _ => Err(Errno::ENOSYS),+    }+}

@@ -4,8 +4,9 @@ //! a handler function: //!   * vectors 0..=31  are CPU exceptions (divide error, page fault, ...) //!   * vectors 32..=47 are where we remap the 8259 PIC's 16 hardware IRQs-//!   * 0x40, 0x41, 0xFF come from the local APIC (see apic.rs): its timer,-//!     the "reschedule" IPI one CPU sends another, and spurious interrupts+//!   * 0x40, 0x41, 0x42, 0xFF come from the local APIC (see apic.rs): its+//!     timer, the "reschedule" and "TLB shootdown" IPIs one CPU sends+//!     another, and spurious interrupts //! //! All CPUs share this one IDT. Device IRQs (PIT, keyboard) only ever //! arrive on CPU 0; APIC timer ticks and IPIs arrive on every CPU.@@ -44,6 +45,7 @@ pub enum InterruptIndex {     Keyboard,             // IRQ 1: PS/2 keyboard     ApicTimer = 0x40,     // each CPU's local APIC timer: time slices (~100 Hz)     Reschedule = 0x41,    // IPI: "wake up, there's work for you"+    TlbShootdown = 0x42,  // IPI: "flush your TLB" (smp::tlb_shootdown)     ApicSpurious = 0xFF,  // the APIC's "never mind" interrupt } @@ -89,6 +91,7 @@ lazy_static! {         idt[PIC_1_OFFSET + 10].set_handler_fn(pci_irq10_handler);         idt[PIC_1_OFFSET + 11].set_handler_fn(pci_irq11_handler);         idt[InterruptIndex::Reschedule.as_u8()].set_handler_fn(reschedule_handler);+        idt[InterruptIndex::TlbShootdown.as_u8()].set_handler_fn(tlb_shootdown_handler);         idt[InterruptIndex::ApicSpurious.as_u8()].set_handler_fn(spurious_handler);         idt     };@@ -145,8 +148,9 @@ fn exception(frame: &TrapFrame, sig: u32, code: i32, addr: u64, what: &'static s /// usually *not* an error: demand paging and copy-on-write (mm.rs) fix the /// page up and the instruction runs again. Only if the address isn't in /// any region (or the region forbids the access) does the program get-/// SIGSEGV. The kernel itself never faults on user memory (syscalls check-/// buffers first), so a kernel-mode page fault is a bug.+/// SIGSEGV. The kernel itself faults on user memory only when another+/// thread changed it during a syscall (below); any other kernel-mode page+/// fault is a bug. fn page_fault(frame: &mut TrapFrame) {     use crate::mm::Fault;     use x86_64::registers::control::Cr2;@@ -178,6 +182,30 @@ fn page_fault(frame: &mut TrapFrame) {         signal::force(sig, SigInfo::fault(sig, code, addr), why);         return;     }+    // The kernel touched user memory a syscall had checked (user_slice), but+    // another thread of the process changed the mapping since: munmap'd it,+    // or fork made it copy-on-write. Fix it up like a user fault; if the+    // memory is really gone, map a blank page so the kernel can finish, and+    // kill the process (Linux would return EFAULT; we have no way back to+    // the syscall from here).+    if (crate::memory::USER_START..crate::memory::USER_END).contains(&addr) {+        if let Some(p) = crate::thread::current_process() {+            if frame.rflags & (1 << 9) != 0 {+                x86_64::instructions::interrupts::enable(); // as it was+            }+            let write = error.contains(PageFaultErrorCode::CAUSED_BY_WRITE);+            let mut mm = p.mm.lock();+            if mm.handle_fault(addr, write, false).is_ok() {+                return;+            }+            serial_println!("[kernel] syscall touched unmapped user memory {:#x} (another thread unmapped it?)", addr);+            if mm.map_scratch(addr).is_some() {+                drop(mm);+                signal::force(signal::SIGSEGV, SigInfo::fault(signal::SIGSEGV, signal::SEGV_MAPERR, addr), "bad user pointer in a syscall");+                return;+            }+        }+    }     // The kernel touched a kernel stack's guard page: that thread overflowed.     if crate::stack::is_guard_page(addr) {         crate::thread::die_of_stack_overflow(addr);@@ -277,6 +305,12 @@ extern "x86-interrupt" fn reschedule_handler(_stack_frame: InterruptStackFrame)      crate::thread::schedule(); } +/// Another CPU changed page tables we may have cached (threads, smp.rs).+extern "x86-interrupt" fn tlb_shootdown_handler(_stack_frame: InterruptStackFrame) {+    crate::smp::handle_tlb_shootdown();+    crate::apic::eoi();+}+ /// Spurious APIC interrupt: ignore it (and no EOI, by the spec). extern "x86-interrupt" fn spurious_handler(_stack_frame: InterruptStackFrame) {} 

@@ -25,11 +25,12 @@ pub mod acpi; // ACPI tables: just enough to list the CPUs (MADT) pub mod allocator; // heap: maps heap pages + the #[global_allocator] pub mod apic; // local APIC: per-CPU timer, inter-processor interrupts pub mod block; // block devices: BlockDevice trait, virtio-blk, partitions-pub mod dev; // /dev: null, random, hvc0, fb0+pub mod dev; // /dev: null, random, hvc0, fb0, tty, ptmx, pts/N pub mod elf; // ELF64 loader for user programs pub mod errno; // Linux-compatible error numbers pub mod fbcon; // text console drawn on the framebuffer pub mod font; // 8x16 bitmap font (Spleen) for fbcon+pub mod futex; // futexes: user-space locks' sleep/wake (threads) pub mod fs; // VFS: Inode/FileSystem/File traits, mount table, ramfs pub mod gdt; // GDT (kernel + user segments) + TSS (RSP0, double-fault stack) pub mod interrupts; // IDT, exception handlers, PIC, timer + keyboard@@ -37,6 +38,7 @@ pub mod net; // networking: virtio-net driver + smoltcp TCP/IP, sockets pub mod memory; // paging, frame allocator, page tables, shared-frame counts pub mod mm; // a process's memory map: regions, demand paging, copy-on-write pub mod pci; // PCI bus enumeration (config space via ports 0xCF8/0xCFC)+pub mod poll; // poll, select, epoll: waiting for many files at once pub mod process; // processes: fork, execve, exit, wait4, process groups pub mod serial; // COM1 serial port pub mod signal; // signals: kill, handlers, signal frames, default actions@@ -47,7 +49,7 @@ pub mod task; // cooperative multitasking: async tasks + executor pub mod trap; // TrapFrame: saved user registers, the common return to ring 3 pub mod time; // microsecond clock (TSC calibrated against the PIT) pub mod thread; // threads + the multi-core scheduler (sleep/wake, wait queues)-pub mod tty; // console: keyboard line discipline + fd 0/1/2+pub mod tty; // terminals: line discipline, termios, console/hvc0/ptys pub mod vga_buffer; // the console: VGA text mode, or fbcon on a GPU pub mod virtio; // modern virtio transport + virtio-rng, -console, -gpu @@ -102,10 +104,10 @@ pub fn init_drivers() -> alloc::vec::Vec<alloc::string::String> {     log } -/// Networking: find the virtio-net card, start smoltcp, DHCP and the netd-/// thread. Needs the scheduler (call after `init_smp`). Returns the boot-/// log line, or None without a network card.-pub fn init_net() -> Option<alloc::string::String> {+/// Networking: loopback, plus the virtio-net card if there is one (smoltcp,+/// DHCP and the netd thread). Needs the scheduler (call after `init_smp`).+/// Returns the boot log lines.+pub fn init_net() -> alloc::vec::Vec<alloc::string::String> {     net::init(&pci::scan()) } 

@@ -98,10 +98,14 @@ fn kernel_main(boot_info: &'static BootInfo) -> ! {         kprintln!("[virtio] {}", line);     } -    // --- Network (net/): virtio-net + smoltcp; DHCP runs in the background ---    match rustbelt::init_net() {-        Some(line) => kprintln!("[net] {}", line),-        None => kprintln!("[net] no virtio-net card (start QEMU with -device virtio-net-pci)"),+    // --- Network (net/): loopback + virtio-net + smoltcp; DHCP runs in the+    // background ----------------------------------------------------------+    let net = rustbelt::init_net();+    for line in &net {+        kprintln!("[net] {}", line);+    }+    if !rustbelt::net::present() {+        kprintln!("[net] no virtio-net card (start QEMU with -device virtio-net-pci)");     }      // --- The shell (user/sh.rs), run by its own kernel thread ---------------@@ -135,6 +139,11 @@ fn shell_supervisor() {         rustbelt::exit_qemu(rustbelt::QemuExitCode::Failed);         return;     }+    // Start hvc0's tty (and its input thread `hvcd`) now rather than on+    // the first open, so its kernel stack isn't counted as a leak below.+    if rustbelt::virtio::console::present() {+        rustbelt::tty::hvc();+    }     loop {         #[cfg(feature = "headless")]         let frames_before = rustbelt::memory::frames_in_use();

@@ -214,12 +214,48 @@ impl Mm {     pub fn unmap(&mut self, start: u64, end: u64) {         self.split_at(start, end);         self.vmas.retain(|v| v.end <= start || v.start >= end);+        let mut freed = Vec::new();         for addr in (start..end).step_by(PAGE_SIZE as usize) {             if let Some(frame) = memory::unmap_user_page(self.l4, addr) {-                memory::release_frame(frame);                 memory::flush_user_page(addr);+                freed.push(frame);             }         }+        // Other threads' CPUs must forget the pages *before* the frames+        // can be reused for something else.+        if !freed.is_empty() {+            self.shootdown();+        }+        for frame in freed {+            memory::release_frame(frame);+        }+    }++    /// Map a blank, writable page at `addr` that belongs to no region (see+    /// the kernel-mode page fault in interrupts.rs). Freed with the address+    /// space.+    pub fn map_scratch(&mut self, addr: u64) -> Option<()> {+        let page = addr & !(PAGE_SIZE - 1);+        let frame = memory::alloc_zeroed_frame()?;+        let mut flags = Flags::PRESENT | Flags::WRITABLE | Flags::USER_ACCESSIBLE;+        if memory::nx_enabled() {+            flags |= Flags::NO_EXECUTE;+        }+        match memory::user_pte(self.l4, page) {+            Some(_) => {+                let old = memory::replace_user_frame(self.l4, page, frame, flags)?;+                memory::flush_user_page(page);+                memory::release_frame(old);+            }+            None => memory::map_user_frame(self.l4, page, frame, flags)?,+        }+        Some(())+    }++    /// Make every other CPU running this address space flush its TLB+    /// (threads: see smp::tlb_shootdown).+    pub fn shootdown(&self) {+        crate::smp::tlb_shootdown(self.l4.start_address().as_u64());     }      /// mprotect: new permissions for `start..end`, which must be fully@@ -248,6 +284,7 @@ impl Mm {                 }             }         }+        self.shootdown(); // permissions may have shrunk         Ok(())     } @@ -345,6 +382,9 @@ impl Mm {                     }                     let flags = vma.pte_flags(true);                     memory::replace_user_frame(self.l4, page, copy, flags).ok_or(Fault::OutOfMemory)?;+                    // Other threads of ours may still see the old frame.+                    memory::flush_user_page(page);+                    self.shootdown();                     memory::release_frame(frame); // our share of the original                 }             }@@ -412,6 +452,7 @@ impl Mm {             if shared { flags } else { flags - Flags::WRITABLE }         });         flush_all();+        self.shootdown(); // our other threads' CPUs: no more writable entries         ok.map(|_| child) // on failure, dropping `child` frees what it got     } 

@@ -0,0 +1,121 @@+//! The one "device" smoltcp talks to: the virtio-net card (if there is+//! one) *plus* a loopback queue.+//!+//! **Loopback.** Programs talk to themselves over the network all the+//! time: a database on 127.0.0.1, a test client and server, a compositor+//! and its clients. Linux has a separate `lo` interface for that. We keep+//! smoltcp's single Ethernet interface and give it a second address,+//! 127.0.0.1/8, next to DHCP's. Then this device does the trick: a frame+//! meant for *ourselves* never goes to the card, it's put on the `lo`+//! queue, and `receive` hands queued frames back to smoltcp first.+//! "Meant for ourselves" is:+//!   * an ARP request asking who has 127.x.x.x or our own DHCP address+//!     (smoltcp then answers itself: "that's me", with its own MAC), and+//!   * any frame sent to our own MAC address.+//! So 127.0.0.1 works with no network card at all (MAC 02:00:00:00:00:01,+//! a "locally administered" one), and connecting to our own 10.0.2.15+//! doesn't leave the machine either, just like Linux.++use super::virtio_net::{MTU, VirtioNet};+use alloc::collections::VecDeque;+use alloc::vec;+use alloc::vec::Vec;+use smoltcp::phy::{self, DeviceCapabilities, Medium};+use smoltcp::time::Instant;++/// Our MAC when there's no card.+pub const LOOPBACK_MAC: [u8; 6] = [0x02, 0, 0, 0, 0, 1];+/// Frames waiting on lo at most (more are dropped, like a full queue).+const LO_QUEUE: usize = 256;++pub struct NetDevice {+    pub nic: Option<VirtioNet>,+    pub mac: [u8; 6],+    /// Our DHCP address (frames ARPing for it are ours).+    pub own_ip: Option<[u8; 4]>,+    lo: VecDeque<Vec<u8>>,+    pub lo_packets: u64,+}++impl NetDevice {+    pub fn new(nic: Option<VirtioNet>) -> NetDevice {+        let mac = nic.as_ref().map_or(LOOPBACK_MAC, |n| n.mac);+        NetDevice { nic, mac, own_ip: None, lo: VecDeque::new(), lo_packets: 0 }+    }++    /// Frames are waiting on lo (net::poll then polls smoltcp again).+    pub fn lo_pending(&self) -> bool {+        !self.lo.is_empty()+    }++    /// Is this outgoing Ethernet frame addressed to us? (See the top.)+    fn is_for_us(&self, frame: &[u8]) -> bool {+        if frame.len() < 14 {+            return false;+        }+        if frame[0..6] == self.mac {+            return true;+        }+        // ARP (ethertype 0x0806): the asked-for IP is at bytes 38..42.+        if frame[12..14] == [0x08, 0x06] && frame.len() >= 42 {+            let target = [frame[38], frame[39], frame[40], frame[41]];+            return target[0] == 127 || Some(target) == self.own_ip;+        }+        false+    }+}++// smoltcp asks a `Device` for *tokens*: an RxToken is "a received frame you+// may look at", a TxToken is "permission to build one frame to send".++pub struct RxToken(Vec<u8>);+pub struct TxToken<'a>(&'a mut NetDevice);++impl phy::Device for NetDevice {+    type RxToken<'a> = RxToken;+    type TxToken<'a> = TxToken<'a>;++    fn receive(&mut self, _now: Instant) -> Option<(RxToken, TxToken<'_>)> {+        // Loopback first, then the card.+        let frame = match self.lo.pop_front() {+            Some(f) => f,+            None => self.nic.as_mut()?.receive_frame()?,+        };+        Some((RxToken(frame), TxToken(self)))+    }++    fn transmit(&mut self, _now: Instant) -> Option<TxToken<'_>> {+        Some(TxToken(self))+    }++    fn capabilities(&self) -> DeviceCapabilities {+        let mut caps = DeviceCapabilities::default();+        caps.medium = Medium::Ethernet;+        caps.max_transmission_unit = MTU;+        caps+    }+}++impl phy::RxToken for RxToken {+    fn consume<R, F: FnOnce(&[u8]) -> R>(self, f: F) -> R {+        f(&self.0)+    }+}++impl phy::TxToken for TxToken<'_> {+    /// smoltcp builds the frame in `f`; we then decide where it goes.+    fn consume<R, F: FnOnce(&mut [u8]) -> R>(self, len: usize, f: F) -> R {+        let dev = self.0;+        let mut frame = vec![0; len.min(MTU)];+        let result = f(&mut frame);+        if dev.is_for_us(&frame) {+            if dev.lo.len() < LO_QUEUE {+                dev.lo_packets += 1;+                dev.lo.push_back(frame);+            }+        } else if let Some(nic) = dev.nic.as_mut() {+            nic.send_frame(frame.len(), |buf| buf.copy_from_slice(&frame[..buf.len()]));+        } // no card: nowhere to send it+        result+    }+}

@@ -2,6 +2,8 @@ //! //! The layers, bottom to top (see the README's networking chapter): //!   * virtio_net.rs  moves raw Ethernet frames in and out of the card.+//!   * device.rs      what smoltcp sees: the card plus a loopback queue,+//!                    so 127.0.0.1 works (even with no card at all). //!   * smoltcp        does everything protocol-shaped: Ethernet addressing, //!                    ARP (who has 10.0.2.2?), IPv4, ICMP (ping), UDP, TCP //!                    (connections, retransmits, windows), DHCP and DNS.@@ -22,6 +24,7 @@ //! Locking: `NET` is taken with interrupts off (wait-queue conditions look //! at sockets with interrupts off; see the locking rule in thread.rs). +pub mod device; pub mod socket; pub mod virtio_net; @@ -35,6 +38,7 @@ use smoltcp::socket::{dhcpv4, dns, tcp}; use smoltcp::time::Instant; use smoltcp::wire::{DnsQueryType, EthernetAddress, HardwareAddress, IpAddress, IpCidr, Ipv4Address}; use spin::Mutex;+use device::NetDevice; use virtio_net::VirtioNet; use x86_64::instructions::interrupts::without_interrupts; use x86_64::instructions::port::Port;@@ -50,7 +54,7 @@ pub struct IpConfig {  pub struct Net {     pub iface: Interface,-    pub device: VirtioNet,+    pub device: NetDevice,     pub sockets: SocketSet<'static>,     dhcp: SocketHandle,     dns: SocketHandle,@@ -71,7 +75,7 @@ static IRQ_PENDING: AtomicBool = AtomicBool::new(false); /// without taking the NET lock. static IO_BASE: AtomicU16 = AtomicU16::new(0); -/// Run `f` on the network state (None if there's no network card).+/// Run `f` on the network state (None before `init`). pub fn with_net<R>(f: impl FnOnce(&mut Net) -> R) -> Option<R> {     without_interrupts(|| NET.lock().as_mut().map(f)) }@@ -81,35 +85,53 @@ pub fn now() -> Instant {     Instant::from_micros(crate::time::now_us() as i64) } -/// Find the first virtio-net card, set up smoltcp on it, start DHCP and-/// the netd thread. Returns a line for the boot log.-pub fn init(devices: &[crate::pci::PciDevice]) -> Option<String> {-    let pci = devices.iter().find(|d| virtio_net::is_virtio_net(d))?;-    let mut device = VirtioNet::new(pci)?;+/// The loopback address, 127.0.0.1/8.+fn loopback_cidr() -> IpCidr {+    IpCidr::new(IpAddress::Ipv4(Ipv4Address::new(127, 0, 0, 1)), 8)+}++/// Is this address on our loopback network (127.x.x.x)?+pub fn is_loopback(ip: [u8; 4]) -> bool {+    ip[0] == 127+}++/// Set up smoltcp on the first virtio-net card (if any) plus loopback,+/// start DHCP and the netd thread. Returns lines for the boot log.+pub fn init(devices: &[crate::pci::PciDevice]) -> Vec<String> {+    let nic = devices.iter().find(|d| virtio_net::is_virtio_net(d)).and_then(VirtioNet::new);+    let irq = nic.as_ref().map(|n| n.irq);+    let io_base = nic.as_ref().map_or(0, |n| n.io_base);+    let mut device = NetDevice::new(nic);     let mac = device.mac;     let mut config = Config::new(HardwareAddress::Ethernet(EthernetAddress(mac)));     // Random seed: TCP's initial sequence numbers and DHCP transaction ids.     // From virtio-rng (rng.rs) -- guessable ones would let an attacker     // inject packets into our connections.     config.random_seed = crate::virtio::rng::u64();-    let iface = Interface::new(config, &mut device, now());+    let mut iface = Interface::new(config, &mut device, now());+    iface.update_ip_addrs(|addrs| {+        addrs.push(loopback_cidr()).ok();+    });     let mut sockets = SocketSet::new(Vec::new());     let dhcp = sockets.add(dhcpv4::Socket::new());     let dns = sockets.add(dns::Socket::new(&[], vec![None, None, None, None]));-    let irq = device.irq;     // Ephemeral (client) ports start at a random spot in 49152..65000, so     // they're not predictable either.     let first_port = 49152 + (crate::virtio::rng::u64() % (65000 - 49152)) as u16;-    IO_BASE.store(device.io_base, Ordering::Relaxed);+    IO_BASE.store(io_base, Ordering::Relaxed);     without_interrupts(|| {         *NET.lock() = Some(Net { iface, device, sockets, dhcp, dns, config: None, closing: Vec::new(), next_port: first_port });     });-    crate::interrupts::enable_pci_irq(irq);     crate::thread::spawn("netd", netd);-    Some(alloc::format!(-        "eth0: virtio-net, MAC {:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}, IRQ {}",-        mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], irq-    ))+    let mut log = alloc::vec![String::from("lo: loopback, 127.0.0.1/8")];+    if let Some(irq) = irq {+        crate::interrupts::enable_pci_irq(irq);+        log.push(alloc::format!(+            "eth0: virtio-net, MAC {:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}, IRQ {}",+            mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], irq+        ));+    }+    log }  /// Interrupt handler part (called by interrupts.rs for PCI IRQ lines).@@ -135,14 +157,24 @@ pub fn kick() { }  /// Let smoltcp process everything that's pending: received frames,-/// queued data to send, timers. Then wake blocked socket calls.+/// queued data to send, timers. Then wake blocked socket calls (and+/// poll/epoll waiters). pub fn poll() {     with_net(|n| {-        n.iface.poll(now(), &mut n.device, &mut n.sockets);+        // Frames smoltcp sent to itself (loopback) are only received on+        // the *next* poll, so go round until lo is empty. Bounded, so+        // two sockets chattering away can't keep us here forever.+        for _ in 0..64 {+            n.iface.poll(now(), &mut n.device, &mut n.sockets);+            if !n.device.lo_pending() {+                break;+            }+        }         n.check_dhcp();         n.reap_closing();     });     CHANGED.wake_all();+    crate::poll::notify(); }  /// The network thread: poll, sleep until a frame arrives or a timer is@@ -174,13 +206,17 @@ impl Net {                 dns: c.dns_servers.iter().map(|d| d.octets()).collect(),             }),         };+        // DHCP's address first: smoltcp uses the first address as the+        // source for destinations outside both networks.         self.iface.update_ip_addrs(|addrs| {             addrs.clear();             if let Some(c) = &new_config {                 let ip = Ipv4Address::from(c.address);                 addrs.push(IpCidr::new(IpAddress::Ipv4(ip), c.prefix_len)).ok();             }+            addrs.push(loopback_cidr()).ok();         });+        self.device.own_ip = new_config.as_ref().map(|c| c.address);         self.iface.routes_mut().remove_default_ipv4_route();         let mut servers = Vec::new();         if let Some(c) = &new_config {@@ -229,7 +265,7 @@ impl Net {  /// Is there a network card at all? pub fn present() -> bool {-    with_net(|_| ()).is_some()+    with_net(|n| n.device.nic.is_some()).unwrap_or(false) }  /// Wait (up to `timeout_us`) until DHCP has given us an address.@@ -306,11 +342,12 @@ pub struct NetInfo {  pub fn info() -> NetInfo {     with_net(|n| {-        let s = n.device.stats;+        let Some(nic) = &n.device.nic else { return NetInfo::default() };+        let s = nic.stats;         let mut info = NetInfo {             present: 1,-            mac: n.device.mac,-            irq: n.device.irq,+            mac: nic.mac,+            irq: nic.irq,             rx_packets: s.rx_packets,             rx_bytes: s.rx_bytes,             tx_packets: s.tx_packets,

@@ -12,7 +12,10 @@ //! Blocking: smoltcp never blocks, it just says "no data yet". So e.g. //! `recv` sleeps on `net::CHANGED` (woken after every `net::poll`) until //! its socket has data, the peer closed, or the timeout (SO_RCVTIMEO) ran-//! out. No busy-waiting: a blocked program costs no CPU.+//! out. No busy-waiting: a blocked program costs no CPU. A *nonblocking*+//! socket (O_NONBLOCK) never sleeps: "would block" becomes EAGAIN, and a+//! TCP connect returns EINPROGRESS while the handshake goes on (poll for+//! POLLOUT to learn when it's done). `File::poll` below says what's ready. //! //! A listening TCP socket is a small *backlog* of smoltcp sockets all //! listening on the port (a smoltcp socket handles exactly one@@ -22,6 +25,8 @@ use super::{Net, poll, with_net}; use crate::errno::Errno; use crate::fs::file::{File, KIND_SOCKET, Stat};+use crate::poll::{POLLERR, POLLHUP, POLLIN, POLLOUT};+use core::sync::atomic::{AtomicBool, Ordering}; use alloc::vec; use alloc::vec::Vec; use smoltcp::iface::SocketHandle;@@ -58,6 +63,8 @@ struct State { pub struct Socket {     pub kind: Kind,     state: Mutex<State>,+    /// O_NONBLOCK: never sleep, say EAGAIN instead.+    nonblock: AtomicBool, }  /// An IPv4 address + port, as plain numbers (the syscall layer converts@@ -74,6 +81,12 @@ fn addr(ep: IpEndpoint) -> Addr {     (ip.octets(), ep.port) } +/// Wait for a usable route to `to`: loopback always has one, anything+/// else needs DHCP to have configured us.+fn route_ready(to: Addr) -> Result<(), Errno> {+    if super::is_loopback(to.0) { Ok(()) } else { super::wait_configured(5_000_000).map(|_| ()) }+}+ fn new_tcp() -> tcp::Socket<'static> {     tcp::Socket::new(tcp::SocketBuffer::new(vec![0; TCP_BUFFER]), tcp::SocketBuffer::new(vec![0; TCP_BUFFER])) }@@ -81,8 +94,8 @@ fn new_tcp() -> tcp::Socket<'static> { impl Socket {     /// socket(): a new, unconnected socket.     pub fn new(kind: Kind) -> Result<Socket, Errno> {-        if !super::present() {-            return Err(Errno::ENETDOWN);+        if with_net(|_| ()).is_none() {+            return Err(Errno::ENETDOWN); // net::init not run         }         let mut state = State { handle: None, listeners: Vec::new(), local_port: 0, remote: None, timeout_us: None };         match kind {@@ -102,7 +115,11 @@ impl Socket {                 });             }         }-        Ok(Socket { kind, state: Mutex::new(state) })+        Ok(Socket::with_state(kind, state))+    }++    fn with_state(kind: Kind, state: State) -> Socket {+        Socket { kind, state: Mutex::new(state), nonblock: AtomicBool::new(false) }     }      /// Lock our state and the network (always in that order, interrupts@@ -121,6 +138,10 @@ impl Socket {         timeout_us: Option<u64>,         mut f: impl FnMut(&mut State, &mut Net) -> Option<Result<R, Errno>>,     ) -> Result<R, Errno> {+        if self.nonblock.load(Ordering::Relaxed) {+            // One look, no sleeping.+            return self.locked(&mut f)?.unwrap_or(Err(Errno::EAGAIN));+        }         let deadline = timeout_us.map_or(u64::MAX, crate::time::deadline_after_us);         // Interruptible: Ctrl+C must be able to stop a program waiting         // for the network.@@ -161,7 +182,7 @@ impl Socket {     /// connect(): TCP does the three-way handshake (SYN, SYN-ACK, ACK) and     /// waits for it; UDP/ICMP just remember the default destination.     pub fn connect(&self, to: Addr) -> Result<(), Errno> {-        super::wait_configured(5_000_000)?;+        route_ready(to)?;         if self.kind != Kind::Tcp {             if self.kind == Kind::Udp && without_interrupts(|| self.state.lock().local_port) == 0 {                 self.bind(0)?;@@ -180,6 +201,9 @@ impl Socket {             Ok(())         })??;         poll(); // send the SYN now+        if self.nonblock.load(Ordering::Relaxed) {+            return Err(Errno::EINPROGRESS); // poll for POLLOUT to see it finish+        }         let result = self.wait(Some(CONNECT_TIMEOUT_US), |st, n| {             match n.sockets.get::<tcp::Socket>(st.handle?).state() {                 tcp::State::SynSent | tcp::State::SynReceived => None,@@ -240,14 +264,16 @@ impl Socket {             let peer = n.sockets.get::<tcp::Socket>(handle).remote_endpoint().map(addr).unwrap_or(([0; 4], 0));             let state =                 State { handle: Some(handle), listeners: Vec::new(), local_port: st.local_port, remote: None, timeout_us: None };-            Some(Ok((Socket { kind: Kind::Tcp, state: Mutex::new(state) }, peer)))+            Some(Ok((Socket::with_state(Kind::Tcp, state), peer)))         })     }      /// send()/sendto()/write(). Blocks while the send buffer is full.     pub fn send(&self, data: &[u8], to: Option<Addr>) -> Result<usize, Errno> {         if self.kind == Kind::Udp && without_interrupts(|| self.state.lock().local_port) == 0 {-            super::wait_configured(5_000_000)?;+            if let Some(to) = to {+                route_ready(to)?;+            }             self.bind(0)?; // first send picks a local port, like Linux         }         let sent = self.wait(None, |st, n| {@@ -379,4 +405,74 @@ impl File for Socket {     fn as_socket(&self) -> Option<&Socket> {         Some(self)     }+    fn set_nonblocking(&self, on: bool) {+        self.nonblock.store(on, Ordering::Relaxed);+    }+    fn nonblocking(&self) -> bool {+        self.nonblock.load(Ordering::Relaxed)+    }+    /// What's ready right now (for poll/select/epoll).+    fn poll(&self) -> u16 {+        let kind = self.kind;+        self.locked(|st, n| {+            let mut ready = 0;+            match kind {+                Kind::Tcp if !st.listeners.is_empty() => {+                    // Listening: "readable" = a connection to accept.+                    let waiting = st.listeners.iter().any(|&h| {+                        !matches!(n.sockets.get::<tcp::Socket>(h).state(), tcp::State::Listen | tcp::State::SynReceived)+                    });+                    if waiting {+                        ready |= POLLIN;+                    }+                }+                Kind::Tcp => match st.handle {+                    // Never connected: nothing to read or write.+                    None => ready |= POLLOUT | POLLHUP,+                    Some(h) => {+                        let sock = n.sockets.get::<tcp::Socket>(h);+                        match sock.state() {+                            tcp::State::SynSent | tcp::State::SynReceived => {} // handshake going on+                            tcp::State::Closed => ready |= POLLIN | POLLERR | POLLHUP, // refused or reset+                            _ => {+                                if sock.can_recv() || !sock.may_recv() {+                                    ready |= POLLIN; // data, or EOF+                                }+                                if sock.can_send() || !sock.may_send() {+                                    ready |= POLLOUT; // room (or EPIPE waiting)+                                }+                                if !sock.may_recv() && !sock.may_send() {+                                    ready |= POLLHUP;+                                }+                            }+                        }+                    }+                },+                Kind::Udp => {+                    if let Some(h) = st.handle {+                        let sock = n.sockets.get::<udp::Socket>(h);+                        if sock.can_recv() {+                            ready |= POLLIN;+                        }+                        if sock.can_send() {+                            ready |= POLLOUT;+                        }+                    }+                }+                Kind::Icmp => {+                    if let Some(h) = st.handle {+                        let sock = n.sockets.get::<icmp::Socket>(h);+                        if sock.can_recv() {+                            ready |= POLLIN;+                        }+                        if sock.can_send() {+                            ready |= POLLOUT;+                        }+                    }+                }+            }+            ready+        })+        .unwrap_or(POLLERR)+    } }

@@ -25,8 +25,6 @@ use crate::pci::PciDevice; use alloc::vec::Vec; use core::ptr::{read_volatile, write_volatile}; use core::sync::atomic::{Ordering, fence};-use smoltcp::phy::{self, DeviceCapabilities, Medium};-use smoltcp::time::Instant; use x86_64::PhysAddr; use x86_64::instructions::port::Port; @@ -216,7 +214,7 @@ impl VirtioNet {      /// Take one received frame (copied out; its buffer goes straight back     /// to the device).-    fn receive_frame(&mut self) -> Option<Vec<u8>> {+    pub fn receive_frame(&mut self) -> Option<Vec<u8>> {         let (i, len) = self.rx.pop_used()?;         let len = len.clamp(HEADER_LEN, BUF_SIZE);         let frame = self.rx.buffer(i)[HEADER_LEN..len].to_vec();@@ -235,7 +233,7 @@ impl VirtioNet {     }      /// Send one frame: `fill` writes its `len` bytes.-    fn send_frame<R>(&mut self, len: usize, fill: impl FnOnce(&mut [u8]) -> R) -> R {+    pub fn send_frame<R>(&mut self, len: usize, fill: impl FnOnce(&mut [u8]) -> R) -> R {         let len = len.min(MTU);         self.reclaim_tx();         // QEMU sends synchronously when notified, so a buffer is always@@ -262,45 +260,5 @@ fn align_up(x: usize, align: usize) -> usize {     x.div_ceil(align) * align } -// --- smoltcp glue --------------------------------------------------------------//-// smoltcp asks a `Device` for *tokens*: an RxToken is "a received frame you-// may look at", a TxToken is "permission to build one frame to send". This-// lets smoltcp work without copying in zero-copy drivers; ours copies the-// received frame out (simple, and frames are small).--pub struct RxToken(Vec<u8>);-pub struct TxToken<'a>(&'a mut VirtioNet);--impl phy::Device for VirtioNet {-    type RxToken<'a> = RxToken;-    type TxToken<'a> = TxToken<'a>;--    fn receive(&mut self, _now: Instant) -> Option<(RxToken, TxToken<'_>)> {-        let frame = self.receive_frame()?;-        Some((RxToken(frame), TxToken(self)))-    }--    fn transmit(&mut self, _now: Instant) -> Option<TxToken<'_>> {-        Some(TxToken(self))-    }--    fn capabilities(&self) -> DeviceCapabilities {-        let mut caps = DeviceCapabilities::default();-        caps.medium = Medium::Ethernet;-        caps.max_transmission_unit = MTU;-        caps-    }-}--impl phy::RxToken for RxToken {-    fn consume<R, F: FnOnce(&[u8]) -> R>(self, f: F) -> R {-        f(&self.0)-    }-}--impl phy::TxToken for TxToken<'_> {-    fn consume<R, F: FnOnce(&mut [u8]) -> R>(self, len: usize, f: F) -> R {-        self.0.send_frame(len, f)-    }-}+// The smoltcp glue (Device, RxToken, TxToken) is in device.rs, which+// also adds the loopback queue.

@@ -0,0 +1,420 @@+//! Waiting for several files at once: poll, select and epoll.+//!+//! A server talking to many clients can't call `read` on one of them and+//! block -- the others would wait. Instead it asks the kernel "tell me+//! which of these fds are ready" and then only reads/writes those, which+//! never blocks. That loop ("wait for events, handle each, repeat") is an+//! **event loop**; every GUI toolkit, web server and terminal emulator is+//! built around one. Three interfaces for the same idea:+//!+//!   * `poll(fds, n, timeout)`: an array of {fd, events wanted, events+//!     returned}. Simple; the kernel looks at every fd on every call.+//!   * `select(n, readfds, writefds, exceptfds, timeout)`: the 1983 BSD+//!     original, with bitmaps of fds instead of an array.+//!   * `epoll`: register the fds *once* in an epoll instance (itself an+//!     fd) with epoll_ctl, then epoll_wait returns just the ready ones.+//!     Linux's answer to servers with 10 000 connections.+//!+//! **How our kernel does it.** Every file can say what's ready right now+//! (`File::poll`). Anything that might make a file ready -- data into a+//! pipe, a key typed, a packet for a socket -- calls `notify()`, which+//! bumps a global counter and wakes every poller. A poller checks all its+//! fds; if none is ready it sleeps until the counter changes (or its+//! timeout), then checks again. Simple, and never misses an event; a big+//! kernel instead keeps per-file wait lists so one packet only wakes the+//! pollers of that socket.+//!+//! Simplification: epoll is *level-triggered* only (EPOLLET is accepted+//! but means the same thing: "ready" is reported as long as it lasts).+//! Regular files are always ready (Linux too), and epoll refuses them+//! with EPERM like Linux does.++use crate::errno::Errno;+use crate::fs::file::{File, KIND_DIR, KIND_FILE, Stat};+use crate::syscall::{read_timespec_us, user_slice, user_slice_mut};+use crate::thread::WaitQueue;+use alloc::sync::{Arc, Weak};+use alloc::vec::Vec;+use core::sync::atomic::{AtomicU64, Ordering};+use spin::Mutex;++// poll(2) event bits (Linux's values). epoll uses the same numbers.+pub const POLLIN: u16 = 0x1; // data to read (or EOF / a connection to accept)+pub const POLLPRI: u16 = 0x2; // urgent data+pub const POLLOUT: u16 = 0x4; // room to write+pub const POLLERR: u16 = 0x8; // error (e.g. pipe without readers)+pub const POLLHUP: u16 = 0x10; // other side hung up+pub const POLLNVAL: u16 = 0x20; // not an open fd+pub const POLLRDNORM: u16 = 0x40;+pub const POLLWRNORM: u16 = 0x100;+pub const POLLRDHUP: u16 = 0x2000;++/// Bumped on every possible readiness change (see the top).+static GENERATION: AtomicU64 = AtomicU64::new(0);+/// Pollers sleep here.+static POLLERS: WaitQueue = WaitQueue::new();++/// Something may have become ready: wake all pollers to look again.+pub fn notify() {+    GENERATION.fetch_add(1, Ordering::SeqCst);+    POLLERS.wake_all();+}++/// Add the "normal data" aliases a program may ask for instead.+fn with_aliases(events: u16) -> u16 {+    let mut e = events;+    if events & POLLIN != 0 {+        e |= POLLRDNORM;+    }+    if events & POLLOUT != 0 {+        e |= POLLWRNORM;+    }+    e+}++/// The poll loop: call `check` (with interrupts on) until it returns+/// Some, the deadline (PIT tick; Some(0) = don't wait) passes -> Ok(None),+/// or a signal arrives -> Err.+pub fn wait_ready<T>(deadline: Option<u64>, mut check: impl FnMut() -> Option<T>) -> Result<Option<T>, Errno> {+    loop {+        let generation = GENERATION.load(Ordering::SeqCst);+        if let Some(result) = check() {+            return Ok(Some(result));+        }+        if deadline.is_some_and(|d| crate::interrupts::ticks() >= d) {+            return Ok(None);+        }+        let changed = || (GENERATION.load(Ordering::SeqCst) != generation).then_some(());+        match deadline {+            None => POLLERS.wait_until_intr(changed)?,+            Some(d) => {+                POLLERS.wait_until_deadline_intr(d, changed)?;+            }+        }+    }+}++/// A timeout in milliseconds (negative = forever) as a deadline.+fn ms_deadline(ms: i64) -> Option<u64> {+    match ms {+        ms if ms < 0 => None,+        0 => Some(0),+        ms => Some(crate::time::deadline_after_us(ms as u64 * 1000)),+    }+}++/// A `struct timespec *` timeout (NULL = forever) as a deadline.+fn timespec_deadline(ptr: u64) -> Result<Option<u64>, Errno> {+    if ptr == 0 {+        return Ok(None);+    }+    Ok(Some(match read_timespec_us(ptr)? {+        0 => 0,+        us => crate::time::deadline_after_us(us),+    }))+}++/// ppoll / pselect6 / epoll_pwait: run with signal mask `mask_ptr` (if+/// given) while waiting. If a signal interrupts, the old mask comes back+/// only after its handler ran (signal.rs), like Linux -- otherwise the+/// signal we waited for could end up blocked again before delivery.+fn with_sigmask<T>(mask_ptr: u64, f: impl FnOnce() -> Result<T, Errno>) -> Result<T, Errno> {+    if mask_ptr == 0 {+        return f();+    }+    let mask = u64::from_le_bytes(user_slice(mask_ptr, 8)?.try_into().unwrap());+    crate::signal::set_temporary_mask(mask);+    let result = f();+    if !matches!(result, Err(Errno::EINTR | Errno::ERESTARTNOHAND)) {+        crate::signal::restore_temporary_mask();+    }+    result+}++// ---------------------------------------------------------------------------+// poll / ppoll+// ---------------------------------------------------------------------------++/// poll(fds, nfds, timeout_ms) (7) and ppoll(fds, nfds, *timespec, *sigmask,+/// 8) (271). `struct pollfd {int fd; short events; short revents;}`.+pub fn sys_poll(fds_ptr: u64, nfds: u64, deadline: Option<u64>) -> Result<u64, Errno> {+    if nfds > 1024 {+        return Err(Errno::EINVAL);+    }+    let p = crate::thread::current_process().ok_or(Errno::EINVAL)?;+    let raw = user_slice(fds_ptr, nfds * 8)?;+    // (fd, events wanted, the file: None = not open)+    let entries: Vec<(i32, u16, Option<Arc<dyn File>>)> = raw+        .chunks(8)+        .map(|c| {+            let fd = i32::from_le_bytes(c[0..4].try_into().unwrap());+            let events = u16::from_le_bytes(c[4..6].try_into().unwrap());+            (fd, events, if fd < 0 { None } else { p.file(fd as u64).ok() })+        })+        .collect();+    drop(p);+    let mut revents = alloc::vec![0u16; entries.len()];+    let ready = wait_ready(deadline, || {+        let mut count = 0;+        for (i, (fd, events, file)) in entries.iter().enumerate() {+            revents[i] = match file {+                _ if *fd < 0 => 0, // ignored entry+                None => POLLNVAL,+                Some(f) => f.poll() & (with_aliases(*events) | POLLERR | POLLHUP),+            };+            count += (revents[i] != 0) as u64;+        }+        (count > 0).then_some(count)+    })+    .map_err(|_| Errno::ERESTARTNOHAND)?+    .unwrap_or(0);+    let out = user_slice_mut(fds_ptr, nfds * 8)?;+    for (i, r) in revents.iter().enumerate() {+        out[8 * i + 6..8 * i + 8].copy_from_slice(&r.to_le_bytes());+    }+    Ok(ready)+}++pub fn sys_poll_ms(fds_ptr: u64, nfds: u64, timeout_ms: i64) -> Result<u64, Errno> {+    sys_poll(fds_ptr, nfds, ms_deadline(timeout_ms))+}++pub fn sys_ppoll(fds_ptr: u64, nfds: u64, ts_ptr: u64, mask_ptr: u64) -> Result<u64, Errno> {+    let deadline = timespec_deadline(ts_ptr)?;+    with_sigmask(mask_ptr, || sys_poll(fds_ptr, nfds, deadline))+}++// ---------------------------------------------------------------------------+// select / pselect6+// ---------------------------------------------------------------------------++/// select(nfds, *readfds, *writefds, *exceptfds, *timeval) (23). Each set+/// is a bitmap: bit n = fd n. On return the sets hold only the ready fds.+pub fn sys_select(nfds: u64, sets: [u64; 3], deadline: Option<u64>) -> Result<u64, Errno> {+    if nfds > 1024 {+        return Err(Errno::EINVAL);+    }+    let words = nfds.div_ceil(64) as usize;+    // Read the three bitmaps (a null pointer = empty set).+    let mut wanted = [[0u64; 16]; 3];+    for (k, &ptr) in sets.iter().enumerate() {+        if ptr != 0 {+            let bytes = user_slice(ptr, words as u64 * 8)?;+            for w in 0..words {+                wanted[k][w] = u64::from_le_bytes(bytes[8 * w..8 * w + 8].try_into().unwrap());+            }+        }+    }+    let p = crate::thread::current_process().ok_or(Errno::EINVAL)?;+    let mut files: Vec<(usize, Arc<dyn File>)> = Vec::new();+    for fd in 0..nfds as usize {+        if (0..3).any(|k| wanted[k][fd / 64] & (1 << (fd % 64)) != 0) {+            files.push((fd, p.file(fd as u64)?)); // EBADF for a closed fd+        }+    }+    drop(p);+    let mut result = [[0u64; 16]; 3];+    let ready = wait_ready(deadline, || {+        result = [[0u64; 16]; 3];+        let mut count = 0;+        for (fd, file) in &files {+            let events = file.poll();+            let bit = 1u64 << (fd % 64);+            // readable: data, EOF/hangup or an error (the read won't block)+            let tests = [POLLIN | POLLHUP | POLLERR, POLLOUT | POLLERR, POLLPRI];+            for k in 0..3 {+                if wanted[k][fd / 64] & bit != 0 && events & tests[k] != 0 {+                    result[k][fd / 64] |= bit;+                    count += 1;+                }+            }+        }+        (count > 0).then_some(count)+    })+    .map_err(|_| Errno::ERESTARTNOHAND)?+    .unwrap_or(0);+    for (k, &ptr) in sets.iter().enumerate() {+        if ptr != 0 {+            let out = user_slice_mut(ptr, words as u64 * 8)?;+            for w in 0..words {+                out[8 * w..8 * w + 8].copy_from_slice(&result[k][w].to_le_bytes());+            }+        }+    }+    Ok(ready)+}++/// select's `struct timeval {seconds, microseconds}` (NULL = forever).+pub fn sys_select_tv(nfds: u64, r: u64, w: u64, e: u64, tv_ptr: u64) -> Result<u64, Errno> {+    let deadline = if tv_ptr == 0 {+        None+    } else {+        let tv = user_slice(tv_ptr, 16)?;+        let secs = i64::from_le_bytes(tv[..8].try_into().unwrap());+        let usecs = i64::from_le_bytes(tv[8..].try_into().unwrap());+        if secs < 0 || !(0..1_000_000).contains(&usecs) {+            return Err(Errno::EINVAL);+        }+        let us = secs as u64 * 1_000_000 + usecs as u64;+        Some(if us == 0 { 0 } else { crate::time::deadline_after_us(us) })+    };+    sys_select(nfds, [r, w, e], deadline)+}++/// pselect6(nfds, r, w, e, *timespec, *{sigmask ptr, size}) (270).+pub fn sys_pselect6(nfds: u64, r: u64, w: u64, e: u64, ts_ptr: u64, mask_arg: u64) -> Result<u64, Errno> {+    let deadline = timespec_deadline(ts_ptr)?;+    let mask_ptr = if mask_arg == 0 { 0 } else { u64::from_le_bytes(user_slice(mask_arg, 8)?.try_into().unwrap()) };+    with_sigmask(mask_ptr, || sys_select(nfds, [r, w, e], deadline))+}++// ---------------------------------------------------------------------------+// epoll+// ---------------------------------------------------------------------------++pub const EPOLL_CTL_ADD: u64 = 1;+pub const EPOLL_CTL_DEL: u64 = 2;+pub const EPOLL_CTL_MOD: u64 = 3;+pub const EPOLLET: u32 = 1 << 31;+pub const EPOLLONESHOT: u32 = 1 << 30;+pub const EPOLL_CLOEXEC: u64 = 0o2000000;++/// One watched fd.+struct Interest {+    fd: u64,+    /// Weak: closing the file (its last fd) makes the entry go away, like+    /// Linux; the epoll instance doesn't keep files open.+    file: Weak<dyn File>,+    events: u32,+    data: u64,+    /// EPOLLONESHOT: reported once, then off until EPOLL_CTL_MOD.+    disabled: bool,+}++/// An epoll instance: the "interest list".+pub struct Epoll {+    interest: Mutex<Vec<Interest>>,+}++impl Epoll {+    /// The ready entries as (events, data), at most `max`.+    fn ready(&self, max: usize) -> Vec<(u32, u64)> {+        let mut out = Vec::new();+        let mut list = self.interest.lock();+        list.retain(|i| i.file.strong_count() > 0); // closed files drop out+        for i in list.iter_mut().filter(|i| !i.disabled) {+            let Some(file) = i.file.upgrade() else { continue };+            let events = file.poll() as u32 & (i.events | (POLLERR | POLLHUP) as u32);+            if events != 0 {+                out.push((events, i.data));+                if i.events & EPOLLONESHOT != 0 {+                    i.disabled = true;+                }+                if out.len() == max {+                    break;+                }+            }+        }+        out+    }+}++impl File for Epoll {+    fn read(&self, _buf: &mut [u8]) -> Result<usize, Errno> {+        Err(Errno::EINVAL)+    }+    fn write(&self, _buf: &[u8]) -> Result<usize, Errno> {+        Err(Errno::EINVAL)+    }+    fn stat(&self) -> Result<Stat, Errno> {+        Ok(Stat { kind: crate::fs::file::KIND_DEVICE, size: 0, mode: 0o600 })+    }+    /// An epoll fd is readable when one of its files is ready (so epoll+    /// instances can be polled themselves). Doesn't consume ONESHOTs.+    fn poll(&self) -> u16 {+        let list = self.interest.lock();+        let ready = list.iter().any(|i| {+            !i.disabled && i.file.upgrade().is_some_and(|f| f.poll() as u32 & (i.events | (POLLERR | POLLHUP) as u32) != 0)+        });+        if ready { POLLIN } else { 0 }+    }+    fn as_epoll(&self) -> Option<&Epoll> {+        Some(self)+    }+}++/// epoll_create1(flags) (291); epoll_create(size) (213) is the same+/// without flags.+pub fn sys_epoll_create1(flags: u64) -> Result<u64, Errno> {+    if flags & !EPOLL_CLOEXEC != 0 {+        return Err(Errno::EINVAL);+    }+    let p = crate::thread::current_process().ok_or(Errno::EINVAL)?;+    p.add_file_with(Arc::new(Epoll { interest: Mutex::new(Vec::new()) }), 0, flags & EPOLL_CLOEXEC != 0)+}++/// `struct epoll_event {u32 events; u64 data;}` -- packed: 12 bytes.+fn read_event(ptr: u64) -> Result<(u32, u64), Errno> {+    let b = user_slice(ptr, 12)?;+    Ok((u32::from_le_bytes(b[0..4].try_into().unwrap()), u64::from_le_bytes(b[4..12].try_into().unwrap())))+}++/// epoll_ctl(epfd, op, fd, *event) (233).+pub fn sys_epoll_ctl(epfd: u64, op: u64, fd: u64, event_ptr: u64) -> Result<u64, Errno> {+    let p = crate::thread::current_process().ok_or(Errno::EINVAL)?;+    let ep_file = p.file(epfd)?;+    let file = p.file(fd)?;+    let ep = ep_file.as_epoll().ok_or(Errno::EINVAL)?;+    if Arc::ptr_eq(&ep_file, &file) {+        return Err(Errno::EINVAL); // can't watch itself+    }+    if matches!(file.stat()?.kind, KIND_FILE | KIND_DIR) {+        return Err(Errno::EPERM); // always ready: pointless (Linux says EPERM too)+    }+    let mut list = ep.interest.lock();+    list.retain(|i| i.file.strong_count() > 0);+    let pos = list.iter().position(|i| i.fd == fd && i.file.upgrade().is_some_and(|f| Arc::ptr_eq(&f, &file)));+    match op {+        EPOLL_CTL_ADD | EPOLL_CTL_MOD => {+            let (events, data) = read_event(event_ptr)?;+            match (op, pos) {+                (EPOLL_CTL_ADD, Some(_)) => return Err(Errno::EEXIST),+                (EPOLL_CTL_ADD, None) => list.push(Interest { fd, file: Arc::downgrade(&file), events, data, disabled: false }),+                (_, None) => return Err(Errno::ENOENT),+                (_, Some(i)) => list[i] = Interest { fd, file: Arc::downgrade(&file), events, data, disabled: false },+            }+        }+        EPOLL_CTL_DEL => {+            list.remove(pos.ok_or(Errno::ENOENT)?);+        }+        _ => return Err(Errno::EINVAL),+    }+    drop(list);+    notify(); // a waiter may care about the new entry+    Ok(0)+}++/// epoll_wait(epfd, *events, max, timeout_ms) (232) and epoll_pwait(...,+/// *sigmask, 8) (281).+pub fn sys_epoll_pwait(epfd: u64, events_ptr: u64, max: i64, timeout_ms: i64, mask_ptr: u64) -> Result<u64, Errno> {+    if max <= 0 || max > 1024 {+        return Err(Errno::EINVAL);+    }+    let file = crate::thread::current_process().ok_or(Errno::EINVAL)?.file(epfd)?;+    let ep = file.as_epoll().ok_or(Errno::EINVAL)?;+    user_slice_mut(events_ptr, max as u64 * 12)?; // check first+    let ready = with_sigmask(mask_ptr, || {+        wait_ready(ms_deadline(timeout_ms), || {+            let r = ep.ready(max as usize);+            (!r.is_empty()).then_some(r)+        })+        .map_err(|_| Errno::EINTR) // epoll_wait is never restarted (Linux)+    })?+    .unwrap_or_default();+    let out = user_slice_mut(events_ptr, ready.len() as u64 * 12)?;+    for (i, (events, data)) in ready.iter().enumerate() {+        out[12 * i..12 * i + 4].copy_from_slice(&events.to_le_bytes());+        out[12 * i + 4..12 * i + 12].copy_from_slice(&data.to_le_bytes());+    }+    Ok(ready.len() as u64)+}

@@ -36,20 +36,29 @@ //! sends SIGINT to that whole group -- every program of the pipeline, but //! not the shell (tty.rs). //!+//! **Threads** (stage 15). A process can run several threads: each is a+//! `Task` with its own registers, kernel stack, thread id, TLS pointer and+//! signal mask, but they all share the process's memory, files and signal+//! handlers. `clone(CLONE_VM | CLONE_THREAD | ...)` makes one (that's what+//! pthread_create does). The first thread's id equals the pid. `exit` ends+//! one thread; `exit_group` (what C's exit() calls) ends them all. The+//! process itself ends when its last thread does.+//! //! **Memory lifetime.** A process's frames and page tables are freed when //! the last `Arc<Process>` goes away (dropping its `Mm`). That's always-//! after its thread is gone: the thread holds an Arc until the scheduler-//! frees it, which happens only once the CPU has switched to another-//! thread and address space. So no CPU can still be using the page tables.+//! after its threads are gone: each holds an Arc (through its Task) until+//! the scheduler frees it, which happens only once the CPU has switched to+//! another thread and address space. So no CPU can still be using the page+//! tables.  use crate::elf::{self, ElfError}; use crate::errno::Errno; use crate::fs::{self, file::File}; use crate::mm::Mm;-use crate::signal::{self, SigInfo, SignalState};+use crate::signal::{self, SigInfo, SignalState, ThreadSig}; use crate::thread::{self, WaitQueue}; use crate::trap::TrapFrame;-use crate::{serial_println, tty::Console};+use crate::serial_println; use alloc::{collections::BTreeMap, string::String, sync::Arc, vec, vec::Vec}; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use spin::Mutex;@@ -66,6 +75,8 @@ pub const MAX_ARGS_SIZE: usize = 64 * 1024; /// Max processes at once: fork() fails with EAGAIN beyond that, which is /// what stops a fork bomb from eating all memory. pub const MAX_PROCESSES: usize = 64;+/// Max threads in one process (clone fails with EAGAIN beyond that).+pub const MAX_THREADS: usize = 64; /// Environment of processes the kernel starts. pub const DEFAULT_ENV: [&str; 3] = ["PATH=/bin:/mnt/disk0/bin", "HOME=/", "TERM=linux"]; @@ -152,12 +163,56 @@ impl FdTable {     } } +/// One thread of a process (what Linux calls a "task"). The scheduler's+/// kernel thread that runs it holds an `Arc<Task>` (thread.rs).+pub struct Task {+    /// Thread id (gettid). The first thread's is the pid.+    tid: AtomicU64,+    pub process: Arc<Process>,+    /// The scheduler thread running it (set when it starts).+    kthread: AtomicUsize,+    /// Its FS segment base: the thread-local-storage pointer (arch_prctl,+    /// clone's CLONE_SETTLS). Loaded into the CPU on every switch.+    pub fs_base: AtomicU64,+    /// set_tid_address / CLONE_CHILD_CLEARTID: when the thread exits, write+    /// 0 there and futex-wake it (that's how pthread_join waits).+    pub clear_child_tid: AtomicU64,+    /// Registers it starts user mode with (taken when it starts).+    start: Mutex<Option<TrapFrame>>,+    /// Told to exit (another thread called exit_group or execve).+    pub killed: AtomicBool,+    /// Its own signal mask and pending signals. Locked with interrupts off,+    /// after the process's `signals` if both are needed.+    pub sig: Mutex<ThreadSig>,+}++impl Task {+    pub fn tid(&self) -> u64 {+        self.tid.load(Ordering::Relaxed)+    }++    /// Wake it if it's blocked (it then checks for signals / being killed).+    pub fn wake(&self) {+        let kthread = self.kthread.load(Ordering::Acquire);+        if kthread != usize::MAX {+            thread::wake(kthread);+        }+    }+}+ pub struct Process {     pub pid: u64,     /// Parent pid; 0 = started by the kernel itself, ORPHAN = parent exited.     pub parent: AtomicU64,     /// Process group (job control; see the top of this file).     pgid: AtomicU64,+    /// Session (setsid; a login session = the processes of one terminal).+    sid: AtomicU64,+    /// Controlling terminal (tty.rs): what /dev/tty opens, whose Ctrl+C+    /// reaches this process's group.+    pub ctty: Mutex<Option<Arc<crate::tty::Tty>>>,+    /// Path of the running program (/proc/self/exe).+    pub exe: Mutex<String>,     /// Program name (changes with execve).     name: Mutex<String>,     /// Print every syscall this process makes (`run --trace`; survives exec).@@ -166,12 +221,18 @@ pub struct Process {     pub cwd: Mutex<String>,     /// The address space.     pub mm: Mutex<Mm>,-    /// Its thread's id (set right after the thread is created).-    tid: AtomicUsize,-    /// CPUs its thread has run on (bit per CPU; kept up by the scheduler).+    /// Its live threads. Locked with interrupts off.+    tasks: Mutex<Vec<Arc<Task>>>,+    /// Scheduler threads that ran its threads and may not be freed yet+    /// (reaping waits for them). Locked with interrupts off.+    kthreads: Mutex<Vec<thread::Tid>>,+    /// exit_group was called (or a fatal signal arrived): every thread+    /// exits, and the process ends with this status. Interrupts off.+    exiting: Mutex<Option<ExitStatus>>,+    /// A thread of it exited (execve waits for the others to go).+    thread_exit: WaitQueue,+    /// CPUs its threads have run on (bit per CPU; kept up by the scheduler).     pub ran_on: AtomicU32,-    /// Registers its thread starts user mode with (taken by the thread).-    start: Mutex<Option<TrapFrame>>,     /// The environment it was started with (`spawn` passes it on).     env: Mutex<Vec<String>>,     fds: Mutex<FdTable>,@@ -180,7 +241,8 @@ pub struct Process {     exited: WaitQueue,     /// Its own wait4 sleeps here; children wake it when they change state.     pub child_wait: WaitQueue,-    /// Signal handlers, mask, pending signals. Locked with interrupts off.+    /// Signal handlers, process-wide pending signals, stop state. Locked+    /// with interrupts off. (Masks are per thread: `Task::sig`.)     pub signals: Mutex<SignalState>,     /// vfork: the parent waits until this child execs or exits.     vfork_done: AtomicBool,@@ -203,13 +265,17 @@ pub struct SpawnArgs {     pub parent: u64,     /// Process group: 0 = a new group of its own.     pub pgid: u64,+    /// Session: 0 = a new session of its own.+    pub sid: u64,+    /// Controlling terminal.+    pub ctty: Option<Arc<crate::tty::Tty>>,     pub trace: bool, }  impl SpawnArgs {     /// Defaults for a process started by the kernel: console stdio, cwd "/".     pub fn kernel(name: &str) -> SpawnArgs {-        let console: Arc<dyn File> = Arc::new(Console);+        let console: Arc<dyn File> = crate::tty::console_file();         SpawnArgs {             args: vec![String::from(name)],             env: DEFAULT_ENV.iter().map(|s| String::from(*s)).collect(),@@ -217,6 +283,8 @@ impl SpawnArgs {             cwd: String::from("/"),             parent: 0,             pgid: 0,+            sid: 0,+            ctty: Some(crate::tty::console()),             trace: false,         }     }@@ -258,13 +326,18 @@ pub fn spawn_image(path: &str, elf_image: &[u8], spawn: SpawnArgs) -> Result<Arc         pid,         parent: AtomicU64::new(spawn.parent),         pgid: AtomicU64::new(if spawn.pgid == 0 { pid } else { spawn.pgid }),+        sid: AtomicU64::new(if spawn.sid == 0 { pid } else { spawn.sid }),+        ctty: Mutex::new(spawn.ctty),+        exe: Mutex::new(String::from(path)),         name: Mutex::new(String::from(name)),         trace: AtomicBool::new(spawn.trace),         cwd: Mutex::new(spawn.cwd),         mm: Mutex::new(mm),-        tid: AtomicUsize::new(usize::MAX),+        tasks: Mutex::new(Vec::new()),+        kthreads: Mutex::new(Vec::new()),+        exiting: Mutex::new(None),+        thread_exit: WaitQueue::new(),         ran_on: AtomicU32::new(0),-        start: Mutex::new(Some(start)),         env: Mutex::new(spawn.env),         fds: Mutex::new(FdTable::new([stdin, stdout, stderr])),         status: Mutex::new(None),@@ -274,27 +347,49 @@ pub fn spawn_image(path: &str, elf_image: &[u8], spawn: SpawnArgs) -> Result<Arc         vfork_done: AtomicBool::new(false),     };     serial_println!("[process] started pid {} ({}), entry {:#x}", pid, name, start.rip);-    Ok(start_thread(process))+    Ok(start_process(process, start, 0, 0)) } -/// Put the process in the table and give it a thread.-fn start_thread(process: Process) -> Arc<Process> {+/// Put the process in the table and start its first thread (tid = pid).+fn start_process(process: Process, start: TrapFrame, fs_base: u64, blocked: u64) -> Arc<Process> {     let process = Arc::new(process);-    let l4 = process.mm.lock().l4;     without_interrupts(|| PROCESSES.lock().insert(process.pid, process.clone()));-    let name = process.name();-    let tid = thread::spawn_with(&name, user_thread_entry, l4, Some(process.clone()));-    process.tid.store(tid, Ordering::Release);+    start_task(&process, process.pid, start, fs_base, blocked);     process } -/// First code of every process thread (still ring 0, but already running+/// Start a thread `tid` of `process` that enters user mode with `start`.+fn start_task(process: &Arc<Process>, tid: u64, start: TrapFrame, fs_base: u64, blocked: u64) -> Arc<Task> {+    let task = Arc::new(Task {+        tid: AtomicU64::new(tid),+        process: process.clone(),+        kthread: AtomicUsize::new(usize::MAX),+        fs_base: AtomicU64::new(fs_base),+        clear_child_tid: AtomicU64::new(0),+        start: Mutex::new(Some(start)),+        killed: AtomicBool::new(false),+        sig: Mutex::new(ThreadSig::new(blocked)),+    });+    without_interrupts(|| process.tasks.lock().push(task.clone()));+    let l4 = process.mm.lock().l4;+    thread::spawn_with(&process.name(), user_thread_entry, l4, Some(task.clone()));+    task+}++/// First code of every user thread (still ring 0, but already running /// with the process's CR3 and kernel stack): enter user mode with the /// prepared registers. fn user_thread_entry() {-    let process = thread::current_process().expect("process thread without process");-    let frame = process.start.lock().take().expect("no start registers");-    drop(process); // return_to_user never returns: don't leak the Arc+    let task = thread::current_task().expect("user thread without task");+    let me = thread::current_id();+    task.kthread.store(me, Ordering::Release);+    without_interrupts(|| task.process.kthreads.lock().push(me)); // reap() joins it+    let frame = task.start.lock().take().expect("no start registers");+    let doomed = task.killed.load(Ordering::Acquire) || task.process.exiting().is_some();+    drop(task); // return_to_user never returns: don't leak the Arc+    if doomed {+        exit_thread(ExitStatus::Exited(0)); // the group is going away already+    }     crate::trap::return_to_user(frame) } @@ -482,13 +577,31 @@ impl Process {         self.env.lock().clone()     } -    /// Wake its thread if it's blocked (it then checks for signals).+    pub fn sid(&self) -> u64 {+        self.sid.load(Ordering::Relaxed)+    }++    /// Wake its threads if they're blocked (they then check for signals).     pub fn wake(&self) {-        let tid = self.tid.load(Ordering::Acquire);-        if tid != usize::MAX {-            thread::wake(tid);+        for task in self.tasks() {+            task.wake();         }     }++    /// Its live threads.+    pub fn tasks(&self) -> Vec<Arc<Task>> {+        without_interrupts(|| self.tasks.lock().clone())+    }++    /// The thread with id `tid`, if it's one of ours and alive.+    pub fn task(&self, tid: u64) -> Option<Arc<Task>> {+        without_interrupts(|| self.tasks.lock().iter().find(|t| t.tid() == tid).cloned())+    }++    /// Set once exit_group has been called (all threads are on their way out).+    pub fn exiting(&self) -> Option<ExitStatus> {+        without_interrupts(|| *self.exiting.lock())+    } }  /// Look up a live (or not yet reaped) process.@@ -513,76 +626,123 @@ pub fn group_members(pgid: u64) -> Vec<Arc<Process>> {     }) } -/// End the current process (exit syscall, or killed by a signal).+/// End the whole current process: every thread of it (exit_group, a+/// fatal signal). The first status given wins.+pub fn exit_group(status: ExitStatus) -> ! {+    if let Some(p) = thread::current_process() {+        without_interrupts(|| {+            p.exiting.lock().get_or_insert(status);+        });+        // The other threads notice at their next return to user mode, or+        // right away if they're sleeping (signal::interrupted).+        p.wake();+        signal::wake_stopped();+    }+    exit_thread(status)+}++/// The old name: end the current process. pub fn exit_current(status: ExitStatus) -> ! {-    if let Some(process) = thread::current_process() {-        // Close all files (a pipe's reader sees EOF once its writers are gone).-        let files = core::mem::take(&mut process.fds.lock().files);-        drop(files);-        crate::tty::process_exited(process.pid);-        let name = process.name();-        match status {-            // Serial only: on screen the shell reports exit codes itself.-            ExitStatus::Exited(code) => {-                serial_println!("[process] pid {} ({}) exited with code {}", process.pid, name, code)-            }-            // A crash (SIGSEGV...) is worth a line on screen; a signal-            // somebody sent on purpose (Ctrl+C, kill) isn't.-            ExitStatus::Killed(sig, why) if why != signal::name(sig) => {-                crate::kprintln!("[process] pid {} ({}) killed: {} ({})", process.pid, name, why, signal::name(sig))-            }-            ExitStatus::Killed(sig, _) => {-                serial_println!("[process] pid {} ({}) killed by {}", process.pid, name, signal::name(sig))-            }+    exit_group(status)+}++/// End the calling thread (the exit syscall). If it's the last one, the+/// process ends too -- with `status`, unless exit_group already set one.+pub fn exit_thread(status: ExitStatus) -> ! {+    if let Some(task) = thread::current_task() {+        let p = task.process.clone();+        // pthread_join sleeps on this word with a futex until it's 0.+        let ctid = task.clear_child_tid.swap(0, Ordering::Relaxed);+        if ctid != 0 && ctid % 4 == 0 && crate::mm::user_access_ok(ctid, 4, true) {+            unsafe { core::ptr::write_volatile(ctid as *mut u32, 0) };+            crate::futex::wake(&p, ctid, 1);+        }+        let last = without_interrupts(|| {+            let mut tasks = p.tasks.lock();+            tasks.retain(|t| !Arc::ptr_eq(t, &task));+            tasks.is_empty()+        });+        if !last {+            serial_println!("[process] pid {} thread {} exited", p.pid, task.tid());+        }+        drop(task);+        p.thread_exit.wake_all();+        if last {+            let status = p.exiting().unwrap_or(status);+            process_exit(p, status);         }-        // No preemption from here on: we're about to disappear anyway.-        x86_64::instructions::interrupts::disable();-        let removed = {-            // The table lock makes "set my status" and "look at my parent"-            // one step, so a parent exiting at the same time on another CPU-            // either sees our status (and reaps us) or orphans us first (and-            // we reap ourselves). Never both, never neither.-            let mut table = PROCESSES.lock();-            *process.status.lock() = Some(status);-            let mut removed = Vec::new();-            let children: Vec<Arc<Process>> = table.values().filter(|c| c.parent() == process.pid).cloned().collect();-            for child in children {-                if child.status().is_some() {-                    removed.extend(table.remove(&child.pid)); // exited: reap it-                } else {-                    child.parent.store(ORPHAN, Ordering::Relaxed); // still running-                }-            }-            // Nobody will wait for us if we're an orphan, or if our parent-            // said so by ignoring SIGCHLD explicitly (POSIX): reap ourselves.-            let parent_ignores = table.get(&process.parent()).is_some_and(|p| {-                let action = p.signals.lock().action(signal::SIGCHLD);-                action.handler == signal::SIG_IGN || action.flags & signal::SA_NOCLDWAIT != 0-            });-            if process.parent() == ORPHAN || parent_ignores {-                removed.extend(table.remove(&process.pid));-            }-            removed-        };-        drop(removed); // (frees their memory if they're completely gone)-        process.exited.wake_all(); // the kernel may be blocked in wait()-        process.vfork_done.store(true, Ordering::Release);-        VFORK_WAIT.wake_all();-        let code = match status {-            ExitStatus::Exited(_) => signal::CLD_EXITED,-            ExitStatus::Killed(sig, _) if signal::default_action(sig) == signal::DefaultAction::Core => signal::CLD_DUMPED,-            ExitStatus::Killed(..) => signal::CLD_KILLED,-        };-        let si_status = match status {-            ExitStatus::Exited(c) => c as i32,-            ExitStatus::Killed(sig, _) => sig as i32,-        };-        signal::notify_parent(&process, code, si_status);-        drop(process);     }     thread::exit_current() } +/// The last thread of `process` is exiting: close everything, become a+/// zombie, tell the parent.+fn process_exit(process: Arc<Process>, status: ExitStatus) {+    // Close all files (a pipe's reader sees EOF once its writers are gone).+    let files = core::mem::take(&mut process.fds.lock().files);+    drop(files);+    crate::tty::process_exited(&process); // (raw mode, controlling terminal)+    let name = process.name();+    match status {+        // Serial only: on screen the shell reports exit codes itself.+        ExitStatus::Exited(code) => {+            serial_println!("[process] pid {} ({}) exited with code {}", process.pid, name, code)+        }+        // A crash (SIGSEGV...) is worth a line on screen; a signal+        // somebody sent on purpose (Ctrl+C, kill) isn't.+        ExitStatus::Killed(sig, why) if why != signal::name(sig) => {+            crate::kprintln!("[process] pid {} ({}) killed: {} ({})", process.pid, name, why, signal::name(sig))+        }+        ExitStatus::Killed(sig, _) => {+            serial_println!("[process] pid {} ({}) killed by {}", process.pid, name, signal::name(sig))+        }+    }+    // No preemption from here on: we're about to disappear anyway.+    x86_64::instructions::interrupts::disable();+    let removed = {+        // The table lock makes "set my status" and "look at my parent"+        // one step, so a parent exiting at the same time on another CPU+        // either sees our status (and reaps us) or orphans us first (and+        // we reap ourselves). Never both, never neither.+        let mut table = PROCESSES.lock();+        *process.status.lock() = Some(status);+        let mut removed = Vec::new();+        let children: Vec<Arc<Process>> = table.values().filter(|c| c.parent() == process.pid).cloned().collect();+        for child in children {+            if child.status().is_some() {+                removed.extend(table.remove(&child.pid)); // exited: reap it+            } else {+                child.parent.store(ORPHAN, Ordering::Relaxed); // still running+            }+        }+        // Nobody will wait for us if we're an orphan, or if our parent+        // said so by ignoring SIGCHLD explicitly (POSIX): reap ourselves.+        let parent_ignores = table.get(&process.parent()).is_some_and(|p| {+            let action = p.signals.lock().action(signal::SIGCHLD);+            action.handler == signal::SIG_IGN || action.flags & signal::SA_NOCLDWAIT != 0+        });+        if process.parent() == ORPHAN || parent_ignores {+            removed.extend(table.remove(&process.pid));+        }+        removed+    };+    drop(removed); // (frees their memory if they're completely gone)+    process.exited.wake_all(); // the kernel may be blocked in wait()+    process.vfork_done.store(true, Ordering::Release);+    VFORK_WAIT.wake_all();+    let code = match status {+        ExitStatus::Exited(_) => signal::CLD_EXITED,+        ExitStatus::Killed(sig, _) if signal::default_action(sig) == signal::DefaultAction::Core => signal::CLD_DUMPED,+        ExitStatus::Killed(..) => signal::CLD_KILLED,+    };+    let si_status = match status {+        ExitStatus::Exited(c) => c as i32,+        ExitStatus::Killed(sig, _) => sig as i32,+    };+    signal::notify_parent(&process, code, si_status);+    drop(process);+}+ /// Block (the calling thread) until `process` has ended and its thread is /// gone, then remove it from the process table. Used by the kernel (tests, /// the shell supervisor). Once the caller drops its own `Arc`, all of the@@ -593,18 +753,15 @@ pub fn wait(process: &Process) -> ExitStatus {     status } -/// Wait for the (exited) process's thread to be freed, then remove the+/// Wait for the (exited) process's threads to be freed, then remove the /// process from the table. fn reap(process: &Process) {-    // (The tid is stored right after the thread is created; on another CPU-    // the thread may have run and exited before that.)-    let tid = loop {-        match process.tid.load(Ordering::Acquire) {-            usize::MAX => thread::yield_now(),-            tid => break tid,-        }-    };-    thread::join(tid);+    // Every thread registers itself before it can exit, and the status is+    // only set when the last one exits: so the list is complete here.+    let kthreads = without_interrupts(|| core::mem::take(&mut *process.kthreads.lock()));+    for tid in kthreads {+        thread::join(tid);+    }     let removed = without_interrupts(|| PROCESSES.lock().remove(&process.pid));     drop(removed); }@@ -613,6 +770,8 @@ fn reap(process: &Process) { pub const WNOHANG: u64 = 1; pub const WUNTRACED: u64 = 2; pub const WCONTINUED: u64 = 8;+/// Linux's "wait for threads too" flags: accepted, nothing to do here.+const WALL_FLAGS: u64 = 0x2000_0000 | 0x4000_0000 | 0x8000_0000;  /// Kernel-side wait for one child (the tests use it): EAGAIN with WNOHANG /// while it runs.@@ -637,7 +796,7 @@ enum Found { /// -1: any child; 0: any child in my process group; < -1: any child in /// group -pid. Returns (pid, status), or pid 0 with WNOHANG and no news. pub fn wait4(me: &Process, pid: i64, options: u64) -> Result<(u64, i32), Errno> {-    if options & !(WNOHANG | WUNTRACED | WCONTINUED) != 0 {+    if options & !(WNOHANG | WUNTRACED | WCONTINUED | WALL_FLAGS) != 0 {         return Err(Errno::EINVAL);     }     let matches = |c: &Process| {@@ -706,17 +865,26 @@ pub fn fork(parent: &Arc<Process>, frame: &TrapFrame, stack: u64, vfork: bool) -     }     let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed);     let signals = without_interrupts(|| parent.signals.lock().fork());+    // Only the calling thread is copied, with its TLS pointer and mask.+    let me = thread::current_task();+    let fs_base = me.as_ref().map_or(0, |t| t.fs_base.load(Ordering::Relaxed));+    let blocked = me.as_ref().map_or(0, |t| without_interrupts(|| t.sig.lock().blocked));     let child = Process {         pid,         parent: AtomicU64::new(parent.pid),         pgid: AtomicU64::new(parent.pgid()),+        sid: AtomicU64::new(parent.sid()),+        ctty: Mutex::new(parent.ctty.lock().clone()),+        exe: Mutex::new(parent.exe.lock().clone()),         name: Mutex::new(parent.name()),         trace: AtomicBool::new(parent.tracing()),         cwd: Mutex::new(parent.cwd.lock().clone()),         mm: Mutex::new(mm),-        tid: AtomicUsize::new(usize::MAX),+        tasks: Mutex::new(Vec::new()),+        kthreads: Mutex::new(Vec::new()),+        exiting: Mutex::new(None),+        thread_exit: WaitQueue::new(),         ran_on: AtomicU32::new(0),-        start: Mutex::new(Some(start)),         env: Mutex::new(parent.env()),         fds: Mutex::new(parent.fds.lock().clone()), // same open files (Arc clones)         status: Mutex::new(None),@@ -726,7 +894,7 @@ pub fn fork(parent: &Arc<Process>, frame: &TrapFrame, stack: u64, vfork: bool) -         vfork_done: AtomicBool::new(false),     };     serial_println!("[process] pid {} forked pid {}", parent.pid, pid);-    let child = start_thread(child);+    let child = start_process(child, start, fs_base, blocked);     if vfork {         // The parent sleeps until the child has exec'd or exited (real         // vfork shares the memory meanwhile; we just use copy-on-write).@@ -735,9 +903,57 @@ pub fn fork(parent: &Arc<Process>, frame: &TrapFrame, stack: u64, vfork: bool) -     Ok(pid) } +/// clone(CLONE_THREAD ...): a new thread in process `p`, starting like the+/// caller returning from clone with rax = 0, on stack `stack`, with TLS+/// pointer `tls` (if Some). Returns its tid, which is also written to the+/// user addresses in `set_tid` (CLONE_PARENT_SETTID / CHILD_SETTID) before+/// the thread can run.+pub fn clone_thread(p: &Arc<Process>, frame: &TrapFrame, stack: u64, tls: Option<u64>, clear_tid: u64, set_tid: &[u64]) -> Result<u64, Errno> {+    let me = thread::current_task().ok_or(Errno::EINVAL)?;+    if p.tasks().len() >= MAX_THREADS || p.exiting().is_some() {+        return Err(Errno::EAGAIN);+    }+    let mut start = *frame;+    start.rax = 0; // "you're the new thread"+    if stack != 0 {+        start.rsp = stack;+    }+    for &ptr in set_tid {+        crate::syscall::user_slice_mut(ptr, 4)?; // check first+    }+    let tid = NEXT_PID.fetch_add(1, Ordering::Relaxed);+    for &ptr in set_tid {+        crate::syscall::user_slice_mut(ptr, 4)?.copy_from_slice(&(tid as u32).to_le_bytes());+    }+    let fs_base = tls.unwrap_or_else(|| me.fs_base.load(Ordering::Relaxed));+    let blocked = without_interrupts(|| me.sig.lock().blocked);+    // Forget scheduler threads of threads that are completely gone.+    without_interrupts(|| p.kthreads.lock().retain(|&k| thread::exists(k)));+    let task = start_task(p, tid, start, fs_base, blocked);+    task.clear_child_tid.store(clear_tid, Ordering::Relaxed);+    serial_println!("[process] pid {} new thread {}", p.pid, tid);+    Ok(tid)+}++/// execve in a process with several threads: the others must go first+/// (the new program starts with one thread). Linux calls this de_thread.+fn kill_other_threads(p: &Arc<Process>) {+    let Some(me) = thread::current_task() else { return };+    for t in p.tasks() {+        if !Arc::ptr_eq(&t, &me) {+            t.killed.store(true, Ordering::Release);+            t.wake();+        }+    }+    signal::wake_stopped();+    p.thread_exit.wait_until(|| (p.tasks.lock().len() == 1).then_some(()));+    // The survivor takes over the main thread id, like Linux.+    me.tid.store(p.pid, Ordering::Relaxed);+}+ /// execve, after the arguments have been copied in: replace the current /// program. `frame` gets the new program's registers.-pub fn execve(p: &Process, frame: &mut TrapFrame, path: &str, argv: &[Vec<u8>], envp: &[Vec<u8>]) -> Result<(), Errno> {+pub fn execve(p: &Arc<Process>, frame: &mut TrapFrame, path: &str, argv: &[Vec<u8>], envp: &[Vec<u8>]) -> Result<(), Errno> {     let size: usize = argv.iter().chain(envp).map(|s| s.len() + 1 + 8).sum();     if size > MAX_ARGS_SIZE {         return Err(Errno::E2BIG);@@ -750,6 +966,7 @@ pub fn execve(p: &Process, frame: &mut TrapFrame, path: &str, argv: &[Vec<u8>],      let args: Vec<&[u8]> = argv.iter().map(|a| a.as_slice()).collect();     let env: Vec<&[u8]> = envp.iter().map(|a| a.as_slice()).collect();     // --- point of no return ---+    kill_other_threads(p);     let loaded = {         let mut mm = p.mm.lock();         mm.clear();@@ -760,6 +977,11 @@ pub fn execve(p: &Process, frame: &mut TrapFrame, path: &str, argv: &[Vec<u8>],      };     *frame = start;     *p.name.lock() = String::from(name);+    *p.exe.lock() = String::from(path);+    if let Some(me) = thread::current_task() {+        me.fs_base.store(0, Ordering::Relaxed); // no TLS until the new libc sets one+        me.clear_child_tid.store(0, Ordering::Relaxed);+    }     *p.env.lock() = envp.iter().map(|e| String::from_utf8_lossy(e).into_owned()).collect();     without_interrupts(|| p.signals.lock().exec());     let closed = p.fds.lock().exec();@@ -789,6 +1011,20 @@ pub fn setpgid(me: &Process, pid: u64, pgid: u64) -> Result<(), Errno> {     Ok(()) } +/// setsid(): start a new session (and process group) led by `p`, with no+/// controlling terminal yet (the next terminal it opens becomes one, see+/// tty.rs). A process-group leader can't: its group would straddle two+/// sessions -- which is why getty forks first.+pub fn setsid(p: &Process) -> Result<u64, Errno> {+    if without_interrupts(|| PROCESSES.lock().values().any(|q| q.pgid() == p.pid)) {+        return Err(Errno::EPERM);+    }+    p.sid.store(p.pid, Ordering::Relaxed);+    p.pgid.store(p.pid, Ordering::Relaxed);+    *p.ctty.lock() = None;+    Ok(p.pid)+}+ /// Send `sig` from the kernel to group `pgid` (Ctrl+C, Ctrl+Z). pub fn signal_group(pgid: u64, sig: u32) {     let _ = signal::send_group(pgid, sig, SigInfo::from_pid(sig, 0));

@@ -36,11 +36,17 @@ //! `struct rt_sigframe`, so real libcs (musl) will work unchanged. //! //! **Stop and continue** (job control): SIGSTOP / SIGTSTP (Ctrl+Z) stop a-//! process -- its thread just sleeps -- until SIGCONT. The parent learns of+//! process -- its threads just sleep -- until SIGCONT. The parent learns of //! it from `wait4(WUNTRACED)` and SIGCHLD.+//!+//! **Threads.** Handlers (and so what a signal *does*) belong to the+//! process, but each thread has its own *mask* and its own pending set.+//! `kill` and Ctrl+C send to the process: whichever thread doesn't block+//! it takes it. A fault (SIGSEGV) or `tgkill` targets one thread. A fatal+//! signal kills all threads; a stop stops them all.  use crate::errno::Errno;-use crate::process::{self, ExitStatus, Process};+use crate::process::{self, ExitStatus, Process, Task}; use crate::thread::{self, WaitQueue}; use crate::trap::TrapFrame; use alloc::sync::Arc;@@ -199,16 +205,70 @@ pub enum Report {     Continued, } +/// Pending signals with their siginfo (standard signals don't queue: one+/// pending instance each, like Linux).+#[derive(Default)]+pub struct Pending {+    pub set: u64,+    info: Vec<(u32, SigInfo)>,+}++impl Pending {+    /// Mark `sig` pending with this siginfo.+    fn raise(&mut self, sig: u32, info: SigInfo) {+        self.set |= bit(sig);+        self.info.retain(|(s, _)| *s != sig);+        self.info.push((sig, info));+    }++    /// Take the lowest signal of `allowed` (SIGKILL first).+    fn take(&mut self, allowed: u64) -> Option<(u32, SigInfo)> {+        let set = self.set & allowed;+        if set == 0 {+            return None;+        }+        let sig = if set & bit(SIGKILL) != 0 { SIGKILL } else { set.trailing_zeros() + 1 };+        self.set &= !bit(sig);+        let info = match self.info.iter().position(|(s, _)| *s == sig) {+            Some(i) => self.info.swap_remove(i).1,+            None => SigInfo::new(sig, SI_KERNEL),+        };+        Some((sig, info))+    }++    fn discard(&mut self, mask: u64) {+        self.set &= !mask;+    }+}++/// One thread's signal state (`Task::sig`, locked with interrupts off).+pub struct ThreadSig {+    pub blocked: u64,+    /// Signals sent to this thread only (faults, tgkill).+    pub pending: Pending,+    /// Why the kernel forced a signal (a fault), for the kill message.+    reason: Option<&'static str>,+    /// The real mask while ppoll/pselect6/epoll_pwait run with a temporary+    /// one (see `set_temporary_mask`).+    saved_mask: Option<u64>,+}++impl ThreadSig {+    pub fn new(blocked: u64) -> ThreadSig {+        ThreadSig { blocked, pending: Pending::default(), reason: None, saved_mask: None }+    }++    /// The signals this thread may take now.+    fn allowed(&self) -> u64 {+        !(self.blocked & !UNBLOCKABLE)+    }+}+ /// A process's signal state. Locked with interrupts off (`Process::signals`). pub struct SignalState {-    pub pending: u64,-    pub blocked: u64,+    /// Signals sent to the whole process (kill, Ctrl+C, SIGCHLD).+    pub pending: Pending,     actions: [SigAction; NSIG as usize],-    /// siginfo of each pending signal (standard signals don't queue: one-    /// pending instance each, like Linux).-    info: Vec<(u32, SigInfo)>,-    /// Why the kernel forced this signal (a fault), for the kill message.-    reason: Option<&'static str>,     pub stopped: bool,     /// A stop/continue the parent hasn't collected with wait4 yet.     pub report: Option<Report>,@@ -216,21 +276,13 @@ pub struct SignalState {  impl SignalState {     pub fn new() -> SignalState {-        SignalState {-            pending: 0,-            blocked: 0,-            actions: [SigAction::default(); NSIG as usize],-            info: Vec::new(),-            reason: None,-            stopped: false,-            report: None,-        }+        SignalState { pending: Pending::default(), actions: [SigAction::default(); NSIG as usize], stopped: false, report: None }     } -    /// fork: the child inherits the handlers and the blocked mask, but no-    /// pending signals.+    /// fork: the child inherits the handlers (and, see process.rs, the+    /// calling thread's mask), but no pending signals.     pub fn fork(&self) -> SignalState {-        SignalState { actions: self.actions, blocked: self.blocked, ..SignalState::new() }+        SignalState { actions: self.actions, ..SignalState::new() }     }      /// execve: the program with the handlers is gone, so caught signals go@@ -255,37 +307,19 @@ impl SignalState {             || (handler == SIG_DFL && matches!(default_action(sig), DefaultAction::Ignore | DefaultAction::Continue))     } -    /// Pending and not blocked: should interrupt a wait.-    fn deliverable(&self) -> u64 {-        self.pending & !(self.blocked & !UNBLOCKABLE)-    }+} -    /// Take the lowest deliverable signal.-    fn dequeue(&mut self) -> Option<(u32, SigInfo)> {-        let set = self.deliverable();-        if set == 0 {-            return None;-        }-        // SIGKILL first, otherwise the lowest number.-        let sig = if set & bit(SIGKILL) != 0 { SIGKILL } else { set.trailing_zeros() + 1 };-        self.pending &= !bit(sig);-        let info = match self.info.iter().position(|(s, _)| *s == sig) {-            Some(i) => self.info.swap_remove(i).1,-            None => SigInfo::new(sig, SI_KERNEL),-        };-        Some((sig, info))-    }+/// Threads of a stopped process wait here for SIGCONT / SIGKILL.+static STOPPED: WaitQueue = WaitQueue::new(); -    /// Mark `sig` pending with this siginfo.-    fn raise(&mut self, sig: u32, info: SigInfo) {-        self.pending |= bit(sig);-        self.info.retain(|(s, _)| *s != sig);-        self.info.push((sig, info));-    }+/// Make stopped threads look at their state again (process.rs: a thread+/// group is exiting).+pub fn wake_stopped() {+    STOPPED.wake_all(); } -/// Thread of a stopped process waits here for SIGCONT / SIGKILL.-static STOPPED: WaitQueue = WaitQueue::new();+/// Signals that make a stopped process continue / that stop it.+const STOP_SIGNALS: u64 = bit(SIGSTOP) | bit(SIGTSTP) | bit(SIGTTIN) | bit(SIGTTOU);  /// Send signal `sig` to `p`. Returns at once; `p` acts on it on its way /// back to user mode.@@ -293,13 +327,17 @@ pub fn send(p: &Arc<Process>, sig: u32, info: SigInfo) {     if p.status().is_some() || !(1..=NSIG).contains(&sig) {         return; // already exited (a zombie): nothing to do     }+    let tasks = p.tasks();     let continued = without_interrupts(|| {         let mut s = p.signals.lock();         let mut continued = false;         if sig == SIGCONT || sig == SIGKILL {             // Continue (even if SIGCONT itself is ignored or blocked), and             // forget pending stops.-            s.pending &= !(bit(SIGSTOP) | bit(SIGTSTP) | bit(SIGTTIN) | bit(SIGTTOU));+            s.pending.discard(STOP_SIGNALS);+            for t in &tasks {+                t.sig.lock().pending.discard(STOP_SIGNALS);+            }             if s.stopped {                 s.stopped = false;                 s.report = Some(Report::Continued);@@ -307,12 +345,13 @@ pub fn send(p: &Arc<Process>, sig: u32, info: SigInfo) {             }         }         if default_action(sig) == DefaultAction::Stop {-            s.pending &= !bit(SIGCONT); // a stop cancels a pending continue+            s.pending.discard(bit(SIGCONT)); // a stop cancels a pending continue         }         // Ignored signals are dropped right away -- unless blocked: the         // program may unblock it after installing a handler.-        if sig == SIGKILL || !s.ignored(sig) || s.blocked & bit(sig) != 0 {-            s.raise(sig, info);+        let blocked = tasks.iter().any(|t| t.sig.lock().blocked & bit(sig) != 0);+        if sig == SIGKILL || !s.ignored(sig) || blocked {+            s.pending.raise(sig, info);         }         continued     });@@ -327,6 +366,28 @@ pub fn send(p: &Arc<Process>, sig: u32, info: SigInfo) {     STOPPED.wake_all(); // a stopped one needs to see SIGKILL } +/// Send `sig` to one thread (tgkill). Its effect (handler, death, stop)+/// is still the process's; only who takes it differs.+pub fn send_thread(t: &Task, sig: u32, info: SigInfo) {+    if t.process.status().is_some() || !(1..=NSIG).contains(&sig) {+        return;+    }+    if sig == SIGKILL || sig == SIGCONT || default_action(sig) == DefaultAction::Stop {+        // These act on the whole process anyway.+        if let Some(p) = process::get(t.process.pid) {+            return send(&p, sig, info);+        }+    }+    without_interrupts(|| {+        let s = t.process.signals.lock();+        let mut ts = t.sig.lock();+        if !s.ignored(sig) || ts.blocked & bit(sig) != 0 {+            ts.pending.raise(sig, info);+        }+    });+    t.wake();+}+ /// Send to every process in process group `pgid`. ESRCH if it's empty. pub fn send_group(pgid: u64, sig: u32, info: SigInfo) -> Result<(), Errno> {     let members = process::group_members(pgid);@@ -344,25 +405,32 @@ pub fn send_group(pgid: u64, sig: u32, info: SigInfo) -> Result<(), Errno> { /// the faulting instruction -- so if it is, the default action (death) /// applies, like Linux's force_sig. pub fn force(sig: u32, info: SigInfo, reason: &'static str) {-    let Some(p) = thread::current_process() else { return };+    let Some(t) = thread::current_task() else { return };     without_interrupts(|| {-        let mut s = p.signals.lock();-        if s.action(sig).handler == SIG_IGN || s.blocked & bit(sig) != 0 {+        let mut s = t.process.signals.lock();+        let mut ts = t.sig.lock();+        if s.action(sig).handler == SIG_IGN || ts.blocked & bit(sig) != 0 {             s.actions[sig as usize - 1] = SigAction::default();-            s.blocked &= !bit(sig);+            ts.blocked &= !bit(sig);         }-        s.raise(sig, info);-        s.reason = Some(reason);+        ts.pending.raise(sig, info);+        ts.reason = Some(reason);     }); } -/// Does the current process have a signal that should interrupt a-/// blocking syscall? (Called from thread.rs with interrupts off.)+/// Should the current thread stop sleeping in a syscall: a signal it can+/// take, its process stopping, or being told to exit? (Called from+/// thread.rs with interrupts off.) pub fn interrupted() -> bool {-    match thread::current_process() {-        Some(p) => without_interrupts(|| p.signals.lock().deliverable() != 0),-        None => false, // kernel threads don't get signals+    let Some(t) = thread::current_task() else { return false }; // kernel threads don't get signals+    if t.killed.load(core::sync::atomic::Ordering::Relaxed) || t.process.exiting().is_some() {+        return true;     }+    without_interrupts(|| {+        let s = t.process.signals.lock();+        let ts = t.sig.lock();+        s.stopped || (s.pending.set | ts.pending.set) & ts.allowed() != 0+    }) }  /// Tell `child`'s parent that it stopped/continued/exited: wake its wait4@@ -399,14 +467,27 @@ pub fn deliver(frame: &mut TrapFrame) {     if !frame.from_user() {         return;     }-    let Some(p) = thread::current_process() else { return };+    let Some(t) = thread::current_task() else { return };+    let p = t.process.clone();     // Writing the signal frame may fault user pages in: interrupts on.     x86_64::instructions::interrupts::enable();     loop {+        // Another thread called exit_group / execve, or died of a signal.+        if t.killed.load(core::sync::atomic::Ordering::Acquire) || p.exiting().is_some() {+            drop((t, p));+            process::exit_thread(ExitStatus::Exited(0));+        }+        // Another thread took a stop signal: we stop too.+        if without_interrupts(|| p.signals.lock().stopped) {+            wait_while_stopped(&t);+            continue;+        }         let next = without_interrupts(|| {             let mut s = p.signals.lock();-            let (sig, info) = s.dequeue()?;-            Some((sig, info, s.action(sig), s.reason.take()))+            let mut ts = t.sig.lock();+            let allowed = ts.allowed();+            let (sig, info) = ts.pending.take(allowed).or_else(|| s.pending.take(allowed))?;+            Some((sig, info, s.action(sig), ts.reason.take()))         });         let Some((sig, info, action, reason)) = next else { break };         match action.handler {@@ -414,11 +495,11 @@ pub fn deliver(frame: &mut TrapFrame) {             SIG_IGN | SIG_DFL => match default_action(sig) {                 DefaultAction::Ignore | DefaultAction::Continue => continue,                 DefaultAction::Stop if sig == SIGKILL => unreachable!(),-                DefaultAction::Stop => stop(&p, sig),+                DefaultAction::Stop => stop(&t, sig),                 DefaultAction::Terminate | DefaultAction::Core => {                     let why = reason.unwrap_or(name(sig));-                    drop(p);-                    process::exit_current(ExitStatus::Killed(sig, why));+                    drop((t, p));+                    process::exit_group(ExitStatus::Killed(sig, why));                 }             },             _ => {@@ -432,17 +513,19 @@ pub fn deliver(frame: &mut TrapFrame) {                     }                     frame.vector = 0; // dealt with: not a syscall return any more                 }-                if setup_frame(&p, frame, sig, &info, &action).is_err() {+                if setup_frame(&t, frame, sig, &info, &action).is_err() {                     // Couldn't write the frame (stack overflowed or bad                     // restorer): like Linux, the process dies of SIGSEGV.-                    drop(p);-                    process::exit_current(ExitStatus::Killed(SIGSEGV, "bad signal frame"));+                    drop((t, p));+                    process::exit_group(ExitStatus::Killed(SIGSEGV, "bad signal frame"));                 }                 without_interrupts(|| {                     let mut s = p.signals.lock();-                    s.blocked |= action.mask & !UNBLOCKABLE;+                    let mut ts = t.sig.lock();+                    ts.saved_mask = None; // (it's in the frame: sigreturn restores it)+                    ts.blocked |= action.mask & !UNBLOCKABLE;                     if action.flags & SA_NODEFER == 0 {-                        s.blocked |= bit(sig); // no nesting of the same signal+                        ts.blocked |= bit(sig); // no nesting of the same signal                     }                     if action.flags & SA_RESETHAND != 0 {                         s.actions[sig as usize - 1] = SigAction::default();@@ -452,24 +535,64 @@ pub fn deliver(frame: &mut TrapFrame) {             }         }     }-    // No handler ran: an interrupted syscall simply runs again.+    // No handler ran: the temporary mask of a ppoll & co. can go, and an+    // interrupted syscall simply runs again.+    restore_temporary_mask();     if frame.is_syscall() && restart_code(frame.rax).is_some() {         restart(frame);     } } -/// Stop the current process until SIGCONT (or SIGKILL).-fn stop(p: &Arc<Process>, sig: u32) {+/// ppoll / pselect6 / epoll_pwait / (sigsuspend-like) waits: block exactly+/// `mask` until `restore_temporary_mask`. If a signal interrupts the wait,+/// the syscall leaves the temporary mask in place and `deliver` restores+/// the real one -- after the handler, through the signal frame -- so the+/// signal that ended the wait can't be blocked again before it's handled.+pub fn set_temporary_mask(mask: u64) {+    if let Some(t) = thread::current_task() {+        without_interrupts(|| {+            let mut ts = t.sig.lock();+            let old = ts.blocked;+            ts.saved_mask.get_or_insert(old);+            ts.blocked = mask & !UNBLOCKABLE;+        });+    }+}++/// Put the real mask back (no-op without a temporary one).+pub fn restore_temporary_mask() {+    if let Some(t) = thread::current_task() {+        without_interrupts(|| {+            let mut ts = t.sig.lock();+            if let Some(mask) = ts.saved_mask.take() {+                ts.blocked = mask;+            }+        });+    }+}++/// Stop the current process (all its threads) until SIGCONT (or SIGKILL).+fn stop(t: &Arc<Task>, sig: u32) {+    let p = &t.process;     without_interrupts(|| {         let mut s = p.signals.lock();         s.stopped = true;         s.report = Some(Report::Stopped(sig));     });     crate::serial_println!("[signal] pid {} stopped by {}", p.pid, name(sig));+    p.wake(); // the other threads stop in deliver     notify_parent(p, CLD_STOPPED, sig as i32);+    wait_while_stopped(t);+}++/// Sleep until the process is continued, killed, or told to exit.+fn wait_while_stopped(t: &Task) {+    let p = &t.process;     STOPPED.wait_until(|| {         let s = p.signals.lock();-        (!s.stopped || s.pending & bit(SIGKILL) != 0).then_some(())+        let killed = (s.pending.set | t.sig.lock().pending.set) & bit(SIGKILL) != 0;+        let exiting = t.killed.load(core::sync::atomic::Ordering::Relaxed) || p.exiting().is_some();+        (!s.stopped || killed || exiting).then_some(())     }); } @@ -537,7 +660,7 @@ const USER_RFLAGS: u64 = 0x0004_0DD5 | (1 << 16); /// Note: we don't save the FPU/SSE registers (`fpstate` = 0). Our kernel /// doesn't use them and our programs' handlers don't touch floats, so they /// survive anyway; a full libc needs XSAVE state here (roadmap).-fn setup_frame(p: &Process, frame: &mut TrapFrame, sig: u32, info: &SigInfo, action: &SigAction) -> Result<(), ()> {+fn setup_frame(t: &Task, frame: &mut TrapFrame, sig: u32, info: &SigInfo, action: &SigAction) -> Result<(), ()> {     if action.flags & SA_RESTORER == 0 || action.restorer == 0 {         return Err(()); // nowhere to return to (x86_64 requires a restorer)     }@@ -546,7 +669,12 @@ fn setup_frame(p: &Process, frame: &mut TrapFrame, sig: u32, info: &SigInfo, act     // moving rsp), aligned so the handler starts with rsp % 16 == 8, as     // if it had been called.     let sp = ((frame.rsp.wrapping_sub(128).wrapping_sub(size)) & !0xf).wrapping_sub(8);-    let blocked = without_interrupts(|| p.signals.lock().blocked);+    // The mask sigreturn restores: the real one, if a ppoll & co. had+    // swapped in a temporary mask.+    let blocked = without_interrupts(|| {+        let ts = t.sig.lock();+        ts.saved_mask.unwrap_or(ts.blocked)+    });     let s = crate::gdt::selectors();     let mcontext = SigContext {         r8: frame.r8,@@ -602,13 +730,13 @@ pub fn sys_rt_sigreturn(frame: &mut TrapFrame) {     let sp = frame.rsp.wrapping_sub(8);     let size = core::mem::size_of::<RtSigFrame>() as u64;     if !crate::mm::user_access_ok(sp, size, false) {-        process::exit_current(ExitStatus::Killed(SIGSEGV, "bad sigreturn frame"));+        process::exit_group(ExitStatus::Killed(SIGSEGV, "bad sigreturn frame"));     }     let rt: RtSigFrame = unsafe { core::ptr::read_unaligned(sp as *const RtSigFrame) };     let m = rt.uc.mcontext;     let user = crate::memory::USER_START..crate::memory::USER_END;     if !user.contains(&m.rip) || !(user.start..=user.end).contains(&m.rsp) {-        process::exit_current(ExitStatus::Killed(SIGSEGV, "bad sigreturn frame"));+        process::exit_group(ExitStatus::Killed(SIGSEGV, "bad sigreturn frame"));     }     let s = crate::gdt::selectors();     *frame = TrapFrame {@@ -635,8 +763,8 @@ pub fn sys_rt_sigreturn(frame: &mut TrapFrame) {         vector: 0, // not a syscall return: rax is the restored value         error: 0,     };-    if let Some(p) = thread::current_process() {-        without_interrupts(|| p.signals.lock().blocked = rt.uc.sigmask & !UNBLOCKABLE);+    if let Some(t) = thread::current_task() {+        without_interrupts(|| t.sig.lock().blocked = rt.uc.sigmask & !UNBLOCKABLE);     } } @@ -662,20 +790,21 @@ pub fn sys_rt_sigaction(p: &Process, sig: u64, new: Option<SigAction>, set_size:             s.actions[sig as usize - 1] = new;             // Setting a signal to "ignored" discards it if pending (POSIX).             if s.ignored(sig) {-                s.pending &= !bit(sig);+                s.pending.discard(bit(sig));             }         }         old     })) } -/// rt_sigprocmask(how, *set, *old, sigsetsize = 8). Returns the old mask.-pub fn sys_rt_sigprocmask(p: &Process, how: u64, set: Option<u64>, set_size: u64) -> Result<u64, Errno> {+/// rt_sigprocmask(how, *set, *old, sigsetsize = 8): the calling thread's+/// mask. Returns the old mask.+pub fn sys_rt_sigprocmask(t: &Task, how: u64, set: Option<u64>, set_size: u64) -> Result<u64, Errno> {     if set_size != 8 {         return Err(Errno::EINVAL);     }     without_interrupts(|| {-        let mut s = p.signals.lock();+        let mut s = t.sig.lock();         let old = s.blocked;         if let Some(set) = set {             s.blocked = match how {

@@ -60,6 +60,57 @@ pub fn cpu_id() -> usize {     APIC_TO_CPU[apic::id() as usize & 0xFF].load(Ordering::Relaxed) as usize } +/// The address space (CR3 value) each CPU is running right now.+static ACTIVE_CR3: [AtomicU64; MAX_CPUS] = [const { AtomicU64::new(0) }; MAX_CPUS];+/// Set by a CPU asking CPU n to flush its TLB; CPU n clears it when done.+static FLUSH_PENDING: [AtomicBool; MAX_CPUS] = [const { AtomicBool::new(false) }; MAX_CPUS];++/// The scheduler tells us which address space `cpu` switched to.+pub fn set_active_cr3(cpu: usize, cr3: u64) {+    ACTIVE_CR3[cpu].store(cr3, Ordering::SeqCst);+}++/// **TLB shootdown.** Each CPU caches page-table entries in its TLB. When+/// a page is unmapped, made read-only or replaced (copy-on-write) in an+/// address space that threads are using on *other* CPUs right now, their+/// TLBs may still hold the old entry -- a thread could keep writing to a+/// page that was just freed. So: interrupt every other CPU running the+/// address space `cr3` and wait until each has flushed its TLB. (One+/// thread per process never needs this: no other CPU runs its address+/// space at the same time.) Call with interrupts *on*, after changing the+/// page tables and before freeing any frame they pointed to.+pub fn tlb_shootdown(cr3: u64) {+    let me = without_interrupts(cpu_id);+    let mut targets = 0u32;+    for cpu in 0..cpu_count() {+        if cpu != me && ACTIVE_CR3[cpu].load(Ordering::SeqCst) == cr3 {+            FLUSH_PENDING[cpu].store(true, Ordering::SeqCst);+            let target = CPU_TO_APIC[cpu].load(Ordering::Relaxed);+            without_interrupts(|| apic::send_ipi(target, interrupts::InterruptIndex::TlbShootdown as u8));+            targets |= 1 << cpu;+        }+    }+    // Wait until they're done. Another CPU may be shooting at *us* at the+    // same time while we have interrupts off: answer its request by hand,+    // or we'd wait for each other forever.+    while (0..MAX_CPUS).any(|cpu| targets & (1 << cpu) != 0 && FLUSH_PENDING[cpu].load(Ordering::SeqCst)) {+        without_interrupts(|| {+            if FLUSH_PENDING[cpu_id()].load(Ordering::SeqCst) {+                handle_tlb_shootdown();+            }+        });+        core::hint::spin_loop();+    }+}++/// The shootdown IPI arrived (interrupts.rs): forget all user translations.+pub fn handle_tlb_shootdown() {+    use x86_64::registers::control::Cr3;+    let (frame, flags) = Cr3::read();+    unsafe { Cr3::write(frame, flags) };+    FLUSH_PENDING[cpu_id()].store(false, Ordering::SeqCst);+}+ /// How many CPUs are running. pub fn cpu_count() -> usize {     CPU_COUNT.load(Ordering::Acquire)

@@ -9,17 +9,20 @@ //! man pages mostly apply. The process calls added in stage 14 (fork, //! execve, wait4, mmap, signals, dup, pipe2, fcntl, process groups) follow //! Linux exactly -- C strings, structs, flags -- so a C library can use-//! them as they are. The older file calls still take paths as pointer +-//! length (Linux: NUL-terminated); relative paths start at the cwd.+//! them as they are, and so do stage 15's (poll/epoll, futex, threads,+//! ttys, nanosleep, readv/writev...). The older file calls still take+//! paths as pointer + length (Linux: NUL-terminated); relative paths start+//! at the cwd. Every blocking call honours O_NONBLOCK (-> EAGAIN). //! //! | nr  | name        | arguments                           | note | //! |-----|-------------|-------------------------------------|------|-//! |   0 | read        | fd, buf, len                        | blocks: console until Enter, pipe until data/EOF |-//! |   1 | write       | fd, buf, len                        | to a pipe without readers: EPIPE + SIGPIPE |-//! |   2 | open        | path, path_len, flags               | O_CREAT 0o100, O_TRUNC 0o1000, O_APPEND, O_CLOEXEC 0o2000000 |+//! |   0 | read        | fd, buf, len                        | blocks: tty until a line (canonical) or a byte (raw), pipe until data/EOF |+//! |   1 | write       | fd, buf, len                        | to a pipe without readers: EPIPE + SIGPIPE; a full pipe blocks |+//! |   2 | open        | path, path_len, flags               | O_CREAT 0o100, O_NOCTTY 0o400, O_TRUNC 0o1000, O_APPEND, O_NONBLOCK 0o4000, O_CLOEXEC 0o2000000 | //! |   3 | close       | fd                                  | | //! |   4 | stat        | path, path_len, *Stat               | our own small `Stat {kind, size, mode}`; follows symlinks | //! |   5 | fstat       | fd, *Stat                           | |+//! |   7 | poll        | *pollfd, n, timeout_ms              | -1 = forever; pipes, sockets, ttys, files, epoll fds | //! |   8 | lseek       | fd, offset, whence                  | SEEK_SET/CUR/END = 0/1/2 | //! |   9 | mmap        | addr, len, prot, flags, fd, offset  | MAP_PRIVATE/SHARED, MAP_ANONYMOUS, MAP_FIXED; files: private only | //! |  10 | mprotect    | addr, len, prot                     | PROT_READ 1, PROT_WRITE 2, PROT_EXEC 4 |@@ -28,44 +31,65 @@ //! |  13 | rt_sigaction | sig, *new, *old, 8                 | kernel `struct sigaction`; handlers need SA_RESTORER | //! |  14 | rt_sigprocmask | how, *set, *old, 8               | SIG_BLOCK 0 / SIG_UNBLOCK 1 / SIG_SETMASK 2 | //! |  15 | rt_sigreturn |                                    | only from a signal handler's restorer |-//! |  16 | ioctl       | fd, request, arg                    | console: TTY_SET_RAW (ours), TIOCGPGRP, TIOCSPGRP, TIOCGWINSZ; /dev/fb0: FB_* (below) |+//! |  16 | ioctl       | fd, request, arg                    | any fd: FIONBIO; ttys: TCGETS/TCSETS(W/F), TIOCGPGRP/SPGRP, TIOCSCTTY, TIOCNOTTY, TIOCGWINSZ/SWINSZ, TIOCGPTN, TIOCSPTLCK, TTY_SET_RAW 1 (ours); /dev/fb0: FB_* (below) |+//! |  17 | pread64     | fd, buf, len, offset                | regular files; leaves the file position alone |+//! |  18 | pwrite64    | fd, buf, len, offset                | |+//! |  19 | readv       | fd, *iovec, n                       | `struct iovec {base, len}`, up to 1024 |+//! |  20 | writev      | fd, *iovec, n                       | | //! |  22 | pipe        | *[u32; 2]                           | [read end, write end] |+//! |  23 | select      | n, *rd, *wr, *ex, *timeval          | fd bitmaps (n <= 1024) | //! |  24 | sched_yield |                                     | | //! |  32 | dup         | fd                                  | lowest free fd | //! |  33 | dup2        | old, new                            | closes `new` first if open | //! |  34 | pause       |                                     | sleeps until a signal; returns EINTR |-//! |  35 | sleep_ms    | milliseconds                        | Linux's nr 35 is nanosleep(timespec); ours is simpler. EINTR on a signal |+//! |  35 | nanosleep   | *timespec, *remaining               | EINTR on a signal (remaining time written back) | //! |  39 | getpid      |                                     | |-//! |  41 | socket      | domain, type, protocol              | AF_INET (2) only; SOCK_STREAM (1) = TCP, SOCK_DGRAM (2) = UDP, SOCK_DGRAM + IPPROTO_ICMP (1) = ping socket |-//! |  42 | connect     | fd, *sockaddr_in, len               | TCP: handshake, blocks (10 s timeout); UDP/ICMP: set default destination |+//! |  41 | socket      | domain, type, protocol              | AF_INET (2) only; type may add SOCK_NONBLOCK/SOCK_CLOEXEC; SOCK_STREAM (1) = TCP, SOCK_DGRAM (2) = UDP, SOCK_DGRAM + IPPROTO_ICMP (1) = ping socket |+//! |  42 | connect     | fd, *sockaddr_in, len               | TCP: handshake, blocks (10 s timeout; nonblocking: EINPROGRESS); UDP/ICMP: set default destination. 127.x = loopback | //! |  43 | accept      | fd, *sockaddr_in or 0, *u32 len or 0 | blocks for a connection (SO_RCVTIMEO applies); returns the new fd | //! |  44 | sendto      | fd, buf, len, flags, *sockaddr_in or 0 | Linux has a 6th arg (addrlen); we only take 5, the address is always 16 bytes | //! |  45 | recvfrom    | fd, buf, len, flags, *sockaddr_in or 0 | blocks (SO_RCVTIMEO -> EAGAIN); 0 = TCP peer closed. No addrlen pointer | //! |  49 | bind        | fd, *sockaddr_in, len               | sets the local port (address ignored: we have one) | //! |  50 | listen      | fd, backlog                         | TCP; backlog 1..4 | //! |  54 | setsockopt  | fd, level, name, *value, len        | only SOL_SOCKET (1) / SO_RCVTIMEO (20) with a `struct timeval` |-//! |  56 | clone       | flags, stack, ...                   | only fork-like flags: SIGCHLD + optional CLONE_VFORK (else ENOSYS) |+//! |  56 | clone       | flags, stack, *ptid, *ctid, tls     | fork-like (SIGCHLD [+ CLONE_VFORK]) or a thread: CLONE_VM|FS|FILES|SIGHAND|THREAD|SYSVSEM [+ SETTLS, PARENT_SETTID, CHILD_CLEARTID] | //! |  57 | fork        |                                     | copy-on-write; parent gets the pid, child 0 | //! |  58 | vfork       |                                     | fork + the parent waits until the child execs or exits | //! |  59 | execve      | path, argv, envp (C strings)        | Linux's stack layout: argc, argv, envp, auxv |-//! |  60 | exit        | code                                | |+//! |  60 | exit        | code                                | ends this *thread* only (exit_group ends the process) | //! |  61 | wait4       | pid, *status, options, *rusage      | WNOHANG 1, WUNTRACED 2, WCONTINUED 8; rusage must be 0 | //! |  62 | kill        | pid, sig                            | pid 0 = my group, -1 = all, < -1 = group -pid | //! |  63 | uname       | *utsname                            | Linux's `struct utsname` (6 x 65-byte strings) |-//! |  72 | fcntl       | fd, cmd, arg                        | F_DUPFD 0, F_GETFD 1, F_SETFD 2, F_GETFL 3, F_SETFL 4, F_DUPFD_CLOEXEC 1030 |-//! |  79 | getcwd      | buf, len                            | returns the length (no NUL) |+//! |  72 | fcntl       | fd, cmd, arg                        | F_DUPFD 0, F_GETFD 1, F_SETFD 2, F_GETFL 3, F_SETFL 4 (O_NONBLOCK), F_DUPFD_CLOEXEC 1030 |+//! |  79 | getcwd      | buf, len                            | like Linux: writes a NUL, returns the length including it | //! |  80 | chdir       | path, path_len                      | | //! |  83 | mkdir       | path, path_len                      | | //! |  84 | rmdir       | path, path_len                      | only empty directories | //! |  87 | unlink      | path, path_len                      | files and symlinks (not what a symlink points to) |-//! |  89 | readlink    | path, path_len, buf, len            | a symlink's target; returns its length (cut to `len`, no NUL) |+//! |  89 | readlink    | path, path_len, buf, len            | a symlink's target; returns its length (cut to `len`, no NUL); /proc/self/exe = my program | //! | 109 | setpgid     | pid, pgid                           | 0 = myself / a new group | //! | 110 | getppid     |                                     | 0 if started by the kernel or orphaned | //! | 111 | getpgrp     |                                     | |+//! | 112 | setsid      |                                     | new session + group, no controlling tty | //! | 121 | getpgid     | pid                                 | |+//! | 124 | getsid      | pid                                 | |+//! | 158 | arch_prctl  | code, addr                          | ARCH_SET_FS 0x1002 / ARCH_GET_FS 0x1003: the TLS pointer |+//! | 186 | gettid      |                                     | thread id (= pid for the first thread) |+//! | 200 | tkill       | tid, sig                            | |+//! | 202 | futex       | *u32, op, val, *timespec, *u32, val3 | FUTEX_WAIT 0, WAKE 1, WAIT_BITSET 9, WAKE_BITSET 10 (+ PRIVATE 128) |+//! | 213 | epoll_create | size                               | = epoll_create1(0) | //! | 217 | getdents64  | fd, buf, len                        | Linux `struct linux_dirent64` records |-//! | 228 | clock_gettime | clock, *timespec                  | CLOCK_REALTIME/MONOTONIC both = time since boot (we have no RTC driver) |-//! | 231 | exit_group  | code                                | = exit (one thread per process) |+//! | 218 | set_tid_address | *u32                            | cleared + futex-woken when this thread exits; returns the tid |+//! | 228 | clock_gettime | clock, *timespec                  | REALTIME 0 (CMOS clock at boot + uptime), MONOTONIC 1 (since boot) |+//! | 231 | exit_group  | code                                | ends every thread of the process |+//! | 232 | epoll_wait  | epfd, *epoll_event, max, timeout_ms | level-triggered; EPOLLONESHOT supported |+//! | 233 | epoll_ctl   | epfd, op, fd, *epoll_event          | ADD 1, DEL 2, MOD 3 |+//! | 234 | tgkill      | tgid, tid, sig                      | a signal for one thread |+//! | 270 | pselect6    | n, *rd, *wr, *ex, *timespec, *{*mask, 8} | |+//! | 271 | ppoll       | *pollfd, n, *timespec, *mask, 8     | sets the mask while waiting |+//! | 281 | epoll_pwait | epfd, *events, max, timeout, *mask, 8 | |+//! | 288 | accept4     | fd, *addr, *len, flags              | flags: SOCK_NONBLOCK, SOCK_CLOEXEC |+//! | 291 | epoll_create1 | flags                             | EPOLL_CLOEXEC | //! | 292 | dup3        | old, new, flags                     | flags: O_CLOEXEC | //! | 293 | pipe2       | *[u32; 2], flags                    | flags: O_CLOEXEC | //! | 318 | getrandom   | buf, len, flags                     | random bytes (virtio-rng); up to 64 KiB per call; flags ignored |@@ -117,7 +141,7 @@ //! OS use, and it's faster.  use crate::errno::Errno;-use crate::fs::file::{InodeFile, Stat};+use crate::fs::file::{File, InodeFile, Stat}; use crate::fs::{self, InodeKind}; use crate::trap::TrapFrame; use crate::{memory, process, signal, thread};@@ -135,7 +159,7 @@ pub const SYS_LSEEK: u64 = 8; pub const SYS_IOCTL: u64 = 16; pub const SYS_PIPE: u64 = 22; pub const SYS_SCHED_YIELD: u64 = 24;-pub const SYS_SLEEP_MS: u64 = 35;+pub const SYS_NANOSLEEP: u64 = 35; pub const SYS_GETPID: u64 = 39; pub const SYS_EXIT: u64 = 60; pub const SYS_UNAME: u64 = 63;@@ -148,6 +172,7 @@ pub const SYS_READLINK: u64 = 89; pub const SYS_SOCKET: u64 = 41; pub const SYS_CONNECT: u64 = 42; pub const SYS_ACCEPT: u64 = 43;+pub const SYS_ACCEPT4: u64 = 288; pub const SYS_SENDTO: u64 = 44; pub const SYS_RECVFROM: u64 = 45; pub const SYS_BIND: u64 = 49;@@ -186,6 +211,28 @@ pub const SYS_EXIT_GROUP: u64 = 231; pub const SYS_DUP3: u64 = 292; pub const SYS_PIPE2: u64 = 293; pub const SYS_TRACE_ME: u64 = 505;+// stage 15+pub const SYS_POLL: u64 = 7;+pub const SYS_PREAD64: u64 = 17;+pub const SYS_PWRITE64: u64 = 18;+pub const SYS_READV: u64 = 19;+pub const SYS_WRITEV: u64 = 20;+pub const SYS_SELECT: u64 = 23;+pub const SYS_SETSID: u64 = 112;+pub const SYS_GETSID: u64 = 124;+pub const SYS_ARCH_PRCTL: u64 = 158;+pub const SYS_GETTID: u64 = 186;+pub const SYS_TKILL: u64 = 200;+pub const SYS_FUTEX: u64 = 202;+pub const SYS_EPOLL_CREATE: u64 = 213;+pub const SYS_SET_TID_ADDRESS: u64 = 218;+pub const SYS_EPOLL_WAIT: u64 = 232;+pub const SYS_EPOLL_CTL: u64 = 233;+pub const SYS_TGKILL: u64 = 234;+pub const SYS_PSELECT6: u64 = 270;+pub const SYS_PPOLL: u64 = 271;+pub const SYS_EPOLL_PWAIT: u64 = 281;+pub const SYS_EPOLL_CREATE1: u64 = 291;  // Socket constants (Linux's values). pub const AF_INET: u64 = 2;@@ -196,6 +243,7 @@ pub const SOL_SOCKET: u64 = 1; pub const SO_RCVTIMEO: u64 = 20;  pub const O_CREAT: u64 = 0o100;+pub const O_NOCTTY: u64 = 0o400; pub const O_TRUNC: u64 = 0o1000; pub const O_APPEND: u64 = 0o2000; pub const O_NONBLOCK: u64 = 0o4000;@@ -208,19 +256,29 @@ pub const F_GETFL: u64 = 3; pub const F_SETFL: u64 = 4; pub const F_DUPFD_CLOEXEC: u64 = 1030; pub const FD_CLOEXEC: u64 = 1;-/// Terminal ioctls (Linux's numbers).-pub const TIOCGPGRP: u64 = 0x540F;-pub const TIOCSPGRP: u64 = 0x5410;-pub const TIOCGWINSZ: u64 = 0x5413;+/// ioctl(FIONBIO, *int): O_NONBLOCK on/off, for any file.+pub const FIONBIO: u64 = 0x5421; /// mmap flags. pub const MAP_SHARED: u64 = 0x01; pub const MAP_PRIVATE: u64 = 0x02; pub const MAP_FIXED: u64 = 0x10; pub const MAP_ANONYMOUS: u64 = 0x20;-/// clone flags we understand.-pub const CLONE_VFORK: u64 = 0x4000;-/// ioctl request (ours): arg 1 = raw mode, 0 = cooked; returns the old mode.-pub const TTY_SET_RAW: u64 = 1;+/// clone flags (Linux's values).+pub const CLONE_VM: u64 = 0x100; // share the address space+pub const CLONE_FS: u64 = 0x200; // share cwd/umask+pub const CLONE_FILES: u64 = 0x400; // share the fd table+pub const CLONE_SIGHAND: u64 = 0x800; // share signal handlers+pub const CLONE_VFORK: u64 = 0x4000; // parent waits until the child execs/exits+pub const CLONE_THREAD: u64 = 0x10000; // same process: a thread+pub const CLONE_SYSVSEM: u64 = 0x40000;+pub const CLONE_SETTLS: u64 = 0x80000; // new thread's FS base = tls argument+pub const CLONE_PARENT_SETTID: u64 = 0x100000; // write the tid to *ptid+pub const CLONE_CHILD_CLEARTID: u64 = 0x200000; // at exit: *ctid = 0 + futex wake+pub const CLONE_DETACHED: u64 = 0x400000;+pub const CLONE_CHILD_SETTID: u64 = 0x1000000; // write the tid to *ctid+/// arch_prctl codes.+pub const ARCH_SET_FS: u64 = 0x1002;+pub const ARCH_GET_FS: u64 = 0x1003; pub const SPAWN_TRACE: u64 = 1; /// Framebuffer ioctls (see the table at the top). pub const FB_GET_INFO: u64 = 0x4600;@@ -241,7 +299,28 @@ pub fn name(nr: u64) -> &'static str {         SYS_IOCTL => "ioctl",         SYS_PIPE => "pipe",         SYS_SCHED_YIELD => "sched_yield",-        SYS_SLEEP_MS => "sleep_ms",+        SYS_NANOSLEEP => "nanosleep",+        SYS_POLL => "poll",+        SYS_PREAD64 => "pread64",+        SYS_PWRITE64 => "pwrite64",+        SYS_READV => "readv",+        SYS_WRITEV => "writev",+        SYS_SELECT => "select",+        SYS_SETSID => "setsid",+        SYS_GETSID => "getsid",+        SYS_ARCH_PRCTL => "arch_prctl",+        SYS_GETTID => "gettid",+        SYS_TKILL => "tkill",+        SYS_FUTEX => "futex",+        SYS_EPOLL_CREATE => "epoll_create",+        SYS_SET_TID_ADDRESS => "set_tid_address",+        SYS_EPOLL_WAIT => "epoll_wait",+        SYS_EPOLL_CTL => "epoll_ctl",+        SYS_TGKILL => "tgkill",+        SYS_PSELECT6 => "pselect6",+        SYS_PPOLL => "ppoll",+        SYS_EPOLL_PWAIT => "epoll_pwait",+        SYS_EPOLL_CREATE1 => "epoll_create1",         SYS_GETPID => "getpid",         SYS_EXIT => "exit",         SYS_WAIT4 => "wait4",@@ -284,6 +363,7 @@ pub fn name(nr: u64) -> &'static str {         SYS_SOCKET => "socket",         SYS_CONNECT => "connect",         SYS_ACCEPT => "accept",+        SYS_ACCEPT4 => "accept4",         SYS_SENDTO => "sendto",         SYS_RECVFROM => "recvfrom",         SYS_BIND => "bind",@@ -434,16 +514,16 @@ extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {         SYS_DUP2 => current().and_then(|p| if a1 == a2 { p.file(a1).map(|_| a1) } else { p.dup_to(a1, a2, false) }),         SYS_DUP3 => sys_dup3(a1, a2, a3),         SYS_PAUSE => Err(thread::pause()),-        SYS_SLEEP_MS => {-            // The PIT fires ~18.2 times a second: 1 tick = ~55 ms. Round up.-            thread::sleep_ticks_intr(a1.div_ceil(55)).map(|_| 0)-        }+        SYS_NANOSLEEP => sys_nanosleep(a1, a2),         SYS_GETPID => current().map(|p| p.pid),-        SYS_CLONE => sys_clone(frame, a1, a2),+        SYS_CLONE => sys_clone(frame, a1, a2, a3, a4, a5),         SYS_FORK => current().and_then(|p| process::fork(&p, frame, 0, false)),         SYS_VFORK => current().and_then(|p| process::fork(&p, frame, 0, true)),         SYS_EXECVE => sys_execve(frame, a1, a2, a3),-        SYS_EXIT | SYS_EXIT_GROUP => process::exit_current(process::ExitStatus::Exited(a1 as i32 as i64)),+        // exit ends only the calling thread (the process ends with its+        // last thread); exit_group ends them all. libc's exit() is the latter.+        SYS_EXIT => process::exit_thread(process::ExitStatus::Exited(a1 as i32 as i64)),+        SYS_EXIT_GROUP => process::exit_group(process::ExitStatus::Exited(a1 as i32 as i64)),         SYS_WAIT4 => sys_wait4(a1 as i32 as i64, a2, a3),         SYS_KILL => current().and_then(|p| signal::sys_kill(&p, a1 as i32 as i64, a2)),         SYS_UNAME => sys_uname(a1),@@ -464,17 +544,43 @@ extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {         SYS_SYSINFO => sys_sysinfo(a1),         SYS_SOCKET => sys_socket(a1, a2, a3),         SYS_CONNECT => with_socket(a1, |s| s.connect(read_sockaddr(a2)?)).map(|_| 0),-        SYS_ACCEPT => sys_accept(a1, a2, a3),+        SYS_ACCEPT => sys_accept(a1, a2, a3, 0),+        SYS_ACCEPT4 => sys_accept(a1, a2, a3, a4),         SYS_SENDTO => sys_sendto(a1, a2, a3, a5),         SYS_RECVFROM => sys_recvfrom(a1, a2, a3, a5),         SYS_BIND => with_socket(a1, |s| s.bind(read_sockaddr(a2)?.1)).map(|_| 0),         SYS_LISTEN => with_socket(a1, |s| s.listen(a2 as usize)).map(|_| 0),         SYS_SETSOCKOPT => sys_setsockopt(a1, a2, a3, a4, a5),-        SYS_CLOCK_GETTIME => sys_clock_gettime(a2),+        SYS_CLOCK_GETTIME => sys_clock_gettime(a1, a2),         SYS_RESOLVE => sys_resolve(a1, a2, a3, a4),         SYS_NETINFO => copy_to_user(a1, &crate::net::info()).map(|_| 0),         SYS_GETRANDOM => sys_getrandom(a1, a2),         SYS_TRACE_ME => current().map(|p| p.trace.store(a1 != 0, Ordering::Relaxed)).map(|_| 0),+        // --- stage 15 ---+        SYS_PREAD64 => sys_pread(a1, a2, a3, a4),+        SYS_PWRITE64 => sys_pwrite(a1, a2, a3, a4),+        SYS_READV => sys_readv(a1, a2, a3),+        SYS_WRITEV => sys_writev(a1, a2, a3),+        SYS_POLL => crate::poll::sys_poll_ms(a1, a2, a3 as i32 as i64),+        SYS_PPOLL => crate::poll::sys_ppoll(a1, a2, a3, a4),+        SYS_SELECT => crate::poll::sys_select_tv(a1, a2, a3, a4, a5),+        SYS_PSELECT6 => crate::poll::sys_pselect6(a1, a2, a3, a4, a5, frame.r9),+        SYS_EPOLL_CREATE => if (a1 as i32) <= 0 { Err(Errno::EINVAL) } else { crate::poll::sys_epoll_create1(0) },+        SYS_EPOLL_CREATE1 => crate::poll::sys_epoll_create1(a1),+        SYS_EPOLL_CTL => crate::poll::sys_epoll_ctl(a1, a2, a3, a4),+        SYS_EPOLL_WAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, 0),+        SYS_EPOLL_PWAIT => crate::poll::sys_epoll_pwait(a1, a2, a3 as i32 as i64, a4 as i32 as i64, a5),+        SYS_FUTEX => current().and_then(|p| crate::futex::sys_futex(&p, a1, a2, a3, a4, a5, frame.r9)),+        SYS_GETTID => task().map(|t| t.tid()),+        SYS_SET_TID_ADDRESS => task().map(|t| {+            t.clear_child_tid.store(a1, Ordering::Relaxed);+            t.tid()+        }),+        SYS_ARCH_PRCTL => sys_arch_prctl(a1, a2),+        SYS_TGKILL => sys_tgkill(Some(a1), a2, a3),+        SYS_TKILL => sys_tgkill(None, a1, a2),+        SYS_SETSID => current().and_then(|p| process::setsid(&p)),+        SYS_GETSID => current().and_then(|p| if a1 == 0 { Ok(p.sid()) } else { process::get(a1).map(|t| t.sid()).ok_or(Errno::ESRCH) }),         _ => Err(Errno::ENOSYS),     };     let ret = match result {@@ -535,7 +641,7 @@ fn trace_line(nr: u64, a: [u64; 4], ret: i64) -> String {         SYS_CHDIR | SYS_MKDIR | SYS_RMDIR | SYS_UNLINK => text(a[0], a[1]),         SYS_SPAWN => format!("{}, {}", text(a[0], a[1]), text(a[2], a[3])),         SYS_SCHED_YIELD | SYS_GETPID => String::new(),-        SYS_CLOSE | SYS_SLEEP_MS | SYS_EXIT | SYS_EXIT_GROUP | SYS_DUP => format!("{}", a[0] as i64),+        SYS_CLOSE | SYS_EXIT | SYS_EXIT_GROUP | SYS_DUP => format!("{}", a[0] as i64),         SYS_PIPE | SYS_UNAME => format!("{:#x}", a[0]),         SYS_FSTAT | SYS_GETCWD | SYS_MOUNTS => format!("{}, {:#x}", a[0], a[1]),         SYS_WAIT4 => format!("{}, {:#x}, {}", a[0] as i32, a[1], a[2]),@@ -555,6 +661,23 @@ fn current() -> Result<Arc<process::Process>, Errno> {     thread::current_process().ok_or(Errno::EINVAL) } +/// The calling thread.+fn task() -> Result<Arc<process::Task>, Errno> {+    thread::current_task().ok_or(Errno::EINVAL)+}++/// Read a `struct timespec {i64 seconds, i64 nanoseconds}` as microseconds+/// (rounded up, so a timeout never ends early).+pub fn read_timespec_us(ptr: u64) -> Result<u64, Errno> {+    let b = user_slice(ptr, 16)?;+    let secs = i64::from_le_bytes(b[..8].try_into().unwrap());+    let nsecs = i64::from_le_bytes(b[8..].try_into().unwrap());+    if secs < 0 || !(0..1_000_000_000).contains(&nsecs) {+        return Err(Errno::EINVAL);+    }+    Ok((secs as u64).saturating_mul(1_000_000).saturating_add((nsecs as u64).div_ceil(1000)))+}+ /// Turn a user pointer + length into a slice, after checking that the user /// may really access that memory. Never trust pointers from user space! pub fn user_slice(ptr: u64, len: u64) -> Result<&'static [u8], Errno> {@@ -607,7 +730,7 @@ fn user_path(ptr: u64, len: u64) -> Result<String, Errno> { }  /// Copy a plain-data struct out to user memory.-fn copy_to_user<T: Copy>(ptr: u64, value: &T) -> Result<(), Errno> {+pub fn copy_to_user<T: Copy>(ptr: u64, value: &T) -> Result<(), Errno> {     let size = core::mem::size_of::<T>();     let dst = user_slice_mut(ptr, size as u64)?;     let src = unsafe { core::slice::from_raw_parts(value as *const T as *const u8, size) };@@ -633,8 +756,15 @@ fn sys_open(path_ptr: u64, path_len: u64, flags: u64) -> Result<u64, Errno> {     };     // Device files (/dev/...) come with their own File.     let cloexec = flags & O_CLOEXEC != 0;-    if let Some(device) = inode.open_device() {-        return current()?.add_file_with(device, 0, cloexec);+    if let Some(device) = inode.open_device()? {+        let p = current()?;+        if flags & O_NONBLOCK != 0 {+            device.set_nonblocking(true);+        }+        if flags & O_NOCTTY == 0 {+            crate::tty::opened(&p, &*device); // maybe our controlling terminal now+        }+        return p.add_file_with(device, 0, cloexec);     }     if flags & O_TRUNC != 0 && inode.kind() == InodeKind::File {         inode.truncate()?;@@ -656,13 +786,17 @@ fn sys_fstat(fd: u64, stat_ptr: u64) -> Result<u64, Errno> { }  fn sys_pipe2(fds_ptr: u64, flags: u64) -> Result<u64, Errno> {-    if flags & !O_CLOEXEC != 0 {-        return Err(Errno::EINVAL); // O_NONBLOCK, O_DIRECT: not supported+    if flags & !(O_CLOEXEC | O_NONBLOCK) != 0 {+        return Err(Errno::EINVAL); // O_DIRECT (packet mode): not supported     }     let cloexec = flags & O_CLOEXEC != 0;     user_slice_mut(fds_ptr, 8)?; // check before creating anything     let process = current()?;     let (reader, writer) = fs::pipe::new();+    if flags & O_NONBLOCK != 0 {+        reader.set_nonblocking(true);+        writer.set_nonblocking(true);+    }     let read_fd = process.add_file_with(Arc::new(reader), 0, cloexec)?;     let write_fd = match process.add_file_with(Arc::new(writer), 0, cloexec) {         Ok(fd) => fd,@@ -682,8 +816,9 @@ fn sys_dup3(old: u64, new: u64, flags: u64) -> Result<u64, Errno> {     current()?.dup_to(old, new, flags & O_CLOEXEC != 0) } -/// fcntl: just the fd flags (close-on-exec) and duplication. Status flags:-/// everything is blocking, read/write (O_NONBLOCK isn't supported yet).+/// fcntl: the fd flags (close-on-exec), duplication, and the one status+/// flag we have: O_NONBLOCK (it belongs to the open file, so it's shared+/// with dup'ed fds). Everything is opened read/write. fn sys_fcntl(fd: u64, cmd: u64, arg: u64) -> Result<u64, Errno> {     let p = current()?;     let file = p.file(fd)?;@@ -693,15 +828,19 @@ fn sys_fcntl(fd: u64, cmd: u64, arg: u64) -> Result<u64, Errno> {         F_DUPFD_CLOEXEC => p.add_file_with(file, arg as usize, true),         F_GETFD => p.cloexec(fd).map(|on| if on { FD_CLOEXEC } else { 0 }),         F_SETFD => p.set_cloexec(fd, arg & FD_CLOEXEC != 0).map(|_| 0),-        F_GETFL => Ok(2), // O_RDWR-        F_SETFL if arg & O_NONBLOCK != 0 => Err(Errno::EINVAL),-        F_SETFL => Ok(0),+        F_GETFL => Ok(2 | if file.nonblocking() { O_NONBLOCK } else { 0 }), // O_RDWR+        F_SETFL => {+            file.set_nonblocking(arg & O_NONBLOCK != 0);+            Ok(0)+        }         _ => Err(Errno::EINVAL),     } } +/// getcwd: like Linux, the path with a NUL; returns the length + 1. fn sys_getcwd(buf: u64, len: u64) -> Result<u64, Errno> {-    let cwd = current()?.cwd.lock().clone();+    let mut cwd = current()?.cwd.lock().clone();+    cwd.push('\0');     if (len as usize) < cwd.len() {         return Err(Errno::ERANGE);     }@@ -719,7 +858,10 @@ fn sys_chdir(path_ptr: u64, path_len: u64) -> Result<u64, Errno> { }  fn sys_readlink(path_ptr: u64, path_len: u64, buf: u64, len: u64) -> Result<u64, Errno> {-    let target = fs::readlink(&user_path(path_ptr, path_len)?)?;+    let path = user_path(path_ptr, path_len)?;+    // There's no /proc yet, but libcs and programs ask "where am I?"+    // with readlink("/proc/self/exe"): answer just that one.+    let target = if path == "/proc/self/exe" { current()?.exe.lock().clone() } else { fs::readlink(&path)? };     let n = target.len().min(len as usize);     user_slice_mut(buf, n as u64)?.copy_from_slice(&target.as_bytes()[..n]);     Ok(n as u64)@@ -757,39 +899,26 @@ fn sys_spawn(path_ptr: u64, path_len: u64, argv_ptr: u64, argv_len: u64, opts_pt         cwd: parent.cwd.lock().clone(),         parent: parent.pid,         pgid: parent.pgid(), // same process group, like fork+        sid: parent.sid(),   // ...and session+        ctty: parent.ctty.lock().clone(),         trace: opts.flags & SPAWN_TRACE != 0,     };     drop(parent);     process::spawn_path(&path, spawn).map(|child| child.pid) } +/// ioctl: FIONBIO for any file; the rest is up to the file (terminals:+/// tty.rs; the framebuffer: below). fn sys_ioctl(fd: u64, request: u64, arg: u64) -> Result<u64, Errno> {-    let process = current()?;-    let file = process.file(fd)?;+    let file = current()?.file(fd)?;     if let Some(fb) = file.as_framebuffer() {         return fb_ioctl(fb, request, arg);     }-    if file.stat()?.kind != crate::fs::file::KIND_CONSOLE {-        return Err(Errno::ENOTTY);-    }-    match request {-        TTY_SET_RAW => Ok(crate::tty::set_raw(arg != 0, process.pid) as u64),-        // Which process group is in the foreground (gets Ctrl+C)?-        TIOCGPGRP => copy_to_user(arg, &(crate::tty::foreground() as u32)).map(|_| 0),-        TIOCSPGRP => {-            let pgid = read_user_u32s::<1>(arg)?[0] as u64;-            if process::group_members(pgid).is_empty() {-                return Err(Errno::ESRCH);-            }-            crate::tty::set_foreground(pgid);-            Ok(0)-        }-        TIOCGWINSZ => {-            let (cols, rows) = crate::vga_buffer::size();-            copy_to_user(arg, &[rows as u16, cols as u16, 0, 0]).map(|_| 0)-        }-        _ => Err(Errno::EINVAL),+    if request == FIONBIO {+        file.set_nonblocking(read_user_u32s::<1>(arg)?[0] != 0);+        return Ok(0);     }+    file.ioctl(request, arg) }  /// The /dev/fb0 ioctls (table at the top of this file).@@ -833,7 +962,7 @@ fn sys_uname(buf: u64) -> Result<u64, Errno> {         "Rustbelt",                     // sysname (`uname -s`)         "qemu",                         // nodename (hostname)         env!("CARGO_PKG_VERSION"),      // release (`uname -r`)-        "stage 14: processes",          // version+        "stage 15: libc plumbing",      // version         "x86_64",                       // machine         "(none)",                       // domainname     ];@@ -945,12 +1074,17 @@ fn sys_socket(domain: u64, ty: u64, protocol: u64) -> Result<u64, Errno> {         (SOCK_DGRAM, IPPROTO_ICMP) => Kind::Icmp,         _ => return Err(Errno::EPROTONOSUPPORT),     };-    current()?.add_file(Arc::new(Socket::new(kind)?))+    // SOCK_NONBLOCK and SOCK_CLOEXEC have O_NONBLOCK's and O_CLOEXEC's values.+    let socket = Socket::new(kind)?;+    socket.set_nonblocking(ty & O_NONBLOCK != 0);+    current()?.add_file_with(Arc::new(socket), 0, ty & O_CLOEXEC != 0) } -fn sys_accept(fd: u64, addr_ptr: u64, len_ptr: u64) -> Result<u64, Errno> {+/// accept(fd, *addr, *len) and accept4(..., flags: SOCK_NONBLOCK|SOCK_CLOEXEC).+fn sys_accept(fd: u64, addr_ptr: u64, len_ptr: u64, flags: u64) -> Result<u64, Errno> {     let (socket, peer) = with_socket(fd, |s| s.accept())?;-    let new_fd = current()?.add_file(Arc::new(socket))?;+    socket.set_nonblocking(flags & O_NONBLOCK != 0);+    let new_fd = current()?.add_file_with(Arc::new(socket), 0, flags & O_CLOEXEC != 0)?;     write_sockaddr(addr_ptr, peer)?;     if len_ptr != 0 {         copy_to_user(len_ptr, &16u32)?;@@ -985,9 +1119,15 @@ fn sys_setsockopt(fd: u64, level: u64, name: u64, value: u64, len: u64) -> Resul     Ok(0) } -/// `struct timespec {i64 seconds, i64 nanoseconds}` since boot.-fn sys_clock_gettime(ts_ptr: u64) -> Result<u64, Errno> {-    let us = crate::time::now_us();+/// clock_gettime(clock, *timespec): CLOCK_REALTIME (0, wall-clock time+/// since 1970, from the CMOS clock at boot) or the MONOTONIC family (1,+/// 4 RAW, 6 COARSE, 7 BOOTTIME: time since boot). CPU-time clocks: EINVAL.+fn sys_clock_gettime(clock: u64, ts_ptr: u64) -> Result<u64, Errno> {+    let us = match clock {+        0 | 5 => crate::time::realtime_us(), // REALTIME, REALTIME_COARSE+        1 | 4 | 6 | 7 => crate::time::now_us(),+        _ => return Err(Errno::EINVAL),+    };     copy_to_user(ts_ptr, &[us / 1_000_000, (us % 1_000_000) * 1000])?;     Ok(0) }@@ -1060,14 +1200,177 @@ fn sys_execve(frame: &mut TrapFrame, path: u64, argv: u64, envp: u64) -> Result<     Ok(0) } -/// clone: only the fork-like uses (no threads yet): exit signal SIGCHLD-/// (or 0) in the low byte, optionally CLONE_VFORK, optionally a new stack.-fn sys_clone(frame: &TrapFrame, flags: u64, stack: u64) -> Result<u64, Errno> {+/// clone(flags, stack, *ptid, *ctid, tls): a new thread (CLONE_THREAD)+/// or a new process (fork-like flags).+///+/// A thread shares everything with its process -- memory, fds, signal+/// handlers -- so CLONE_THREAD needs CLONE_VM and CLONE_SIGHAND, like+/// Linux. (CLONE_FILES / CLONE_FS are accepted and implied: our threads+/// always share the fd table and cwd.) pthread_create passes them all plus+/// SETTLS (its thread block), PARENT_SETTID and CHILD_CLEARTID (so+/// pthread_join can sleep on a futex until the thread is gone).+fn sys_clone(frame: &TrapFrame, flags: u64, stack: u64, ptid: u64, ctid: u64, tls: u64) -> Result<u64, Errno> {+    let p = current()?;+    if flags & CLONE_THREAD != 0 {+        const ALLOWED: u64 = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM+            | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID | CLONE_DETACHED;+        if flags & (CLONE_VM | CLONE_SIGHAND) != CLONE_VM | CLONE_SIGHAND || flags & !(ALLOWED | 0xff) != 0 {+            return Err(Errno::EINVAL);+        }+        let mut set_tid = Vec::new();+        if flags & CLONE_PARENT_SETTID != 0 {+            set_tid.push(ptid);+        }+        if flags & CLONE_CHILD_SETTID != 0 {+            set_tid.push(ctid);+        }+        let clear = if flags & CLONE_CHILD_CLEARTID != 0 { ctid } else { 0 };+        let tls = (flags & CLONE_SETTLS != 0).then_some(tls);+        return process::clone_thread(&p, frame, stack, tls, clear, &set_tid);+    }+    // A process: exit signal SIGCHLD (or 0) in the low byte, optionally+    // CLONE_VFORK (+ CLONE_VM: posix_spawn's "vfork"; we copy-on-write+    // instead of sharing, which a vfork child can't tell apart).     let exit_signal = flags & 0xff;-    if flags & !(0xff | CLONE_VFORK) != 0 || (exit_signal != 0 && exit_signal != signal::SIGCHLD as u64) {-        return Err(Errno::ENOSYS); // CLONE_VM, CLONE_THREAD...: threads are future work+    let allowed = 0xff | CLONE_VFORK | CLONE_PARENT_SETTID | if flags & CLONE_VFORK != 0 { CLONE_VM } else { 0 };+    if flags & !allowed != 0 || (exit_signal != 0 && exit_signal != signal::SIGCHLD as u64) {+        return Err(Errno::EINVAL); // shared memory/files between processes: not supported     }-    process::fork(&current()?, frame, stack, flags & CLONE_VFORK != 0)+    let pid = process::fork(&p, frame, stack, flags & CLONE_VFORK != 0)?;+    if flags & CLONE_PARENT_SETTID != 0 {+        copy_to_user(ptid, &(pid as u32))?;+    }+    Ok(pid)+}++/// arch_prctl(ARCH_SET_FS, addr) / (ARCH_GET_FS, *addr): the FS segment+/// base, which libc points at the thread's own data (thread-local storage:+/// `fs:[0]` is "this thread's block" on x86_64). Saved per thread and+/// reloaded by the scheduler on every switch.+fn sys_arch_prctl(code: u64, addr: u64) -> Result<u64, Errno> {+    let t = task()?;+    match code {+        ARCH_SET_FS => {+            if addr >= crate::memory::USER_END {+                return Err(Errno::EPERM); // must be a (canonical) user address+            }+            t.fs_base.store(addr, Ordering::Relaxed);+            x86_64::registers::model_specific::FsBase::write(x86_64::VirtAddr::new(addr));+            Ok(0)+        }+        ARCH_GET_FS => copy_to_user(addr, &t.fs_base.load(Ordering::Relaxed)).map(|_| 0),+        _ => Err(Errno::EINVAL),+    }+}++/// tgkill(tgid, tid, sig) / tkill(tid, sig): a signal for one thread+/// (pthread_kill, raise). sig 0 = just check it exists.+fn sys_tgkill(tgid: Option<u64>, tid: u64, sig: u64) -> Result<u64, Errno> {+    if sig >= signal::NSIG as u64 {+        return Err(Errno::EINVAL);+    }+    let me = current()?;+    let procs = match tgid {+        Some(pid) => alloc::vec![process::get(pid).ok_or(Errno::ESRCH)?],+        None => process::all(),+    };+    let t = procs.iter().find_map(|p| p.task(tid)).ok_or(Errno::ESRCH)?;+    if sig != 0 {+        signal::send_thread(&t, sig as u32, signal::SigInfo::from_pid(sig as u32, me.pid));+    }+    Ok(0)+}++/// nanosleep(*req, *rem). Our timer ticks every ~55 ms, so sleeps are+/// rounded up to that. Cut short by a signal: the time left goes to *rem.+fn sys_nanosleep(req: u64, rem: u64) -> Result<u64, Errno> {+    let us = read_timespec_us(req)?;+    if us == 0 {+        thread::yield_now();+        return Ok(0);+    }+    // The PIT ticks only every ~55 ms, so sleep whole ticks first and then+    // yield until the (TSC-precise) clock says the time is really up.+    let target = crate::time::now_us() + us;+    let deadline = crate::time::deadline_after_us(us).saturating_sub(1);+    let mut result = match deadline.checked_sub(crate::interrupts::ticks()) {+        Some(t) if t > 0 => thread::sleep_ticks_intr(t),+        _ => Ok(()),+    };+    while result.is_ok() && crate::time::now_us() < target {+        if crate::signal::interrupted() {+            result = Err(Errno::ERESTARTNOHAND);+        } else {+            thread::yield_now();+        }+    }+    if result.is_err() && rem != 0 {+        let left = target.saturating_sub(crate::time::now_us());+        copy_to_user(rem, &[left / 1_000_000, left % 1_000_000 * 1000])?;+    }+    result.map(|_| 0)+}++/// pread64 / pwrite64: at `offset`, without moving the file offset.+fn sys_pread(fd: u64, buf: u64, len: u64, offset: u64) -> Result<u64, Errno> {+    let file = current()?.file(fd)?;+    if (offset as i64) < 0 {+        return Err(Errno::EINVAL);+    }+    file.read_at(user_slice_mut(buf, len)?, offset).map(|n| n as u64)+}++fn sys_pwrite(fd: u64, buf: u64, len: u64, offset: u64) -> Result<u64, Errno> {+    let file = current()?.file(fd)?;+    if (offset as i64) < 0 {+        return Err(Errno::EINVAL);+    }+    file.write_at(user_slice(buf, len)?, offset).map(|n| n as u64)+}++/// The `struct iovec {void *base; size_t len;}` array of readv/writev.+fn read_iovecs(ptr: u64, count: u64) -> Result<Vec<(u64, u64)>, Errno> {+    if count > 1024 {+        return Err(Errno::EINVAL);+    }+    let raw = user_slice(ptr, count * 16)?;+    Ok(raw.chunks(16).map(|c| (u64::from_le_bytes(c[..8].try_into().unwrap()), u64::from_le_bytes(c[8..].try_into().unwrap()))).collect())+}++/// readv: one read into a kernel buffer, then spread over the buffers+/// (so a pipe or terminal sees a single read, like Linux).+fn sys_readv(fd: u64, iov: u64, count: u64) -> Result<u64, Errno> {+    let file = current()?.file(fd)?;+    let iovs = read_iovecs(iov, count)?;+    let total: u64 = iovs.iter().map(|v| v.1).sum::<u64>().min(1 << 20);+    for &(base, len) in &iovs {+        user_slice_mut(base, len)?; // check first+    }+    let mut buf = alloc::vec![0u8; total as usize];+    let n = file.read(&mut buf)?;+    let mut done = 0;+    for &(base, len) in &iovs {+        let k = (len as usize).min(n - done);+        user_slice_mut(base, k as u64)?.copy_from_slice(&buf[done..done + k]);+        done += k;+    }+    Ok(n as u64)+}++/// writev: gather the buffers and write them in one go (so the pieces of+/// one printf can't be interleaved with another thread's output).+fn sys_writev(fd: u64, iov: u64, count: u64) -> Result<u64, Errno> {+    let process = current()?;+    let file = process.file(fd)?;+    let mut buf = Vec::new();+    for (base, len) in read_iovecs(iov, count)? {+        buf.extend_from_slice(user_slice(base, len)?);+    }+    let result = file.write(&buf).map(|n| n as u64);+    if result == Err(Errno::EPIPE) {+        signal::send(&process, signal::SIGPIPE, signal::SigInfo::from_pid(signal::SIGPIPE, 0));+    }+    result }  fn sys_wait4(pid: i64, status_ptr: u64, options: u64) -> Result<u64, Errno> {@@ -1100,7 +1403,7 @@ fn sys_rt_sigaction(sig: u64, new_ptr: u64, old_ptr: u64, size: u64) -> Result<u  fn sys_rt_sigprocmask(how: u64, set_ptr: u64, old_ptr: u64, size: u64) -> Result<u64, Errno> {     let set = if set_ptr == 0 { None } else { Some(u64::from_le_bytes(user_slice(set_ptr, 8)?.try_into().unwrap())) };-    let old = signal::sys_rt_sigprocmask(&*current()?, how, set, size)?;+    let old = signal::sys_rt_sigprocmask(&*task()?, how, set, size)?;     if old_ptr != 0 {         copy_to_user(old_ptr, &old)?;     }

@@ -61,7 +61,7 @@ //! address" pointing at `thread_trampoline`, which calls the thread's function.  use crate::errno::Errno;-use crate::process::Process;+use crate::process::{Process, Task}; use crate::smp::{self, MAX_CPUS}; use crate::stack::KernelStack; use alloc::{@@ -100,8 +100,9 @@ struct Thread {     stack: Option<KernelStack>,     /// Level-4 page table to run with (the kernel's, or a process's own).     cr3: PhysFrame,-    /// The process this thread belongs to (None = pure kernel thread).-    process: Option<Arc<Process>>,+    /// The user thread (and through it the process) this kernel thread+    /// runs (None = pure kernel thread).+    task: Option<Arc<Task>>,     state: State,     /// Some CPU is (still) running on this thread's stack.     on_cpu: bool,@@ -215,13 +216,13 @@ impl Scheduler {     } } -fn new_thread(name: &str, stack: Option<KernelStack>, cr3: PhysFrame, process: Option<Arc<Process>>) -> Thread {+fn new_thread(name: &str, stack: Option<KernelStack>, cr3: PhysFrame, task: Option<Arc<Task>>) -> Thread {     Thread {         name: String::from(name),         rsp: 0,         stack,         cr3,-        process,+        task,         state: State::Ready,         on_cpu: false,         woken: false,@@ -290,13 +291,14 @@ pub fn spawn(name: &str, entry: fn()) -> Tid {     spawn_with(name, entry, crate::memory::kernel_l4_frame(), None) } -/// Start a thread for `process`: runs `entry` (in the kernel) with the-/// process's address space `cr3` active; `entry` then drops to ring 3.-pub fn spawn_with(name: &str, entry: fn(), cr3: PhysFrame, process: Option<Arc<Process>>) -> Tid {-    new_kernel_thread(name, entry, cr3, process, true)+/// Start a kernel thread for user thread `task`: runs `entry` (in the+/// kernel) with the process's address space `cr3` active; `entry` then+/// drops to ring 3.+pub fn spawn_with(name: &str, entry: fn(), cr3: PhysFrame, task: Option<Arc<Task>>) -> Tid {+    new_kernel_thread(name, entry, cr3, task, true) } -fn new_kernel_thread(name: &str, entry: fn(), cr3: PhysFrame, process: Option<Arc<Process>>, queue: bool) -> Tid {+fn new_kernel_thread(name: &str, entry: fn(), cr3: PhysFrame, task: Option<Arc<Task>>, queue: bool) -> Tid {     let stack = KernelStack::new().expect("out of memory for a kernel stack");      // Build the initial stack, top down (stacks grow towards lower addresses):@@ -316,7 +318,7 @@ fn new_kernel_thread(name: &str, entry: fn(), cr3: PhysFrame, process: Option<Ar     let rsp = top - 8 * initial.len() as u64;     unsafe { core::ptr::copy_nonoverlapping(initial.as_ptr(), rsp as *mut u64, initial.len()) }; -    let mut thread = new_thread(name, Some(stack), cr3, process);+    let mut thread = new_thread(name, Some(stack), cr3, task);     thread.rsp = rsp;     let id = interrupts::without_interrupts(|| {         let mut s = lock();@@ -369,8 +371,14 @@ pub fn schedule() {         t.state = State::Running;         t.on_cpu = true;         t.ran_on |= bit;-        if let Some(process) = &t.process {-            process.ran_on.fetch_or(bit, core::sync::atomic::Ordering::Relaxed);+        if let Some(task) = &t.task {+            task.process.ran_on.fetch_or(bit, core::sync::atomic::Ordering::Relaxed);+            // Thread-local storage: each user thread has its own FS base+            // (set with arch_prctl). The CPU has one FS_BASE register, so+            // it's reloaded on every switch to a user thread.+            x86_64::registers::model_specific::FsBase::write(x86_64::VirtAddr::new_truncate(+                task.fs_base.load(core::sync::atomic::Ordering::Relaxed),+            ));         }         if let Some(stack) = &t.stack {             if t.tlb_clean_on & bit == 0 {@@ -387,6 +395,8 @@ pub fn schedule() {             // every address space. Writing CR3 also flushes the TLB.             unsafe { Cr3::write(t.cr3, Cr3::read().1) };         }+        // Which address space this CPU runs now (TLB shootdowns, memory.rs).+        smp::set_active_cr3(cpu, t.cr3.start_address().as_u64());         CPU_IDLING[cpu].store(t.idle, Ordering::Relaxed);         let new_rsp = t.rsp;         s.current[cpu] = Some(next);@@ -414,7 +424,7 @@ fn finish_switch() {         let state = t.state;         // A finished thread: take out what it owns (its stack, and its         // share of the process) to free below, outside the lock.-        let leftovers = (state == State::Finished).then(|| (prev, t.stack.take(), t.process.take()));+        let leftovers = (state == State::Finished).then(|| (prev, t.stack.take(), t.task.take()));         (leftovers, state == State::Ready)     };     if requeued {@@ -456,7 +466,16 @@ pub fn current_process() -> Option<Arc<Process>> {     interrupts::without_interrupts(|| {         let mut s = lock();         let cur = s.current(smp::cpu_id());-        s.thread(cur).process.clone()+        s.thread(cur).task.as_ref().map(|t| t.process.clone())+    })+}++/// The user thread the running kernel thread carries, if any.+pub fn current_task() -> Option<Arc<Task>> {+    interrupts::without_interrupts(|| {+        let mut s = lock();+        let cur = s.current(smp::cpu_id());+        s.thread(cur).task.clone()     }) } @@ -666,7 +685,7 @@ pub fn die_of_stack_overflow(addr: u64) -> ! {         addr & !0xfff     );     if current_process().is_some() {-        crate::process::exit_current(crate::process::ExitStatus::Killed(crate::signal::SIGSEGV, "kernel stack overflow"));+        crate::process::exit_group(crate::process::ExitStatus::Killed(crate::signal::SIGSEGV, "kernel stack overflow"));     }     exit_current() }

@@ -42,8 +42,67 @@ pub fn now_us() -> u64 {     (rdtsc() - TSC_START.load(Ordering::Relaxed)) * 1000 / per_ms } -/// The PIT tick at which `us` microseconds from now will have passed-/// (rounded up, at least the next tick). For sleeping/timeouts.+/// The PIT tick by which `us` microseconds from now will surely have+/// passed. For sleeping/timeouts, which may run late but never early: we+/// may be anywhere inside the current ~55 ms tick, so one extra tick. pub fn deadline_after_us(us: u64) -> u64 {-    crate::interrupts::ticks() + us.div_ceil(PIT_TICK_US).max(1)+    crate::interrupts::ticks() + us.div_ceil(PIT_TICK_US) + 1+}++// ---------------------------------------------------------------------------+// Wall-clock time (CLOCK_REALTIME)+// ---------------------------------------------------------------------------++/// Unix time (microseconds since 1970-01-01 UTC) at boot (now_us = 0),+/// read from the CMOS clock on first use. 0 = not read yet.+static BOOT_REALTIME_US: AtomicU64 = AtomicU64::new(0);++/// Read a CMOS register (the PC's battery-backed real-time clock).+fn cmos(reg: u8) -> u8 {+    use x86_64::instructions::port::Port;+    unsafe {+        Port::<u8>::new(0x70).write(reg);+        Port::<u8>::new(0x71).read()+    }+}++/// The RTC's date and time as Unix seconds. QEMU's RTC runs on UTC.+fn rtc_unix_seconds() -> u64 {+    x86_64::instructions::interrupts::without_interrupts(|| {+        // Don't read in the middle of an update (bit 7 of status A).+        while cmos(0x0A) & 0x80 != 0 {+            core::hint::spin_loop();+        }+        let [sec, min, hour, day, month, year] = [0x00, 0x02, 0x04, 0x07, 0x08, 0x09].map(cmos);+        let status_b = cmos(0x0B);+        // Usually the values are BCD (0x59 = 59) unless bit 2 says binary.+        let bin = |v: u8| if status_b & 0x04 != 0 { v as u64 } else { ((v >> 4) * 10 + (v & 0xf)) as u64 };+        let mut h = bin(hour & 0x7f);+        if status_b & 0x02 == 0 && hour & 0x80 != 0 {+            h = (h + 12) % 24; // 12-hour clock, PM+        }+        let (y, m, d) = (2000 + bin(year), bin(month), bin(day));+        days_from_civil(y, m, d) * 86400 + h * 3600 + bin(min) * 60 + bin(sec)+    })+}++/// Days since 1970-01-01 for a date (Howard Hinnant's algorithm).+fn days_from_civil(y: u64, m: u64, d: u64) -> u64 {+    let y = if m <= 2 { y - 1 } else { y };+    let era = y / 400;+    let yoe = y - era * 400;+    let mp = (m + 9) % 12;+    let doy = (153 * mp + 2) / 5 + d - 1;+    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;+    era * 146097 + doe - 719468+}++/// Microseconds since 1970-01-01 UTC (clock_gettime(CLOCK_REALTIME)).+pub fn realtime_us() -> u64 {+    let mut boot = BOOT_REALTIME_US.load(Ordering::Relaxed);+    if boot == 0 {+        boot = (rtc_unix_seconds() * 1_000_000).saturating_sub(now_us()).max(1);+        BOOT_REALTIME_US.store(boot, Ordering::Relaxed);+    }+    boot + now_us() }

@@ -1,231 +1,1019 @@-//! The console "tty": keyboard input, and the `File` behind file-//! descriptors 0, 1 and 2.+//! Terminals ("ttys"): the console, /dev/hvc0, and pseudo-terminals. //!-//! Two input modes, like a Linux terminal:+//! **Why terminals are weird.** A 1970s terminal was a keyboard and a+//! screen on a serial line, and the kernel sat between that line and the+//! programs. Typing went through a "line discipline" that let you fix+//! typos *before* a program saw the line, echoed what you typed, and+//! turned a few keys into signals (Ctrl+C = stop that program). All of+//! that survives: every terminal -- the console, the virtio console, a+//! terminal window -- is a `Tty` running the same line discipline,+//! configured with a `termios` struct (TCGETS/TCSETS ioctls; libc's+//! tcgetattr/tcsetattr): //!-//! * **cooked** (canonical, the default): typed characters are echoed and-//!   collected in a line buffer (Backspace edits it); only when Enter is-//!   pressed does the whole line become readable from fd 0. So `read(0)`-//!   gets a complete, already-edited line. Programs like `cat` use this.-//! * **raw**: every key goes to the reader immediately and nothing is echoed;-//!   arrow keys arrive as ANSI escape sequences (`ESC [ A` = up). The shell-//!   switches to raw while it edits its command line, so it can do its own-//!   echo, Backspace and up-arrow history. (Linux: `tcsetattr` / `ioctl`.)+//! * **canonical** mode (ICANON, the default): typed bytes collect in a+//!   line buffer you can edit (Erase, ^U kills the line, ^D = end of+//!   file); `read` only sees whole lines. **Raw**/non-canonical mode:+//!   every byte is readable at once (editors, our shell's line editor).+//! * ECHO: the tty prints what you type (passwords turn it off).+//! * ISIG: ^C, ^\ and ^Z send SIGINT, SIGQUIT and SIGTSTP to the+//!   terminal's *foreground process group* -- the job the shell put in+//!   front (TIOCSPGRP). That's why ^C kills `cat` but not the shell.+//! * Output: ONLCR turns "\n" into "\r\n" (a real terminal needs the+//!   carriage return to go back to column 0). //!-//! In cooked mode Ctrl+C and Ctrl+Z don't become input: they send SIGINT /-//! SIGTSTP to the *foreground process group* (set by the shell with-//! ioctl TIOCSPGRP) -- that's how Ctrl+C stops the running program but not-//! the shell. (The virtio console, hvc0, has no job control.)+//! **Sessions.** `setsid()` starts a session (a login, roughly); its first+//! terminal opened becomes the session's *controlling terminal*, which+//! `/dev/tty` always refers to, and whose ^C goes to the session's+//! foreground group. When the session leader exits, the terminal is let+//! go and the foreground job gets SIGHUP ("hang up", from modems). //!-//! Output (`Console::write`) understands two control characters besides-//! '\n': '\x08' erases the character before the cursor, and '\x0c' (form-//! feed) clears the screen.+//! **Pseudo-terminals** (ptys) are terminals with a program instead of a+//! keyboard and screen on the other side: opening /dev/ptmx gives the+//! *master* end and creates /dev/pts/N, the *slave* end. Bytes written to+//! the master are "typed" into the slave (through the line discipline);+//! what programs write to the slave comes out of the master. Terminal+//! emulators, ssh and tmux are all programs holding a pty master.  use crate::errno::Errno;-use crate::fs::file::{File, KIND_CONSOLE, Stat};+use crate::fs::file::{File, KIND_CONSOLE, KIND_DEVICE, Stat};+use crate::poll::{self, POLLHUP, POLLIN, POLLOUT};+use crate::process::Process; use crate::signal; use crate::thread::WaitQueue; use crate::{print, serial_print, vga_buffer};-use alloc::string::String;+use alloc::{collections::VecDeque, format, string::String, sync::Arc, vec::Vec}; use conquer_once::spin::OnceCell;-use core::sync::atomic::{AtomicBool, AtomicU64, Ordering};-use crossbeam_queue::ArrayQueue;+use core::sync::atomic::{AtomicBool, Ordering}; use pc_keyboard::KeyCode; use spin::Mutex;+use x86_64::instructions::interrupts::without_interrupts; -/// The line being typed in cooked mode (not yet readable).-static LINE: Mutex<String> = Mutex::new(String::new());-/// Bytes waiting for `read(0, ...)`: finished lines (cooked) or keys (raw).-static INPUT: OnceCell<ArrayQueue<u8>> = OnceCell::uninit();-/// Raw mode on?-static RAW: AtomicBool = AtomicBool::new(false);-/// pid of the process that switched raw mode on (0 = none). If it exits-/// without switching back, the console goes back to cooked mode.-static RAW_OWNER: AtomicU64 = AtomicU64::new(0);+// --- termios (Linux's x86_64 layout and values) ----------------------------- -/// The foreground process group: Ctrl+C / Ctrl+Z signal it (0 = none).-/// The shell sets it to each job it runs, and back to itself after.-static FOREGROUND: AtomicU64 = AtomicU64::new(0);--pub fn foreground() -> u64 {-    FOREGROUND.load(Ordering::SeqCst)+/// `struct termios`: 36 bytes.+#[repr(C)]+#[derive(Clone, Copy)]+pub struct Termios {+    pub iflag: u32,+    pub oflag: u32,+    pub cflag: u32,+    pub lflag: u32,+    pub line: u8,+    pub cc: [u8; 19], } -pub fn set_foreground(pgid: u64) {-    FOREGROUND.store(pgid, Ordering::SeqCst);+// iflag+const INLCR: u32 = 0x40; // input '\n' -> '\r'+const IGNCR: u32 = 0x80; // drop input '\r'+const ICRNL: u32 = 0x100; // input '\r' -> '\n' (Enter sends '\r')+const IXON: u32 = 0x400;+// oflag+const OPOST: u32 = 0x1; // process output at all+const ONLCR: u32 = 0x4; // output '\n' -> "\r\n"+// lflag+const ISIG: u32 = 0x1;+const ICANON: u32 = 0x2;+const ECHO: u32 = 0x8;+const ECHOE: u32 = 0x10;+const ECHOK: u32 = 0x20;+const NOFLSH: u32 = 0x80;+const ECHOCTL: u32 = 0x200;+const ECHOKE: u32 = 0x800;+const IEXTEN: u32 = 0x8000;+// c_cc indexes+const VINTR: usize = 0;+const VQUIT: usize = 1;+const VERASE: usize = 2;+const VKILL: usize = 3;+const VEOF: usize = 4;+const VTIME: usize = 5;+const VMIN: usize = 6;+const VSUSP: usize = 10;+const VEOL: usize = 11;++impl Termios {+    /// What a fresh Linux terminal starts with (`stty sane`).+    pub const fn sane() -> Termios {+        let mut cc = [0u8; 19];+        cc[VINTR] = 3; // ^C+        cc[VQUIT] = 0x1c; // ^\+        cc[VERASE] = 0x7f; // DEL (we also take ^H, what our keyboard sends)+        cc[VKILL] = 0x15; // ^U+        cc[VEOF] = 4; // ^D+        cc[VMIN] = 1;+        cc[8] = 0x11; // VSTART ^Q+        cc[9] = 0x13; // VSTOP ^S+        cc[VSUSP] = 0x1a; // ^Z+        cc[12] = 0x12; // VREPRINT ^R+        cc[14] = 0x17; // VWERASE ^W+        cc[15] = 0x16; // VLNEXT ^V+        Termios {+            iflag: ICRNL | IXON,+            oflag: OPOST | ONLCR,+            cflag: 0xbf, // B38400 | CS8 | CREAD | HUPCL+            lflag: ISIG | ICANON | ECHO | ECHOE | ECHOK | ECHOCTL | ECHOKE | IEXTEN,+            line: 0,+            cc,+        }+    }+    fn bytes(&self) -> [u8; 36] {+        unsafe { core::mem::transmute(*self) }+    }+    fn from_bytes(b: &[u8]) -> Termios {+        unsafe { core::ptr::read_unaligned(b.as_ptr() as *const Termios) }+    } } -/// Threads blocked in `Console::read`, waiting for input.-static INPUT_WAIT: WaitQueue = WaitQueue::new();+// ioctl numbers+pub const TCGETS: u64 = 0x5401;+pub const TCSETS: u64 = 0x5402; // now+pub const TCSETSW: u64 = 0x5403; // after output drains (ours never queues)+pub const TCSETSF: u64 = 0x5404; // ...and throw away unread input+pub const TIOCSCTTY: u64 = 0x540E;+pub const TIOCGPGRP: u64 = 0x540F;+pub const TIOCSPGRP: u64 = 0x5410;+pub const TIOCGWINSZ: u64 = 0x5413;+pub const TIOCSWINSZ: u64 = 0x5414;+pub const FIONREAD: u64 = 0x541B;+pub const FIONBIO: u64 = 0x5421;+pub const TIOCNOTTY: u64 = 0x5422;+pub const TIOCGSID: u64 = 0x5429;+pub const TIOCGPTN: u64 = 0x8004_5430;+pub const TIOCSPTLCK: u64 = 0x4004_5431;+/// Ours (from stage 6): raw mode on/off in one call. Undone automatically+/// if the process that switched it on exits. Returns the old setting.+pub const TTY_SET_RAW: u64 = 1;++/// How many input bytes a tty buffers (like Linux's N_TTY: 4 KiB).+const INPUT_MAX: usize = 4096;+/// How much pty output can wait for the master to read it.+const OUTPUT_MAX: usize = 64 * 1024; -fn input() -> &'static ArrayQueue<u8> {-    INPUT.get_or_init(|| ArrayQueue::new(4096))+// --- the tty ----------------------------------------------------------------++/// Where a tty's output goes / its input comes from.+#[derive(Clone, Copy, PartialEq)]+enum Driver {+    Console, // keyboard in; VGA/framebuffer + serial out+    Hvc,     // virtio console (both ways)+    Pty,     // the master end, through `output` } -/// Switch between raw (`true`) and cooked mode for process `pid`. Returns-/// the old mode.-pub fn set_raw(raw: bool, pid: u64) -> bool {-    LINE.lock().clear(); // a half-typed cooked line doesn't survive a switch-    RAW_OWNER.store(if raw { pid } else { 0 }, Ordering::SeqCst);-    RAW.swap(raw, Ordering::SeqCst)+pub struct Tty {+    pub name: String,+    driver: Driver,+    /// Pty number (the N of /dev/pts/N).+    index: usize,+    /// Locked with interrupts off (touched in wait conditions).+    state: Mutex<TtyState>,+    readers: WaitQueue,+    writers: WaitQueue, } -/// A process ended: if it left the console in raw mode, undo that (so the-/// next program, or a respawned shell, gets normal line input).-pub fn process_exited(pid: u64) {-    if RAW_OWNER.compare_exchange(pid, 0, Ordering::SeqCst, Ordering::SeqCst).is_ok() {-        RAW.store(false, Ordering::SeqCst);-    }+struct TtyState {+    termios: Termios,+    /// The canonical-mode line being edited (not yet readable).+    line: Vec<u8>,+    /// Readable input: finished lines (canonical) or bytes. An empty chunk+    /// is an end of file (^D on an empty line): `read` returns 0 once.+    input: VecDeque<Vec<u8>>,+    winsize: [u16; 4], // rows, cols, x pixels, y pixels+    /// Session it's the controlling terminal of (0 = none).+    session: u64,+    /// Foreground process group (gets ^C).+    pgrp: u64,+    /// TTY_SET_RAW: who switched raw on, and the settings before.+    raw_owner: Option<(u64, Termios)>,+    // --- ptys only ---+    output: VecDeque<u8>, // slave output, for the master to read+    master_open: bool,+    slaves_open: usize,+    slave_opened: bool, // (master reads EIO only once a slave came and went)+    locked: bool,       // TIOCSPTLCK: slave can't be opened yet } -pub fn is_raw() -> bool {-    RAW.load(Ordering::SeqCst)+/// What the line discipline decided while holding the lock; done after.+#[derive(Default)]+struct Effects {+    echo: Vec<u8>,+    signal: Option<u32>, } -/// Make `bytes` readable and wake readers. (Queue full: drop -- nobody reads.)-fn deliver(bytes: &[u8]) {-    for &b in bytes {-        let _ = input().push(b);+impl Tty {+    fn new(name: String, driver: Driver, index: usize) -> Arc<Tty> {+        let (cols, rows) = vga_buffer::size();+        Arc::new(Tty {+            name,+            driver,+            index,+            state: Mutex::new(TtyState {+                termios: Termios::sane(),+                line: Vec::new(),+                input: VecDeque::new(),+                winsize: if driver == Driver::Console { [rows as u16, cols as u16, 0, 0] } else { [24, 80, 0, 0] },+                session: 0,+                pgrp: 0,+                raw_owner: None,+                output: VecDeque::new(),+                master_open: true,+                slaves_open: 0,+                slave_opened: false,+                locked: driver == Driver::Pty,+            }),+            readers: WaitQueue::new(),+            writers: WaitQueue::new(),+        })     }-    INPUT_WAIT.wake_all();-} -/// Called by the keyboard task for every typed character.-pub fn input_char(c: char) {-    if is_raw() {-        let mut utf8 = [0; 4];-        deliver(c.encode_utf8(&mut utf8).as_bytes());-        return;-    }-    let mut line = LINE.lock();-    match c {-        '\u{8}' => {-            // Backspace: only erase what's in the current line.-            if line.pop().is_some() {-                vga_buffer::backspace();-                serial_print!("\x08"); // uart_16550 turns this into "\x08 \x08"-            }-        }-        '\n' => {-            print!("\n");-            serial_print!("\n");-            line.push('\n');-            deliver(line.as_bytes());-            line.clear();-        }-        // Ctrl+C / Ctrl+Z: the "line discipline" turns them into signals-        // for the foreground job (Linux: the ISIG flag of termios). Raw mode-        // passes them through as bytes 3 / 26 instead (above).-        '\u{3}' | '\u{1a}' => {-            let (echo, sig) = if c == '\u{3}' { ("^C\n", signal::SIGINT) } else { ("^Z\n", signal::SIGTSTP) };-            print!("{}", echo);-            serial_print!("{}", echo);-            line.clear(); // the half-typed line is thrown away-            drop(line);-            let fg = foreground();-            if fg != 0 {-                crate::process::signal_group(fg, sig);-            }-        }-        c if (c as u32) < 0x20 && c != '\t' => {} // other control keys: ignore-        c => {-            print!("{}", c);-            serial_print!("{}", c);-            line.push(c);-        }-    }-}--/// Called by the keyboard task for keys that aren't characters (arrows...).-pub fn input_key(key: KeyCode) {-    if is_raw() {-        // The ANSI/VT100 sequences a Linux terminal would send.-        let seq: &[u8] = match key {-            KeyCode::ArrowUp => b"\x1b[A",-            KeyCode::ArrowDown => b"\x1b[B",-            KeyCode::ArrowRight => b"\x1b[C",-            KeyCode::ArrowLeft => b"\x1b[D",-            KeyCode::Home => b"\x1b[H",-            KeyCode::End => b"\x1b[F",-            KeyCode::Delete => b"\x1b[3~",-            _ => return,-        };-        deliver(seq);-    } else {-        print!("<{:?}>", key);-        serial_print!("<{:?}>", key);+    fn lock(&self) -> spin::MutexGuard<'_, TtyState> {+        self.state.lock()     }-} -/// The console as a file: reading gets keyboard input, writing prints to the-/// screen and serial port.-pub struct Console;+    /// Foreground process group.+    pub fn foreground(&self) -> u64 {+        without_interrupts(|| self.lock().pgrp)+    } -/// Held for the whole of one console `write` (see `Console::write`).-static CONSOLE_WRITE: spin::Mutex<()> = spin::Mutex::new(());+    pub fn set_foreground(&self, pgid: u64) {+        without_interrupts(|| self.lock().pgrp = pgid);+    } -impl File for Console {-    /// Blocks until input is available. Cooked: returns up to `buf.len()`-    /// bytes of a line, stopping after its '\n'. Raw: whatever keys are there.-    fn read(&self, buf: &mut [u8]) -> Result<usize, Errno> {-        if buf.is_empty() {-            return Ok(0);+    /// Session it's the controlling terminal of (0 = none).+    pub fn session(&self) -> u64 {+        without_interrupts(|| self.lock().session)+    }++    // --- input: the line discipline -----------------------------------------++    /// Bytes arriving from the keyboard / the host / the pty master.+    pub fn input(&self, bytes: &[u8]) {+        let (fx, pgrp) = without_interrupts(|| {+            let mut s = self.lock();+            let mut fx = Effects::default();+            for &b in bytes {+                s.input_byte(b, &mut fx, self.driver);+            }+            (fx, s.pgrp)+        });+        if !fx.echo.is_empty() {+            let _ = self.output(&fx.echo, false); // (echo into a full pty: dropped)+        }+        if let (Some(sig), true) = (fx.signal, pgrp != 0) {+            crate::process::signal_group(pgrp, sig);         }-        // Block until input arrives: the thread is parked on INPUT_WAIT-        // (using no CPU) and `deliver` wakes it. The closure runs with-        // interrupts off; the ArrayQueue is lock-free, so that's fine.-        INPUT_WAIT.wait_until_intr(|| {-            let mut n = 0;-            while n < buf.len() {-                let Some(b) = input().pop() else { break };-                buf[n] = b;-                n += 1;+        self.readers.wake_all();+        poll::notify();+    }++    // --- output -------------------------------------------------------------++    /// Send bytes towards the screen/host/master, with output processing+    /// (ONLCR). `block`: a program's write waits for room in a pty;+    /// echo doesn't (it's dropped instead).+    fn output(&self, bytes: &[u8], block: bool) -> Result<usize, Errno> {+        let termios = without_interrupts(|| self.lock().termios);+        let asked = bytes.len(); // what we report: the caller's count, not the cooked one+        let cooked: Vec<u8>;+        let onlcr = self.driver != Driver::Console && termios.oflag & (OPOST | ONLCR) == OPOST | ONLCR && bytes.contains(&b'\n');+        let bytes = if onlcr {+            let mut v = Vec::with_capacity(bytes.len() + 16);+            for &b in bytes {                 if b == b'\n' {-                    break;+                    v.push(b'\r');                 }+                v.push(b);             }-            if n > 0 { Some(n) } else { None }-        })+            cooked = v;+            &cooked[..]+        } else {+            bytes+        };+        match self.driver {+            Driver::Console => console_output(bytes),+            Driver::Hvc => {+                crate::virtio::console::write(bytes);+            }+            Driver::Pty => {+                let mut done = 0;+                let r = self.writers.wait_until_intr(|| {+                    let mut s = self.lock();+                    if !s.master_open {+                        return Some(Err(Errno::EIO)); // hung up+                    }+                    let n = (OUTPUT_MAX - s.output.len()).min(bytes.len() - done);+                    s.output.extend(&bytes[done..done + n]);+                    done += n;+                    if n > 0 {+                        self.readers.wake_all(); // (the master's readers)+                        poll::notify();+                    }+                    (done == bytes.len() || !block).then_some(Ok(()))+                });+                match r {+                    Ok(Err(e)) => return Err(e),+                    Err(e) if done == 0 => return Err(e),+                    _ => {}+                }+                if done < bytes.len() {+                    // Only part of it fit (nonblocking, or a signal). Report+                    // how many of the *caller's* bytes went out: every '\n'+                    // sent cost two cooked bytes ("\r\n"), and a lone '\r'+                    // added in front of an unsent '\n' doesn't count.+                    if done == 0 {+                        return Err(Errno::EAGAIN);+                    }+                    let mut sent = done;+                    if onlcr {+                        sent -= bytes[..done].iter().filter(|&&b| b == b'\n').count();+                        if bytes[done] == b'\n' {+                            sent -= 1;+                        }+                    }+                    return Ok(sent);+                }+            }+        }+        Ok(asked)     } -    fn write(&self, buf: &[u8]) -> Result<usize, Errno> {-        // One write = one unbroken piece of output, even when programs on-        // other CPUs write at the same time. Taken in thread context only-        // (never in an interrupt handler), so while waiting we just yield.-        let _turn = loop {-            match CONSOLE_WRITE.try_lock() {-                Some(guard) => break guard,-                None => crate::thread::yield_now(),+    // --- the slave/terminal side, as a file ---------------------------------++    fn read(&self, buf: &mut [u8], nonblocking: bool) -> Result<usize, Errno> {+        if buf.is_empty() {+            return Ok(0);+        }+        let (vmin, vtime, canon) = without_interrupts(|| {+            let t = self.lock().termios;+            (t.cc[VMIN], t.cc[VTIME], t.lflag & ICANON != 0)+        });+        // Non-canonical with VMIN = 0: don't wait (VTIME = 0) or wait at+        // most VTIME tenths of a second for something.+        let deadline = match (canon, vmin, vtime) {+            (false, 0, 0) => Some(0),+            (false, 0, t) => Some(crate::time::deadline_after_us(t as u64 * 100_000)),+            _ => None,+        };+        let mut take = || {+            let mut s = self.lock();+            if self.driver == Driver::Pty && !s.master_open && s.input.is_empty() {+                return Some(Ok(0)); // hung up: end of file+            }+            match s.take_input(buf, canon) {+                Some(n) => Some(Ok(n)),+                None if nonblocking => Some(Err(Errno::EAGAIN)),+                None => None,             }         };-        // Print valid UTF-8 as is, anything else as '?'. Control characters-        // are handled one by one.-        for chunk in buf.utf8_chunks() {-            for piece in chunk.valid().split_inclusive(['\x08', '\x0c']) {-                let (text, control) = match piece.strip_suffix(['\x08', '\x0c']) {-                    Some(text) => (text, piece.chars().last()),-                    None => (piece, None),+        let r = match deadline {+            None => self.readers.wait_until_intr(take)?,+            Some(0) => without_interrupts(|| take()).unwrap_or(Ok(0)),+            Some(d) => self.readers.wait_until_deadline_intr(d, take)?.unwrap_or(Ok(0)),+        };+        poll::notify(); // (room in the input buffer)+        r+    }++    fn poll_slave(&self) -> u16 {+        without_interrupts(|| {+            let s = self.lock();+            let mut events = if s.input.is_empty() { 0 } else { POLLIN };+            if self.driver == Driver::Pty {+                if !s.master_open {+                    events |= POLLHUP | POLLIN;+                } else if s.output.len() < OUTPUT_MAX {+                    events |= POLLOUT;+                }+            } else {+                events |= POLLOUT;+            }+            events+        })+    }++    // --- the pty master side ------------------------------------------------++    fn master_read(&self, buf: &mut [u8], nonblocking: bool) -> Result<usize, Errno> {+        if buf.is_empty() {+            return Ok(0);+        }+        let r = self.readers.wait_until_intr(|| {+            let mut s = self.lock();+            if s.output.is_empty() {+                return match () {+                    _ if s.slave_opened && s.slaves_open == 0 => Some(Err(Errno::EIO)), // Linux: EIO+                    _ if nonblocking => Some(Err(Errno::EAGAIN)),+                    _ => None,                 };-                print!("{}", text);-                serial_print!("{}", text);-                match control {-                    Some('\x08') => {-                        vga_buffer::backspace();-                        serial_print!("\x08");+            }+            let n = buf.len().min(s.output.len());+            for (dst, src) in buf.iter_mut().zip(s.output.drain(..n)) {+                *dst = src;+            }+            Some(Ok(n))+        })?;+        self.writers.wake_all();+        poll::notify();+        r+    }++    fn poll_master(&self) -> u16 {+        without_interrupts(|| {+            let s = self.lock();+            let mut events = POLLOUT;+            if !s.output.is_empty() {+                events |= POLLIN;+            }+            if s.slave_opened && s.slaves_open == 0 {+                events |= POLLHUP;+            }+            events+        })+    }++    // --- ioctls ---------------------------------------------------------------++    fn ioctl(&self, file: &TtyFile, cmd: u64, arg: u64) -> Result<u64, Errno> {+        use crate::syscall::{copy_to_user as copy_out, user_slice};+        let p = crate::thread::current_process().ok_or(Errno::EINVAL)?;+        let read_u32 = || -> Result<u32, Errno> { Ok(u32::from_le_bytes(user_slice(arg, 4)?.try_into().unwrap())) };+        match cmd {+            TCGETS => copy_out(arg, &without_interrupts(|| self.lock().termios).bytes()).map(|_| 0),+            TCSETS | TCSETSW | TCSETSF => {+                let t = Termios::from_bytes(user_slice(arg, 36)?);+                without_interrupts(|| self.lock().set_termios(t, cmd == TCSETSF));+                self.readers.wake_all();+                Ok(0)+            }+            TTY_SET_RAW => {+                let was_raw = without_interrupts(|| {+                    let mut s = self.lock();+                    let was_raw = s.termios.lflag & ICANON == 0;+                    if arg != 0 && !was_raw {+                        let old = s.termios;+                        s.raw_owner = Some((p.pid, old));+                        let mut raw = old;+                        raw.lflag &= !(ICANON | ECHO | ISIG | IEXTEN);+                        raw.iflag &= !(ICRNL | INLCR | IGNCR | IXON);+                        raw.cc[VMIN] = 1;+                        raw.cc[VTIME] = 0;+                        s.set_termios(raw, false);+                    } else if arg == 0 && was_raw {+                        let old = s.raw_owner.take().map_or(Termios::sane(), |(_, t)| t);+                        s.set_termios(old, false);+                    }+                    was_raw+                });+                self.readers.wake_all();+                Ok(was_raw as u64)+            }+            TIOCGWINSZ => copy_out(arg, &without_interrupts(|| self.lock().winsize)).map(|_| 0),+            TIOCSWINSZ => {+                let b = user_slice(arg, 8)?;+                let ws = [0, 2, 4, 6].map(|i| u16::from_le_bytes([b[i], b[i + 1]]));+                let pgrp = without_interrupts(|| {+                    let mut s = self.lock();+                    s.winsize = ws;+                    s.pgrp+                });+                if pgrp != 0 {+                    crate::process::signal_group(pgrp, signal::SIGWINCH);+                }+                Ok(0)+            }+            FIONREAD => {+                let n: usize = without_interrupts(|| {+                    let s = self.lock();+                    if file.master { s.output.len() } else { s.input.iter().map(|c| c.len()).sum() }+                });+                copy_out(arg, &(n as u32)).map(|_| 0)+            }+            FIONBIO => {+                file.set_nonblocking(read_u32()? != 0);+                Ok(0)+            }+            TIOCGPTN if file.master => copy_out(arg, &(self.index as u32)).map(|_| 0),+            TIOCSPTLCK if file.master => {+                let lock = read_u32()? != 0;+                without_interrupts(|| self.lock().locked = lock);+                Ok(0)+            }+            TIOCGPGRP => copy_out(arg, &(self.foreground() as u32)).map(|_| 0),+            TIOCSPGRP => {+                let pgid = read_u32()? as u64;+                let session = self.session();+                if session != 0 && session != p.sid() {+                    return Err(Errno::ENOTTY); // not our terminal+                }+                if crate::process::group_members(pgid).is_empty() {+                    return Err(Errno::ESRCH);+                }+                self.set_foreground(pgid);+                Ok(0)+            }+            TIOCGSID => match self.session() {+                0 => Err(Errno::ENOTTY),+                sid => copy_out(arg, &(sid as u32)).map(|_| 0),+            },+            TIOCSCTTY => attach(&p, self.self_arc()?, arg == 1).map(|_| 0),+            TIOCNOTTY => {+                let mine = p.ctty.lock().as_ref().is_some_and(|t| core::ptr::eq(&**t, self));+                if !mine {+                    return Err(Errno::ENOTTY);+                }+                if p.sid() == p.pid {+                    hang_up(self);+                }+                *p.ctty.lock() = None;+                Ok(0)+            }+            _ => Err(Errno::ENOTTY),+        }+    }++    /// The Arc for `self` (from the registry).+    fn self_arc(&self) -> Result<Arc<Tty>, Errno> {+        all_ttys().into_iter().find(|t| core::ptr::eq(&**t, self)).ok_or(Errno::ENOTTY)+    }+}++impl TtyState {+    fn set_termios(&mut self, t: Termios, flush: bool) {+        let was_canon = self.termios.lflag & ICANON != 0;+        self.termios = t;+        if flush {+            self.input.clear();+            self.line.clear();+        }+        // Leaving canonical mode: a half-edited line becomes readable.+        if was_canon && t.lflag & ICANON == 0 && !self.line.is_empty() {+            let line = core::mem::take(&mut self.line);+            self.input.push_back(line);+        }+    }++    fn buffered(&self) -> usize {+        self.line.len() + self.input.iter().map(|c| c.len()).sum::<usize>()+    }++    /// One input byte through the line discipline.+    fn input_byte(&mut self, mut b: u8, fx: &mut Effects, driver: Driver) {+        let t = self.termios;+        // Input translations: Enter sends '\r'; programs want '\n'.+        if b == b'\r' {+            if t.iflag & IGNCR != 0 {+                return;+            }+            if t.iflag & ICRNL != 0 {+                b = b'\n';+            }+        } else if b == b'\n' && t.iflag & INLCR != 0 {+            b = b'\r';+        }+        // ^C ^\ ^Z: signals for the foreground job.+        if t.lflag & ISIG != 0 {+            let sig = match b {+                _ if b == t.cc[VINTR] && b != 0 => Some(signal::SIGINT),+                _ if b == t.cc[VQUIT] && b != 0 => Some(signal::SIGQUIT),+                _ if b == t.cc[VSUSP] && b != 0 => Some(signal::SIGTSTP),+                _ => None,+            };+            if let Some(sig) = sig {+                if t.lflag & NOFLSH == 0 {+                    self.line.clear(); // the half-typed line is thrown away+                    self.input.clear();+                }+                if t.lflag & ECHO != 0 {+                    // "^C" -- plus a newline, so the next prompt starts+                    // on a fresh line (Linux leaves that to the shell).+                    fx.echo.extend_from_slice(&[b'^', b + 0x40, b'\n']);+                }+                fx.signal = Some(sig);+                return;+            }+        }+        if t.lflag & ICANON == 0 {+            // Raw: readable at once.+            if self.buffered() < INPUT_MAX {+                match self.input.back_mut() {+                    Some(chunk) if !chunk.is_empty() => chunk.push(b),+                    _ => self.input.push_back(alloc::vec![b]),+                }+                if t.lflag & ECHO != 0 {+                    fx.echo.push(b);+                }+            }+            return;+        }+        // Canonical: edit the line.+        let echo = t.lflag & ECHO != 0;+        let erase: &[u8] = if driver == Driver::Console { b"\x08" } else { b"\x08 \x08" };+        match b {+            _ if b == t.cc[VERASE] || b == 0x08 => {+                if self.line.pop().is_some() && echo && t.lflag & ECHOE != 0 {+                    fx.echo.extend_from_slice(erase);+                }+            }+            _ if b == t.cc[VKILL] => {+                for _ in 0..self.line.len() {+                    if echo && t.lflag & ECHOK != 0 {+                        fx.echo.extend_from_slice(erase);+                    }+                }+                self.line.clear();+            }+            _ if b == t.cc[VEOF] => {+                // ^D: hand over the line so far (without the ^D); on an+                // empty line that's a 0-byte read = end of file.+                let line = core::mem::take(&mut self.line);+                self.input.push_back(line);+            }+            b'\n' => {+                if echo {+                    fx.echo.push(b'\n');+                }+                self.line.push(b'\n');+                let line = core::mem::take(&mut self.line);+                self.input.push_back(line);+            }+            _ if b == t.cc[VEOL] && b != 0 => {+                self.line.push(b);+                let line = core::mem::take(&mut self.line);+                self.input.push_back(line);+            }+            _ if self.buffered() >= INPUT_MAX - 1 => {} // full (room kept for '\n')+            _ => {+                self.line.push(b);+                if echo {+                    if b < 0x20 && b != b'\t' && t.lflag & ECHOCTL != 0 {+                        fx.echo.extend_from_slice(&[b'^', b + 0x40]); // ^X+                    } else {+                        fx.echo.push(b);                     }-                    Some(_) => vga_buffer::clear_screen(), // serial: nothing to clear-                    None => {}                 }             }-            if !chunk.invalid().is_empty() {-                print!("?");-                serial_print!("?");+        }+    }++    /// Copy readable input to `buf`: canonical = at most one line (or an+    /// EOF = 0), raw = whatever is there. None = nothing to read.+    fn take_input(&mut self, buf: &mut [u8], canon: bool) -> Option<usize> {+        let mut n = 0;+        while n < buf.len() {+            let Some(chunk) = self.input.front_mut() else { break };+            if chunk.is_empty() {+                // An end of file: returned on its own (as 0) if first.+                if n == 0 {+                    self.input.pop_front();+                    return Some(0);+                }+                break;+            }+            let k = chunk.len().min(buf.len() - n);+            buf[n..n + k].copy_from_slice(&chunk[..k]);+            chunk.drain(..k);+            n += k;+            if chunk.is_empty() {+                self.input.pop_front();+                if canon {+                    break; // one line per read+                }+            }+        }+        (n > 0).then_some(n)+    }+}++/// Print to the screen and serial port (the console's output). '\x08'+/// erases the character before the cursor and '\x0c' clears the screen.+fn console_output(buf: &[u8]) {+    // One write = one unbroken piece of output, even when programs on+    // other CPUs write at the same time. Taken in thread context only, so+    // while waiting we just yield.+    let _turn = loop {+        match CONSOLE_WRITE.try_lock() {+            Some(guard) => break guard,+            None => crate::thread::yield_now(),+        }+    };+    for chunk in buf.utf8_chunks() {+        for piece in chunk.valid().split_inclusive(['\x08', '\x0c']) {+            let (text, control) = match piece.strip_suffix(['\x08', '\x0c']) {+                Some(text) => (text, piece.chars().last()),+                None => (piece, None),+            };+            print!("{}", text);+            serial_print!("{}", text);+            match control {+                Some('\x08') => {+                    vga_buffer::backspace();+                    serial_print!("\x08");+                }+                Some(_) => vga_buffer::clear_screen(), // serial: nothing to clear+                None => {}             }         }-        Ok(buf.len())+        if !chunk.invalid().is_empty() {+            print!("?");+            serial_print!("?");+        }+    }+}++static CONSOLE_WRITE: Mutex<()> = Mutex::new(());++// --- the registry: console, hvc0, ptys --------------------------------------++static CONSOLE: OnceCell<Arc<Tty>> = OnceCell::uninit();+static HVC: OnceCell<Arc<Tty>> = OnceCell::uninit();+/// Ptys by number; None = free.+static PTYS: Mutex<Vec<Option<Arc<Tty>>>> = Mutex::new(Vec::new());+/// Most ptys at once.+pub const MAX_PTYS: usize = 64;++/// The console tty (keyboard + screen).+pub fn console() -> Arc<Tty> {+    CONSOLE.get_or_init(|| Tty::new(String::from("console"), Driver::Console, 0)).clone()+}++/// The console as a file (fds 0/1/2 of programs started by the kernel).+pub fn console_file() -> Arc<dyn File> {+    Arc::new(TtyFile::new(console(), false))+}++/// /dev/hvc0's tty. Its input is fed by the `hvcd` kernel thread, started+/// the first time hvc0 is opened (until then, tests can read the raw+/// virtio console directly).+pub fn hvc() -> Arc<Tty> {+    HVC.get_or_init(|| {+        crate::thread::spawn("hvcd", hvc_input_thread);+        Tty::new(String::from("hvc0"), Driver::Hvc, 0)+    })+    .clone()+}++fn hvc_input_thread() {+    let mut buf = [0u8; 256];+    loop {+        if let Ok(n) = crate::virtio::console::read(&mut buf) {+            hvc().input(&buf[..n]);+        }+    }+}++fn all_ttys() -> Vec<Arc<Tty>> {+    let mut v: Vec<Arc<Tty>> = without_interrupts(|| PTYS.lock().iter().flatten().cloned().collect());+    v.extend(CONSOLE.get().cloned());+    v.extend(HVC.get().cloned());+    v+}++/// Open /dev/ptmx: a new pty; returns the master end.+pub fn open_ptmx() -> Result<Arc<dyn File>, Errno> {+    let tty = without_interrupts(|| {+        let mut ptys = PTYS.lock();+        let index = match ptys.iter().position(|p| p.is_none()) {+            Some(i) => i,+            None if ptys.len() < MAX_PTYS => {+                ptys.push(None);+                ptys.len() - 1+            }+            None => return Err(Errno::ENOSPC),+        };+        let tty = Tty::new(format!("pts/{}", index), Driver::Pty, index);+        ptys[index] = Some(tty.clone());+        Ok(tty)+    })?;+    Ok(Arc::new(TtyFile::new(tty, true)))+}++/// Pty numbers in use (the /dev/pts listing).+pub fn pty_numbers() -> Vec<usize> {+    without_interrupts(|| PTYS.lock().iter().enumerate().filter(|(_, p)| p.is_some()).map(|(i, _)| i).collect())+}++/// Open /dev/pts/N.+pub fn open_pts(n: usize) -> Result<Arc<dyn File>, Errno> {+    let tty = without_interrupts(|| PTYS.lock().get(n).cloned().flatten()).ok_or(Errno::ENOENT)?;+    let ok = without_interrupts(|| {+        let s = tty.lock();+        s.master_open && !s.locked+    });+    if !ok {+        return Err(Errno::EIO); // locked (not unlockpt'ed yet) or hung up+    }+    Ok(Arc::new(TtyFile::new(tty, false)))+}++/// Open /dev/hvc0 as a terminal.+pub fn open_hvc() -> Arc<dyn File> {+    Arc::new(TtyFile::new(hvc(), false))+}++/// Open /dev/tty: the caller's controlling terminal.+pub fn open_ctty() -> Result<Arc<dyn File>, Errno> {+    let p = crate::thread::current_process().ok_or(Errno::ENXIO)?;+    let tty = p.ctty.lock().clone().ok_or(Errno::ENXIO)?;+    Ok(Arc::new(TtyFile::new(tty, false)))+}++/// How many ptys exist (leak checks).+pub fn pty_count() -> usize {+    pty_numbers().len()+}++// --- sessions and controlling terminals ---------------------------------------++/// Make `tty` the controlling terminal of `p`'s session. Only a session+/// leader without one may, and only if the tty is free (or `steal`).+pub fn attach(p: &Process, tty: Arc<Tty>, steal: bool) -> Result<(), Errno> {+    if p.sid() != p.pid || p.ctty.lock().is_some() {+        return Err(Errno::EPERM);     }+    without_interrupts(|| {+        let mut s = tty.lock();+        if s.session != 0 && s.session != p.sid() && !steal {+            return Err(Errno::EPERM);+        }+        s.session = p.sid();+        s.pgrp = p.pgid();+        Ok(())+    })?;+    *p.ctty.lock() = Some(tty);+    Ok(())+} +/// open() of a terminal without O_NOCTTY: a session leader without a+/// controlling terminal gets this one (System V style, what Linux does).+pub fn opened(p: &Process, file: &dyn File) {+    if let Some(tf) = file.as_tty() {+        if !tf.master && p.sid() == p.pid && p.ctty.lock().is_none() && tf.tty.session() == 0 {+            let _ = attach(p, tf.tty.clone(), false);+        }+    }+}++/// The session leader let go of `tty`: no session owns it any more, and+/// its foreground job is told with SIGHUP (+ SIGCONT in case it's stopped).+fn hang_up(tty: &Tty) {+    let pgrp = without_interrupts(|| {+        let mut s = tty.lock();+        s.session = 0;+        core::mem::replace(&mut s.pgrp, 0)+    });+    if pgrp != 0 {+        crate::process::signal_group(pgrp, signal::SIGHUP);+        crate::process::signal_group(pgrp, signal::SIGCONT);+    }+}++/// A process ended: undo its TTY_SET_RAW, and if it led the session of+/// its controlling terminal, hang that up.+pub fn process_exited(p: &Process) {+    for tty in all_ttys() {+        let restored = without_interrupts(|| {+            let mut s = tty.lock();+            match s.raw_owner {+                Some((pid, old)) if pid == p.pid => {+                    s.raw_owner = None;+                    s.set_termios(old, false);+                    true+                }+                _ => false,+            }+        });+        if restored {+            tty.readers.wake_all();+        }+    }+    if let Some(tty) = p.ctty.lock().take() {+        if tty.session() == p.pid {+            hang_up(&tty);+        }+    }+}++// --- the old console API (keyboard driver, tests) ---------------------------++pub fn foreground() -> u64 {+    console().foreground()+}++pub fn set_foreground(pgid: u64) {+    console().set_foreground(pgid);+}++/// Called by the keyboard task for every typed character.+pub fn input_char(c: char) {+    let mut utf8 = [0; 4];+    console().input(c.encode_utf8(&mut utf8).as_bytes());+}++/// Keys that aren't characters: the ANSI/VT100 sequences a Linux terminal+/// sends for them.+pub fn input_key(key: KeyCode) {+    let seq: &[u8] = match key {+        KeyCode::ArrowUp => b"\x1b[A",+        KeyCode::ArrowDown => b"\x1b[B",+        KeyCode::ArrowRight => b"\x1b[C",+        KeyCode::ArrowLeft => b"\x1b[D",+        KeyCode::Home => b"\x1b[H",+        KeyCode::End => b"\x1b[F",+        KeyCode::Delete => b"\x1b[3~",+        _ => return,+    };+    console().input(seq);+}++// --- a terminal as a file -----------------------------------------------------++/// An open terminal: the console, hvc0, a pty slave -- or a pty master.+pub struct TtyFile {+    pub tty: Arc<Tty>,+    pub master: bool,+    nonblocking: AtomicBool,+}++impl TtyFile {+    fn new(tty: Arc<Tty>, master: bool) -> TtyFile {+        without_interrupts(|| {+            let mut s = tty.lock();+            if master {+                s.master_open = true;+            } else if tty.driver == Driver::Pty {+                s.slaves_open += 1;+                s.slave_opened = true;+            }+        });+        if tty.driver == Driver::Hvc && !master {+            crate::virtio::console::OPEN_COUNT.fetch_add(1, Ordering::Relaxed);+        }+        TtyFile { tty, master, nonblocking: AtomicBool::new(false) }+    }+}++impl File for TtyFile {+    fn read(&self, buf: &mut [u8]) -> Result<usize, Errno> {+        if self.master {+            self.tty.master_read(buf, self.nonblocking())+        } else {+            self.tty.read(buf, self.nonblocking())+        }+    }+    fn write(&self, buf: &[u8]) -> Result<usize, Errno> {+        if self.master {+            self.tty.input(buf); // "typed" into the slave+            Ok(buf.len())+        } else {+            self.tty.output(buf, !self.nonblocking())+        }+    }     fn stat(&self) -> Result<Stat, Errno> {-        Ok(Stat { kind: KIND_CONSOLE, size: 0, mode: 0 })+        // Terminals look like the console (that's what isatty checks).+        Ok(Stat { kind: if self.master { KIND_DEVICE } else { KIND_CONSOLE }, size: 0, mode: 0o620 })+    }+    fn poll(&self) -> u16 {+        if self.master { self.tty.poll_master() } else { self.tty.poll_slave() }+    }+    fn set_nonblocking(&self, on: bool) {+        self.nonblocking.store(on, Ordering::Relaxed);+    }+    fn nonblocking(&self) -> bool {+        self.nonblocking.load(Ordering::Relaxed)+    }+    fn ioctl(&self, cmd: u64, arg: u64) -> Result<u64, Errno> {+        self.tty.ioctl(self, cmd, arg)+    }+    fn as_tty(&self) -> Option<&TtyFile> {+        Some(self)+    }+}++impl Drop for TtyFile {+    fn drop(&mut self) {+        let tty = &self.tty;+        if tty.driver == Driver::Hvc && !self.master {+            crate::virtio::console::OPEN_COUNT.fetch_sub(1, Ordering::Relaxed);+        }+        if tty.driver != Driver::Pty {+            return;+        }+        let master_gone = without_interrupts(|| {+            let mut s = tty.lock();+            if self.master {+                s.master_open = false;+            } else {+                s.slaves_open -= 1;+            }+            self.master+        });+        if master_gone {+            // The terminal "hung up": its session hears SIGHUP, readers+            // get EOF, writers EIO; the number is free again.+            hang_up(tty);+            without_interrupts(|| PTYS.lock()[tty.index] = None);+        }+        tty.readers.wake_all();+        tty.writers.wake_all();+        poll::notify();     } }

@@ -0,0 +1,111 @@+//! Stage 15, the plumbing a C library needs: threads and futexes, poll and+//! epoll, nonblocking I/O, loopback networking, ttys and ptys. As in+//! stage 14 the detailed checks live in user programs (threads,+//! echoserver, ptytest, nbtest: exit code = failed checks); this file runs+//! them and adds what only the kernel can see: nothing is left behind+//! (threads, processes, futex waiters, ptys, sockets, memory frames).+#![no_std]+#![no_main]+#![feature(custom_test_frameworks)]+#![test_runner(rustbelt::test_runner)]+#![reexport_test_harness_main = "test_main"]++extern crate alloc;++use alloc::string::String;+use bootloader::{BootInfo, entry_point};+use core::panic::PanicInfo;+use rustbelt::process::{self, ExitStatus, SpawnArgs};+use rustbelt::{fs, futex, memory, net, thread, tty};++entry_point!(main);++fn main(boot_info: &'static BootInfo) -> ! {+    rustbelt::init();+    rustbelt::init_memory(boot_info);+    rustbelt::init_smp(); // all 4 CPUs run threads+    fs::init();+    rustbelt::init_storage();+    rustbelt::init_net(); // brings up 127.0.0.1 (and eth0 if QEMU has one)+    test_main();+    rustbelt::hlt_loop();+}++fn run(name: &str, args: &[&str]) -> ExitStatus {+    let mut spawn = SpawnArgs::kernel(name);+    spawn.args.extend(args.iter().map(|a| String::from(*a)));+    let p = process::spawn_path(&alloc::format!("/mnt/disk0/bin/{}", name), spawn).expect("spawn failed");+    process::wait(&p)+}++/// Snapshot of everything a buggy program could leak.+#[derive(Debug, PartialEq)]+struct Counts {+    threads: usize,+    processes: usize,+    futex_waiters: usize,+    ptys: usize,+    sockets: usize,+    frames: usize,+}++fn counts() -> Counts {+    thread::sleep_ticks(3); // let exited threads' stacks be freed+    Counts {+        threads: thread::thread_count(),+        processes: process::count(),+        futex_waiters: futex::waiters(),+        ptys: tty::pty_count(),+        sockets: net::socket_count(),+        frames: memory::frames_in_use(),+    }+}++/// Run a test program twice: once to warm caches, then again to check that+/// it passes and that every count comes back to where it was.+fn run_without_leaks(name: &str) {+    assert_eq!(run(name, &[]), ExitStatus::Exited(0), "{} failed (warm-up)", name);+    let before = counts();+    assert_eq!(run(name, &[]), ExitStatus::Exited(0), "{} failed", name);+    let after = counts();+    assert_eq!(after, before, "{} leaked something", name);+    assert_eq!(after.futex_waiters, 0, "futex waiters left behind");+    assert_eq!(after.ptys, 0, "ptys left behind");+}++/// 4 threads sum 1..=40,000,000 under a futex mutex, on all 4 CPUs.+#[test_case]+fn threads_and_futexes() {+    run_without_leaks("threads");+}++/// One epoll and one poll server over 127.0.0.1, three forked clients each.+#[test_case]+fn poll_epoll_echo_over_loopback() {+    run_without_leaks("echoserver");+}++/// pty round trips, canonical vs raw, ^C -> SIGINT, hangup on close.+#[test_case]+fn ptys_and_line_discipline() {+    run_without_leaks("ptytest");+}++/// EAGAIN everywhere, fcntl/FIONBIO, writev/pread/pwrite, clocks.+#[test_case]+fn nonblocking_io_and_libc_extras() {+    run_without_leaks("nbtest");+}++/// Loopback works without DHCP and without the NIC: 127.0.0.1 is local+/// whatever eth0 is doing.+#[test_case]+fn loopback_is_always_there() {+    assert!(net::is_loopback([127, 0, 0, 1]));+    assert!(!net::is_loopback([10, 0, 2, 15]));+}++#[panic_handler]+fn panic(info: &PanicInfo) -> ! {+    rustbelt::test_panic_handler(info)+}

@@ -35,7 +35,8 @@ fn main(boot_info: &'static BootInfo) -> ! {     rustbelt::init_smp();     fs::init();     rustbelt::init_storage();-    rustbelt::init_net().expect("no virtio-net card: test-args need -device virtio-net-pci");+    rustbelt::init_net();+    assert!(net::present(), "no virtio-net card: test-args need -device virtio-net-pci");     test_main();     rustbelt::hlt_loop(); }

@@ -176,7 +176,7 @@ fn scripted_shell_session() {         "piped text\n",         "wc -l\n2\n",         "saved\nmore\n",-        "Rustbelt 0.14.0",+        "Rustbelt 0.15.0",         "B:/ $ exit 3\n",     ] {         assert!(out.contains(expected), "missing {:?} in:\n{}", expected, out);

@@ -150,7 +150,7 @@ fn dev_lists_the_devices() {     let dev = fs::lookup("/dev").expect("no /dev");     let mut names: Vec<String> = dev.list().expect("list /dev").into_iter().map(|e| e.name).collect();     names.sort();-    assert_eq!(names, ["fb0", "hvc0", "null", "random", "urandom"]);+    assert_eq!(names, ["console", "fb0", "hvc0", "null", "ptmx", "pts", "random", "tty", "urandom"]);     assert_eq!(fs::lookup("/dev/fb0").unwrap().size(), 1024 * 768 * 4); } 

@@ -0,0 +1,206 @@+//! echoserver -- an event-loop echo server over loopback (127.0.0.1).+//!+//! One process serves several clients at once without threads: it never+//! blocks on any single socket. Every socket is nonblocking, and the loop+//! asks the kernel "which of my sockets are ready?" (epoll_wait, or plain+//! poll), then only accepts/reads/writes those:+//!+//!     loop {+//!         ready = epoll_wait(...)        // sleep until something happens+//!         for each ready fd:+//!             listener -> accept new clients, watch them too+//!             client   -> read; echo it back; 0 bytes = client left+//!     }+//!+//! The self-test forks 3 client processes that talk to it over loopback,+//! once with epoll and once with poll, then checks UDP over loopback.+//! With an argument (`echoserver PORT`) it just serves forever.+#![no_std]+#![no_main]++#[macro_use]+#[path = "ulib.rs"]+mod ulib;++extern crate alloc;++use alloc::format;+use alloc::vec::Vec;+use ulib::*;++const LOOPBACK: Ip = Ip([127, 0, 0, 1]);+const CLIENTS: usize = 3;+const ROUNDS: usize = 3;++/// A client process: connect, send ROUNDS messages, check each echo.+fn client(port: u16, id: usize) -> i64 {+    let fd = socket(SOCK_STREAM, 0);+    if connect(fd, LOOPBACK, port) < 0 {+        return 10;+    }+    for round in 0..ROUNDS {+        let msg = format!("hello {} from client {}", round, id);+        write_all(fd, msg.as_bytes());+        let mut buf = [0u8; 64];+        let mut got = 0;+        while got < msg.len() {+            let n = read(fd, &mut buf[got..]);+            if n <= 0 {+                return 11;+            }+            got += n as usize;+        }+        if &buf[..got] != msg.as_bytes() {+            return 12;+        }+    }+    close(fd); // the server sees end-of-file+    0+}++fn listener(port: u16) -> i64 {+    let fd = socket(SOCK_STREAM | O_NONBLOCK, 0); // SOCK_NONBLOCK+    if bind(fd, port) < 0 || listen(fd, 4) < 0 {+        return -1;+    }+    fd+}++/// Accept everyone waiting (nonblocking: stop at EAGAIN).+fn accept_all(lfd: i64, mut add: impl FnMut(i64)) {+    loop {+        let (fd, _) = accept(lfd);+        if fd < 0 {+            break; // -EAGAIN: nobody else waiting+        }+        set_nonblocking(fd, true);+        add(fd);+    }+}++/// Read what's there and echo it back. Returns false when the client left.+fn echo(fd: i64, served: &mut usize) -> bool {+    let mut buf = [0u8; 512];+    loop {+        let n = read(fd, &mut buf);+        if n == -EAGAIN {+            return true; // drained for now+        }+        if n <= 0 {+            return false; // 0 = end of file (or an error)+        }+        write_all(fd, &buf[..n as usize]);+        *served += 1;+    }+}++/// The epoll flavour: register fds once, then wait for just the ready ones.+/// Returns the number of clients served to the end (or -1 on a stall).+fn serve_epoll(lfd: i64, want: usize) -> i64 {+    let ep = epoll_create1(0);+    epoll_ctl(ep, EPOLL_CTL_ADD, lfd, EPOLLIN, lfd as u64);+    let (mut finished, mut served) = (0, 0);+    let mut events = [EpollEvent::default(); 8];+    while finished < want {+        let n = epoll_wait(ep, &mut events, 5000);+        if n <= 0 {+            close(ep);+            return -1; // 5 s with nothing happening: something's wrong+        }+        for ev in &events[..n as usize] {+            let fd = ev.data as i64;+            if fd == lfd {+                accept_all(lfd, |c| {+                    epoll_ctl(ep, EPOLL_CTL_ADD, c, EPOLLIN, c as u64);+                });+            } else if !echo(fd, &mut served) {+                epoll_ctl(ep, EPOLL_CTL_DEL, fd, 0, 0);+                close(fd);+                finished += 1;+            }+        }+    }+    close(ep);+    println!("echoserver: epoll: {} clients, {} echoes", finished, served);+    finished as i64+}++/// The poll flavour: pass the whole fd list on every call.+fn serve_poll(lfd: i64, want: usize) -> i64 {+    let mut fds = alloc::vec![PollFd { fd: lfd as i32, events: POLLIN, revents: 0 }];+    let (mut finished, mut served) = (0, 0);+    while finished < want {+        if poll(&mut fds, 5000) <= 0 {+            return -1;+        }+        let mut new = Vec::new();+        let mut gone = Vec::new();+        for p in fds.iter() {+            if p.revents == 0 {+                continue;+            }+            if p.fd as i64 == lfd {+                accept_all(lfd, |c| new.push(c));+            } else if !echo(p.fd as i64, &mut served) {+                close(p.fd as i64);+                gone.push(p.fd);+                finished += 1;+            }+        }+        fds.retain(|p| !gone.contains(&p.fd));+        fds.extend(new.into_iter().map(|c| PollFd { fd: c as i32, events: POLLIN, revents: 0 }));+    }+    println!("echoserver: poll: {} clients, {} echoes", finished, served);+    finished as i64+}++/// Start the clients, run the server, collect the clients' results.+fn round(c: &mut Checks, port: u16, use_epoll: bool) {+    let lfd = listener(port);+    c.check(lfd >= 0, "listen on a loopback port");+    let mut pids = Vec::new();+    for id in 0..CLIENTS {+        let pid = fork();+        if pid == 0 {+            close(lfd);+            exit(client(port, id));+        }+        pids.push(pid);+    }+    let finished = if use_epoll { serve_epoll(lfd, CLIENTS) } else { serve_poll(lfd, CLIENTS) };+    close(lfd);+    let ok = pids.iter().all(|&pid| wait(pid) == 0);+    let what = if use_epoll { "epoll" } else { "poll" };+    c.check(finished == CLIENTS as i64, &format!("{}: served {} clients to the end", what, CLIENTS));+    c.check(ok, &format!("{}: every client got its echoes back", what));+}++fn main() -> i64 {+    if let Some(port) = arg(1).and_then(|a| a.parse().ok()) {+        let lfd = listener(port);+        if lfd < 0 {+            eprintln!("echoserver: can't listen on port {}", port);+            return 1;+        }+        println!("echoserver: listening on port {} (epoll)", port);+        loop {+            serve_epoll(lfd, usize::MAX);+        }+    }+    let mut c = Checks::new("echoserver");+    round(&mut c, 7070, true);+    round(&mut c, 7071, false);++    // UDP over loopback: a datagram to ourselves.+    let a = socket(SOCK_DGRAM, 0);+    let b = socket(SOCK_DGRAM, 0);+    bind(b, 7072);+    c.check(sendto(a, b"ping", LOOPBACK, 7072) == 4, "UDP sendto 127.0.0.1");+    set_recv_timeout(b, 2000);+    let mut buf = [0u8; 16];+    let (n, from) = recvfrom(b, &mut buf);+    c.check(n == 4 && &buf[..4] == b"ping" && from.ip() == LOOPBACK, "UDP datagram arrives, from 127.0.0.1");+    close(a);+    close(b);+    c.done()+}

@@ -1,14 +1,22 @@ //! getty [-1] DEVICE -- run a shell on another terminal, e.g. /dev/hvc0. //! //! Real Unix systems run one "getty" per terminal line: it opens the line-//! and starts a login shell on it. Ours skips the login: it opens DEVICE-//! and spawns `sh` with stdin, stdout and stderr all on it. When the shell-//! exits (`exit`) a new one starts, unless `-1` was given.+//! and starts a login shell on it. Ours skips the login. Since stage 15 it+//! does it the classic Unix way, with fork + exec:+//!+//!   1. fork: the child will become the shell.+//!   2. setsid (in the child): a new *session* with no terminal yet.+//!   3. open DEVICE: a session leader without a terminal that opens one+//!      gets it as its *controlling terminal*. That's what lets ^C on+//!      that terminal send SIGINT to the right programs.+//!   4. dup2 it onto fds 0, 1 and 2 (stdin, stdout, stderr).+//!   5. execve the shell: it keeps the fds and the session.+//!+//! getty waits for the shell; when it exits (`exit`) a new one starts,+//! unless `-1` was given. //! //! Try it: `getty /dev/hvc0 &` here, then on the host-//! `socat - UNIX-CONNECT:target/hvc0.sock` and type commands. That shell-//! is not "interactive" (no line editing): it reads one line at a time and-//! prints each line after the prompt, like a script.+//! `socat - UNIX-CONNECT:target/hvc0.sock` and type commands. #![no_std] #![no_main] @@ -20,6 +28,27 @@ use ulib::*;  const SHELL: &str = "/mnt/disk0/bin/sh"; +/// The child's side (steps 2-5). Only returns on failure.+fn become_shell(dev: &str) -> i64 {+    setsid();+    let fd = open(dev, O_RDWR); // becomes our controlling terminal+    if fd < 0 {+        eprintln!("getty: {}: {}", dev, strerror(fd));+        return 126;+    }+    for target in 0..3 {+        dup2(fd, target);+    }+    if fd > 2 {+        close(fd);+    }+    let greeting = "\nRustbelt getty: shell on this terminal (type `exit` to end it)\n";+    write(1, greeting.as_bytes());+    let err = execve(SHELL, &["sh"], &environ());+    eprintln!("getty: {}: {}", SHELL, strerror(err));+    127+}+ fn main() -> i64 {     let (once, dev) = match (arg(1), arg(2)) {         (Some("-1"), Some(dev)) => (true, dev),@@ -30,24 +59,17 @@ fn main() -> i64 {         }     };     loop {-        let fd = open(dev, 0);-        if fd < 0 {-            eprintln!("getty: {}: {}", dev, strerror(fd));-            return 1;+        let pid = fork();+        if pid == 0 {+            exit(become_shell(dev));         }-        let greeting = "\nRustbelt getty: shell on this terminal (type `exit` to end it)\n";-        write(fd, greeting.as_bytes());-        let opts = SpawnOptions { stdin: fd, stdout: fd, stderr: fd, flags: 0 };-        let pid = spawn(SHELL, "sh", &opts);         if pid < 0 {-            eprintln!("getty: {}: {}", SHELL, strerror(pid));-            close(fd);+            eprintln!("getty: fork: {}", strerror(pid));             return 1;         }         let status = wait(pid);-        close(fd);-        if once {-            return status;+        if once || status == 126 || status == 127 {+            return status; // done, or the terminal/shell can't be opened         }         sleep_ms(500); // don't spin if the shell keeps dying at once     }

@@ -0,0 +1,117 @@+//! nbtest -- checks O_NONBLOCK: "would block" becomes EAGAIN instead.+//!+//! A blocking read on an empty pipe sleeps until data comes. With+//! O_NONBLOCK (set with fcntl(F_SETFL), ioctl(FIONBIO), pipe2 or+//! SOCK_NONBLOCK) it returns -EAGAIN at once ("try again later"), which+//! is what an event loop wants: it only reads when poll said "ready", and+//! must never get stuck on one fd.+#![no_std]+#![no_main]++#[macro_use]+#[path = "ulib.rs"]+mod ulib;++extern crate alloc;++use ulib::*;++const FIONBIO: u64 = 0x5421;++fn main() -> i64 {+    let mut c = Checks::new("nbtest");+    let mut buf = [0u8; 4096];++    // Pipes.+    let (r, w) = pipe2(O_NONBLOCK).unwrap();+    c.check(read(r, &mut buf) == -EAGAIN, "pipe: read with nothing there: EAGAIN");+    let mut p = [PollFd { fd: r as i32, events: POLLIN, revents: 0 }];+    c.check(poll(&mut p, 0) == 0, "pipe: poll says not readable");+    let mut total = 0;+    loop {+        let n = write(w, &buf);+        if n < 0 {+            c.check(n == -EAGAIN, "pipe: write to a full pipe: EAGAIN");+            break;+        }+        total += n;+    }+    println!("nbtest: the pipe took {} bytes before it was full", total);+    c.check(total == 65536, "pipe: holds 64 KiB, like Linux");+    let mut p = [PollFd { fd: w as i32, events: POLLOUT, revents: 0 }];+    c.check(poll(&mut p, 0) == 0, "pipe: poll says a full pipe isn't writable");+    c.check(read(r, &mut buf) == 4096, "pipe: then reading works");+    c.check(poll(&mut p, 0) == 1 && p[0].revents & POLLOUT != 0, "pipe: room again -> POLLOUT");+    close(w);+    while read(r, &mut buf) > 0 {}+    c.check(read(r, &mut buf) == 0, "pipe: writer gone -> end of file, not EAGAIN");+    close(r);++    // fcntl and FIONBIO switch it on and off.+    let (r, w) = pipe().unwrap();+    c.check(fcntl(r, F_GETFL, 0) as u64 & O_NONBLOCK == 0, "fcntl F_GETFL: blocking by default");+    set_nonblocking(r, true);+    c.check(fcntl(r, F_GETFL, 0) as u64 & O_NONBLOCK != 0, "fcntl F_SETFL O_NONBLOCK sticks");+    c.check(read(r, &mut buf) == -EAGAIN, "fcntl-made nonblocking pipe: EAGAIN");+    let off = 0i32;+    ioctl(r, FIONBIO, &off as *const i32 as u64);+    c.check(fcntl(r, F_GETFL, 0) as u64 & O_NONBLOCK == 0, "ioctl FIONBIO 0 turns it off");+    close(r);+    close(w);++    // Sockets (over loopback).+    let l = socket(SOCK_STREAM, 0);+    bind(l, 7080);+    listen(l, 2);+    set_nonblocking(l, true);+    c.check(accept(l).0 == -EAGAIN, "socket: accept with nobody calling: EAGAIN");+    let s = socket(SOCK_STREAM | O_NONBLOCK, 0);+    let r = connect(s, Ip([127, 0, 0, 1]), 7080);+    c.check(r == 0 || r == -EINPROGRESS, "socket: nonblocking connect: EINPROGRESS (or done at once)");+    let mut p = [PollFd { fd: s as i32, events: POLLOUT, revents: 0 }];+    c.check(poll(&mut p, 2000) == 1 && p[0].revents & POLLOUT != 0, "socket: poll POLLOUT = connected");+    let mut p = [PollFd { fd: l as i32, events: POLLIN, revents: 0 }];+    c.check(poll(&mut p, 2000) == 1, "socket: listener POLLIN = a connection to accept");+    let (a, _) = accept(l);+    c.check(a >= 0, "socket: accept it");+    c.check(read(s, &mut buf) == -EAGAIN, "socket: recv with no data: EAGAIN");+    write(a, b"hi");+    let mut p = [PollFd { fd: s as i32, events: POLLIN, revents: 0 }];+    poll(&mut p, 2000);+    c.check(read(s, &mut buf) == 2, "socket: data after POLLIN");+    close(a);+    close(s);+    close(l);++    // A terminal: the slave side of a pty.+    if let Ok((m, n)) = openpty() {+        let slave = open(&alloc::format!("/dev/pts/{}", n), O_RDWR | O_NOCTTY | O_NONBLOCK);+        c.check(read(slave, &mut buf) == -EAGAIN, "tty: read with no line typed: EAGAIN");+        close(slave);+        close(m);+    } else {+        c.check(false, "openpty");+    }++    // Small extras libc needs: pread/pwrite, writev, clocks, nanosleep.+    let fd = open("/tmp/nbtest.txt", O_CREAT | O_TRUNC | O_RDWR);+    writev(fd, &[b"hello ", b"vectored ", b"world"]);+    pwrite(fd, b"HELLO", 0);+    let n = pread(fd, &mut buf, 6);+    c.check(&buf[..n.max(0) as usize] == b"vectored world", "writev + pread at an offset");+    c.check(lseek(fd, 0, SEEK_CUR) == 20, "pread/pwrite leave the file position alone");+    lseek(fd, 0, SEEK_SET);+    let n = read(fd, &mut buf);+    c.check(&buf[..n.max(0) as usize] == b"HELLO vectored world", "pwrite overwrote the start");+    close(fd);+    unlink("/tmp/nbtest.txt");+    let (secs, _) = clock_gettime(0);+    c.check(secs > 1_600_000_000, "CLOCK_REALTIME is a real date (after 2020)");+    let t0 = now_us();+    nanosleep_us(30_000);+    c.check(now_us() - t0 >= 25_000, "nanosleep 30 ms");+    let mut link = [0u8; 64];+    let n = readlink("/proc/self/exe", &mut link);+    c.check(n > 0 && link[..n as usize].ends_with(b"nbtest"), "readlink /proc/self/exe = this program");+    c.done()+}

@@ -0,0 +1,106 @@+//! ptytest -- checks pseudo-terminals (ptys) and the line discipline.+//!+//! A pty is a pair: the *master* side is held by a terminal emulator (or+//! ssh, tmux, a compositor's terminal window), the *slave* side+//! (/dev/pts/N) looks exactly like a real terminal to the program running+//! on it. Between them sits the line discipline: in *canonical* mode it+//! collects a whole line (with backspace editing) before the program can+//! read it, echoes what's typed, turns ^C into SIGINT; in *raw* mode bytes+//! pass straight through (what editors and games use).+#![no_std]+#![no_main]++#[macro_use]+#[path = "ulib.rs"]+mod ulib;++extern crate alloc;++use alloc::format;+use ulib::*;++/// Read whatever the other side has (waits up to 1 s for the first byte).+fn read_some(fd: i64, buf: &mut [u8]) -> usize {+    let mut p = [PollFd { fd: fd as i32, events: POLLIN, revents: 0 }];+    if poll(&mut p, 1000) <= 0 {+        return 0;+    }+    read(fd, buf).max(0) as usize+}++fn main() -> i64 {+    let mut c = Checks::new("ptytest");+    let (master, n) = match openpty() {+        Ok(x) => x,+        Err(e) => {+            println!("ptytest: openpty: {}", strerror(e));+            return 1;+        }+    };+    let path = format!("/dev/pts/{}", n);+    let slave = open(&path, O_RDWR | O_NOCTTY);+    c.check(slave >= 0, &format!("open {} (the slave side)", path));+    let mut buf = [0u8; 64];++    // Canonical mode: a line is readable only once it's complete.+    write(master, b"hel");+    set_nonblocking(slave, true);+    c.check(read(slave, &mut buf) == -EAGAIN, "canonical: half a line can't be read yet (EAGAIN)");+    write(master, b"lo\x7f\x7fp me\r"); // two backspaces, then Enter ('\r')+    let got = read_some(slave, &mut buf);+    c.check(&buf[..got] == b"help me\n", "canonical: backspace edits the line, Enter ('\\r') ends it as '\\n'");+    let got = read_some(master, &mut buf);+    c.check(got > 0 && buf[..got].ends_with(b"\r\n"), "echo: what's typed comes back to the master");++    // Output processing: the program's '\n' reaches the terminal as "\r\n".+    write(slave, b"out\n");+    let got = read_some(master, &mut buf);+    c.check(&buf[..got] == b"out\r\n", "output: '\\n' becomes \"\\r\\n\" (ONLCR)");++    // Raw mode: every byte at once, no echo, no conversions.+    let saved = tcgetattr(slave).unwrap_or_default();+    let mut raw = saved;+    cfmakeraw(&mut raw);+    c.check(tcsetattr(slave, &raw) == 0, "TCSETS: switch to raw mode");+    write(master, b"x\r\x03");+    let got = read_some(slave, &mut buf);+    c.check(&buf[..got] == b"x\r\x03", "raw: bytes arrive at once, unchanged (even ^C)");+    set_nonblocking(master, true);+    c.check(read(master, &mut buf) == -EAGAIN, "raw: no echo");+    set_nonblocking(master, false);+    tcsetattr(slave, &saved);+    c.check(tcgetattr(slave).map(|t| t.lflag & ICANON != 0).unwrap_or(false), "TCGETS: canonical mode back on");+    // (We keep `slave` open: with no slave open at all, the master reports+    // a hang-up (POLLHUP) instead of waiting for the child to open it.)++    // ^C: a child makes the pty its controlling terminal (setsid + open)+    // and waits; typing ^C on the master must kill it with SIGINT.+    let pid = fork();+    if pid == 0 {+        setsid();+        let fd = open(&path, O_RDWR); // session leader: becomes our ctty+        if fd < 0 {+            exit(3);+        }+        write(fd, b"ready\n");+        loop {+            pause();+        }+    }+    let got = read_some(master, &mut buf); // "ready\r\n"+    c.check(&buf[..got] == b"ready\r\n", "child opened the pty as its controlling terminal");+    write(master, b"\x03");+    let status = match wait4(pid, 0) {+        Ok((_, s)) => s,+        Err(_) => 0,+    };+    c.check(wifsignaled(status) && wtermsig(status) == SIGINT, "^C on the master: SIGINT for the foreground job");+    let got = read_some(master, &mut buf);+    c.check(got >= 2 && &buf[..2] == b"^C", "^C is echoed as \"^C\"");++    // Hang-up: closing the master gives the slave end-of-file.+    close(master);+    c.check(read(slave, &mut buf) == 0, "master closed: the slave reads end-of-file");+    close(slave);+    c.done()+}

@@ -212,7 +212,10 @@ impl Shell {         let mut line: Vec<u8> = Vec::new();         let mut hist = self.history.len(); // position while browsing history         let mut esc = 0; // 0: normal, 1: after ESC, 2: after ESC [-        let mut buf = [0u8; 16];+        // One byte per read: if several lines were typed (or pasted) ahead,+        // the rest stays in the tty for the next prompt instead of being+        // lost when we stop at the first Enter.+        let mut buf = [0u8; 1];         'read: loop {             let n = read(0, &mut buf);             if n <= 0 {

@@ -0,0 +1,135 @@+//! threads [N] -- several threads in one process, pthread-style.+//!+//! Threads share *everything* a process owns: memory, open files, signal+//! handlers. Each has only its own registers, stack and TLS block (and a+//! thread id). ulib's `spawn_thread` = pthread_create: clone with+//! CLONE_VM | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | ..., a fresh+//! stack, CLONE_SETTLS for the TLS block, and CLONE_CHILD_CLEARTID so+//! `join` can sleep on a futex until the kernel says "thread gone".+//!+//! The demo: a parallel sum. 1..=COUNT is split into N slices (default 4,+//! one per CPU); each thread adds up its slice, then adds its result to a+//! shared total under a futex mutex. Many small increments of a shared+//! counter (also under the mutex) check the mutex really excludes; a+//! counter bumped *without* it shows why that matters.+#![no_std]+#![no_main]++#[macro_use]+#[path = "ulib.rs"]+mod ulib;++extern crate alloc;++use alloc::sync::Arc;+use alloc::vec::Vec;+use core::cell::UnsafeCell;+use core::sync::atomic::{AtomicU32, AtomicU64, Ordering};+use ulib::*;++const COUNT: u64 = 40_000_000;+const BUMPS: u64 = 20_000;++/// Data guarded by a FutexMutex (a tiny `Mutex<T>`).+struct Shared {+    lock: FutexMutex,+    total: UnsafeCell<u64>,+    bumps: UnsafeCell<u64>,+    /// Bumped with a plain read + write, no lock: races lose updates.+    racy: UnsafeCell<u64>,+}+unsafe impl Sync for Shared {}+unsafe impl Send for Shared {}++fn main() -> i64 {+    let n: u64 = arg(1).and_then(|a| a.parse().ok()).unwrap_or(4).clamp(1, 16);+    let mut c = Checks::new("threads");+    let shared = Arc::new(Shared {+        lock: FutexMutex::new(),+        total: UnsafeCell::new(0),+        bumps: UnsafeCell::new(0),+        racy: UnsafeCell::new(0),+    });+    let cpus_seen = Arc::new(AtomicU64::new(0)); // bit i = some thread ran on CPU i+    let tids = Arc::new(AtomicU64::new(0)); // sum of the threads' tids+    let tls_ok = Arc::new(AtomicU32::new(0));+    let start = now_us();+    let mut threads = Vec::new();+    for i in 0..n {+        let (shared, cpus_seen, tids, tls_ok) = (shared.clone(), cpus_seen.clone(), tids.clone(), tls_ok.clone());+        let from = COUNT * i / n + 1;+        let to = COUNT * (i + 1) / n;+        // Each thread gets `i` as its TLS value.+        let t = spawn_thread(i, move || {+            let mut sum = 0u64;+            for x in from..=to {+                sum = sum.wrapping_add(core::hint::black_box(x));+            }+            cpus_seen.fetch_or(1 << sysinfo().this_cpu, Ordering::Relaxed);+            tids.fetch_add(gettid() as u64, Ordering::Relaxed);+            if tls_value() == i {+                tls_ok.fetch_add(1, Ordering::Relaxed);+            }+            for _ in 0..BUMPS {+                shared.lock.lock();+                unsafe { *shared.bumps.get() += 1 };+                shared.lock.unlock();+                unsafe {+                    let v = core::ptr::read_volatile(shared.racy.get());+                    core::ptr::write_volatile(shared.racy.get(), v + 1);+                }+            }+            shared.lock.lock();+            unsafe { *shared.total.get() += sum };+            shared.lock.unlock();+        });+        match t {+            Ok(t) => threads.push(t),+            Err(e) => {+                println!("threads: spawn_thread: {}", strerror(e));+                return 1;+            }+        }+    }+    let spawned_tids: u64 = threads.iter().map(|t| t.tid as u64).sum();+    for t in threads {+        t.join();+    }+    let ms = (now_us() - start) / 1000;+    let total = unsafe { *shared.total.get() };+    let bumps = unsafe { *shared.bumps.get() };+    let racy = unsafe { *shared.racy.get() };+    let cpus = cpus_seen.load(Ordering::Relaxed).count_ones();+    println!("threads: {} threads summed 1..={} = {} in {} ms on {} CPU(s)", n, COUNT, total, ms, cpus);+    println!("threads: locked counter {} (want {}), unlocked counter {} (races lose updates)", bumps, n * BUMPS, racy);+    c.check(total == COUNT * (COUNT + 1) / 2, "parallel sum is right");+    c.check(bumps == n * BUMPS, "the futex mutex never let two threads in at once");+    c.check(tls_ok.load(Ordering::Relaxed) == n as u32, "every thread saw its own TLS value (fs:[8])");+    c.check(tids.load(Ordering::Relaxed) == spawned_tids, "gettid in each thread = the tid clone returned");+    c.check(gettid() == getpid(), "the main thread's tid is the pid");++    // A futex wait whose value doesn't match returns at once (EAGAIN); one+    // that matches sleeps until its timeout.+    let word = AtomicU32::new(7);+    c.check(futex_wait(&word, 8, None) == -EAGAIN, "futex_wait with a stale value: EAGAIN");+    let t0 = now_us();+    let r = futex_wait(&word, 7, Some(50_000));+    c.check(r == -ETIMEDOUT && now_us() - t0 >= 40_000, "futex_wait times out after 50 ms");+    c.check(futex_wake(&word, 1) == 0, "futex_wake with nobody waiting wakes 0");++    // exit vs exit_group: a thread that calls exit (60) ends only itself.+    let alive = Arc::new(AtomicU32::new(0));+    let a2 = alive.clone();+    let t = spawn_thread(0, move || {+        a2.store(1, Ordering::SeqCst);+        // falls off the end: ulib calls exit(0) = just this thread+    })+    .unwrap();+    t.join();+    c.check(alive.load(Ordering::SeqCst) == 1, "a thread's exit ends only that thread");++    // tgkill: a signal for one particular thread (here: ourselves).+    c.check(tgkill(getpid(), gettid(), 0) == 0, "tgkill(pid, tid, 0) finds the thread");+    c.check(tgkill(getpid(), 99999, 0) == -ESRCH, "tgkill to a missing thread: ESRCH");+    c.done()+}

@@ -29,7 +29,7 @@ pub const SYS_LSEEK: u64 = 8; pub const SYS_IOCTL: u64 = 16; pub const SYS_PIPE: u64 = 22; pub const SYS_SCHED_YIELD: u64 = 24;-pub const SYS_SLEEP_MS: u64 = 35;+pub const SYS_NANOSLEEP: u64 = 35; pub const SYS_GETPID: u64 = 39; pub const SYS_EXIT: u64 = 60; pub const SYS_WAIT4: u64 = 61;@@ -227,7 +227,12 @@ pub fn sched_yield() {     syscall3(SYS_SCHED_YIELD, 0, 0, 0); } pub fn sleep_ms(ms: u64) {-    syscall3(SYS_SLEEP_MS, ms, 0, 0);+    nanosleep_us(ms * 1000);+}+/// nanosleep(*timespec, *remaining): 0, or -EINTR if a signal woke us.+pub fn nanosleep_us(us: u64) -> i64 {+    let ts = [us / 1_000_000, (us % 1_000_000) * 1000];+    syscall3(SYS_NANOSLEEP, ts.as_ptr() as u64, 0, 0) } pub fn getpid() -> i64 {     syscall3(SYS_GETPID, 0, 0, 0)@@ -590,9 +595,11 @@ pub fn sysinfo() -> SysInfo {     syscall3(SYS_SYSINFO, &mut info as *mut SysInfo as u64, 0, 0);     info }-/// Current directory into `buf`; returns its length (or -errno).+/// Current directory into `buf`; returns its length (or -errno). The+/// kernel, like Linux, also writes a NUL and counts it; we don't. pub fn getcwd(buf: &mut [u8]) -> i64 {-    syscall3(SYS_GETCWD, buf.as_mut_ptr() as u64, buf.len() as u64, 0)+    let n = syscall3(SYS_GETCWD, buf.as_mut_ptr() as u64, buf.len() as u64, 0);+    if n > 0 { n - 1 } else { n } } pub fn chdir(path: &str) -> i64 {     syscall3(SYS_CHDIR, path.as_ptr() as u64, path.len() as u64, 0)@@ -1068,8 +1075,9 @@ fn panic(info: &core::panic::PanicInfo) -> ! { // up gives us more (zeroed, demand-paged) memory. // Same idea as the kernel's fixed-size-block allocator: sizes are rounded up // to a power of two (16 B .. 1 MiB) and freed blocks go on one free list-// per size, to be reused by the next allocation of that size. Programs are-// single-threaded, so no lock.+// per size, to be reused by the next allocation of that size. Programs may+// have several threads (stage 15), so a tiny spinlock guards the heap; a+// thread that finds it taken yields its CPU instead of spinning long.  const CLASSES: usize = 17; // 16 << 16 = 1 MiB @@ -1078,9 +1086,21 @@ struct Heap {     used: UnsafeCell<u64>,  // bump pointer: end of what's handed out     end: UnsafeCell<u64>,   // the current break     free: UnsafeCell<[usize; CLASSES]>, // free list heads (addresses, 0 = empty)+    lock: AtomicBool, } -unsafe impl Sync for Heap {} // single-threaded programs+unsafe impl Sync for Heap {} // the fields are only touched under `lock`++impl Heap {+    fn lock(&self) {+        while self.lock.swap(true, Ordering::Acquire) {+            sched_yield();+        }+    }+    fn unlock(&self) {+        self.lock.store(false, Ordering::Release);+    }+}  #[global_allocator] static HEAP: Heap = Heap {@@ -1088,6 +1108,7 @@ static HEAP: Heap = Heap {     used: UnsafeCell::new(0),     end: UnsafeCell::new(0),     free: UnsafeCell::new([0; CLASSES]),+    lock: AtomicBool::new(false), };  /// Size class for a layout: 0 = 16 bytes, 1 = 32, ...@@ -1098,6 +1119,26 @@ fn class_of(layout: Layout) -> usize {  unsafe impl GlobalAlloc for Heap {     unsafe fn alloc(&self, layout: Layout) -> *mut u8 {+        self.lock();+        let p = unsafe { self.alloc_locked(layout) };+        self.unlock();+        p+    }++    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {+        let class = class_of(layout);+        self.lock();+        unsafe {+            let free = &mut *self.free.get();+            *(ptr as *mut usize) = free[class];+            free[class] = ptr as usize;+        }+        self.unlock();+    }+}++impl Heap {+    unsafe fn alloc_locked(&self, layout: Layout) -> *mut u8 {         let class = class_of(layout);         if class >= CLASSES {             return core::ptr::null_mut(); // too big: "out of memory"@@ -1133,14 +1174,347 @@ unsafe impl GlobalAlloc for Heap {             start as *mut u8         }     }+} -    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {-        let class = class_of(layout);-        unsafe {-            let free = &mut *self.free.get();-            *(ptr as *mut usize) = free[class];-            free[class] = ptr as usize;+// --- Stage 15: what a libc's plumbing needs ----------------------------------++pub const SYS_POLL: u64 = 7;+pub const SYS_PREAD64: u64 = 17;+pub const SYS_PWRITE64: u64 = 18;+pub const SYS_READV: u64 = 19;+pub const SYS_WRITEV: u64 = 20;+pub const SYS_CLONE: u64 = 56;+pub const SYS_EXIT_THREAD: u64 = 60; // Linux's "exit": just this thread+pub const SYS_SETSID: u64 = 112;+pub const SYS_ARCH_PRCTL: u64 = 158;+pub const SYS_GETTID: u64 = 186;+pub const SYS_FUTEX: u64 = 202;+pub const SYS_EPOLL_WAIT: u64 = 232;+pub const SYS_EPOLL_CTL: u64 = 233;+pub const SYS_TGKILL: u64 = 234;+pub const SYS_EPOLL_CREATE1: u64 = 291;++pub const O_NOCTTY: u64 = 0o400;+pub const O_NONBLOCK: u64 = 0o4000;+pub const O_RDWR: u64 = 2;+pub const F_GETFL: u64 = 3;+pub const F_SETFL: u64 = 4;+pub const EINPROGRESS: i64 = 115;++/// Turn O_NONBLOCK on or off for `fd` (fcntl F_GETFL + F_SETFL).+pub fn set_nonblocking(fd: i64, on: bool) -> i64 {+    let flags = fcntl(fd, F_GETFL, 0);+    if flags < 0 {+        return flags;+    }+    let flags = if on { flags as u64 | O_NONBLOCK } else { flags as u64 & !O_NONBLOCK };+    fcntl(fd, F_SETFL, flags)+}++pub fn pread(fd: i64, buf: &mut [u8], offset: u64) -> i64 {+    syscall6(SYS_PREAD64, fd as u64, buf.as_mut_ptr() as u64, buf.len() as u64, offset, 0, 0)+}+pub fn pwrite(fd: i64, buf: &[u8], offset: u64) -> i64 {+    syscall6(SYS_PWRITE64, fd as u64, buf.as_ptr() as u64, buf.len() as u64, offset, 0, 0)+}+/// writev: several buffers in one call (`struct iovec {base, len}`).+pub fn writev(fd: i64, bufs: &[&[u8]]) -> i64 {+    let iov: alloc::vec::Vec<[u64; 2]> = bufs.iter().map(|b| [b.as_ptr() as u64, b.len() as u64]).collect();+    syscall3(SYS_WRITEV, fd as u64, iov.as_ptr() as u64, iov.len() as u64)+}++/// Linux 228: 0 = CLOCK_REALTIME (wall clock), 1 = CLOCK_MONOTONIC.+pub fn clock_gettime(clock: u64) -> (u64, u64) {+    let mut ts = [0u64; 2];+    syscall3(SYS_CLOCK_GETTIME, clock, ts.as_mut_ptr() as u64, 0);+    (ts[0], ts[1])+}++// poll / epoll (see the kernel's poll.rs).+pub const POLLIN: u16 = 0x1;+pub const POLLOUT: u16 = 0x4;+pub const POLLERR: u16 = 0x8;+pub const POLLHUP: u16 = 0x10;+pub const POLLNVAL: u16 = 0x20;++/// `struct pollfd`.+#[repr(C)]+#[derive(Clone, Copy, Default)]+pub struct PollFd {+    pub fd: i32,+    pub events: u16,+    pub revents: u16,+}+/// Wait until one of `fds` is ready (timeout_ms -1 = forever); returns+/// how many are, 0 on timeout.+pub fn poll(fds: &mut [PollFd], timeout_ms: i64) -> i64 {+    syscall3(SYS_POLL, fds.as_mut_ptr() as u64, fds.len() as u64, timeout_ms as u64)+}++pub const EPOLL_CTL_ADD: u64 = 1;+pub const EPOLL_CTL_DEL: u64 = 2;+pub const EPOLL_CTL_MOD: u64 = 3;+pub const EPOLLIN: u32 = 0x1;+pub const EPOLLOUT: u32 = 0x4;++/// `struct epoll_event` is *packed* on x86_64: 4 + 8 = 12 bytes.+#[repr(C, packed)]+#[derive(Clone, Copy, Default)]+pub struct EpollEvent {+    pub events: u32,+    pub data: u64,+}+pub fn epoll_create1(flags: u64) -> i64 {+    syscall3(SYS_EPOLL_CREATE1, flags, 0, 0)+}+pub fn epoll_ctl(epfd: i64, op: u64, fd: i64, events: u32, data: u64) -> i64 {+    let ev = EpollEvent { events, data };+    syscall5(SYS_EPOLL_CTL, epfd as u64, op, fd as u64, &ev as *const EpollEvent as u64, 0)+}+pub fn epoll_wait(epfd: i64, out: &mut [EpollEvent], timeout_ms: i64) -> i64 {+    syscall5(SYS_EPOLL_WAIT, epfd as u64, out.as_mut_ptr() as u64, out.len() as u64, timeout_ms as u64, 0)+}++// Sessions and terminals.+pub fn setsid() -> i64 {+    syscall3(SYS_SETSID, 0, 0, 0)+}+pub const TCGETS: u64 = 0x5401;+pub const TCSETS: u64 = 0x5402;+pub const TIOCSCTTY: u64 = 0x540E;+pub const TIOCGPTN: u64 = 0x8004_5430;+pub const TIOCSPTLCK: u64 = 0x4004_5431;+pub const ICANON: u32 = 0x2;+pub const ECHO: u32 = 0x8;+pub const ISIG: u32 = 0x1;+pub const VMIN: usize = 6;+pub const VTIME: usize = 5;++/// Linux's `struct termios` (36 bytes): the terminal's settings.+#[repr(C)]+#[derive(Clone, Copy, Default)]+pub struct Termios {+    pub iflag: u32,+    pub oflag: u32,+    pub cflag: u32,+    pub lflag: u32,+    pub line: u8,+    pub cc: [u8; 19],+}+pub fn tcgetattr(fd: i64) -> Result<Termios, i64> {+    let mut t = Termios::default();+    match ioctl(fd, TCGETS, &mut t as *mut Termios as u64) {+        0 => Ok(t),+        e => Err(e),+    }+}+pub fn tcsetattr(fd: i64, t: &Termios) -> i64 {+    ioctl(fd, TCSETS, t as *const Termios as u64)+}+/// Raw mode, like libc's cfmakeraw: no line editing, echo or signals,+/// no character conversions; read returns as soon as 1 byte is there.+pub fn cfmakeraw(t: &mut Termios) {+    t.iflag &= !(0x1 | 0x2 | 0x8 | 0x20 | 0x40 | 0x80 | 0x100 | 0x400); // IGNBRK BRKINT PARMRK ISTRIP INLCR IGNCR ICRNL IXON+    t.oflag &= !0x1; // OPOST+    t.lflag &= !(ECHO | 0x40 | ICANON | ISIG | 0x8000); // ECHO ECHONL ICANON ISIG IEXTEN+    t.cflag = (t.cflag & !0x130) | 0x30; // CS8, no parity+    t.cc[VMIN] = 1;+    t.cc[VTIME] = 0;+}++/// Open a new pseudo-terminal: returns (master fd, slave number N); the+/// slave is /dev/pts/N. Like posix_openpt + grantpt + unlockpt.+pub fn openpty() -> Result<(i64, u32), i64> {+    let master = open("/dev/ptmx", O_RDWR | O_NOCTTY);+    if master < 0 {+        return Err(master);+    }+    let mut n = 0u32;+    let unlock = 0u32;+    let r = ioctl(master, TIOCGPTN, &mut n as *mut u32 as u64);+    let r = if r == 0 { ioctl(master, TIOCSPTLCK, &unlock as *const u32 as u64) } else { r };+    if r != 0 {+        close(master);+        return Err(r);+    }+    Ok((master, n))+}++// Threads: clone + futex + TLS.+pub const FUTEX_WAIT: u64 = 0;+pub const FUTEX_WAKE: u64 = 1;+pub const FUTEX_PRIVATE_FLAG: u64 = 128;+pub const CLONE_VM: u64 = 0x100;+pub const CLONE_FS: u64 = 0x200;+pub const CLONE_FILES: u64 = 0x400;+pub const CLONE_SIGHAND: u64 = 0x800;+pub const CLONE_THREAD: u64 = 0x10000;+pub const CLONE_SYSVSEM: u64 = 0x40000;+pub const CLONE_SETTLS: u64 = 0x80000;+pub const CLONE_PARENT_SETTID: u64 = 0x100000;+pub const CLONE_CHILD_CLEARTID: u64 = 0x200000;+pub const ARCH_SET_FS: u64 = 0x1002;+pub const ARCH_GET_FS: u64 = 0x1003;++use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};++pub fn gettid() -> i64 {+    syscall3(SYS_GETTID, 0, 0, 0)+}+pub fn tgkill(tgid: i64, tid: i64, sig: i32) -> i64 {+    syscall3(SYS_TGKILL, tgid as u64, tid as u64, sig as u64)+}+/// Sleep while `*word == expected` (until futex_wake, or `timeout_us`).+pub fn futex_wait(word: &AtomicU32, expected: u32, timeout_us: Option<u64>) -> i64 {+    let ts = timeout_us.map(|us| [us / 1_000_000, (us % 1_000_000) * 1000]);+    let ts_ptr = ts.as_ref().map_or(0, |t| t.as_ptr() as u64);+    syscall6(SYS_FUTEX, word.as_ptr() as u64, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, expected as u64, ts_ptr, 0, 0)+}+/// Wake up to `n` threads sleeping on `word`; returns how many woke.+pub fn futex_wake(word: &AtomicU32, n: u32) -> i64 {+    syscall6(SYS_FUTEX, word.as_ptr() as u64, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, n as u64, 0, 0, 0)+}++/// A mutex built on a futex (Ulrich Drepper's "Futexes Are Tricky",+/// mutex #2). The word is 0 = unlocked, 1 = locked, 2 = locked and+/// someone may be sleeping. Taking a free lock is one atomic instruction,+/// no syscall; only contention goes to the kernel.+pub struct FutexMutex {+    state: AtomicU32,+}+impl FutexMutex {+    pub const fn new() -> FutexMutex {+        FutexMutex { state: AtomicU32::new(0) }+    }+    pub fn lock(&self) {+        if self.state.compare_exchange(0, 1, Ordering::Acquire, Ordering::Relaxed).is_ok() {+            return; // fast path: it was free+        }+        // Slow path: mark "contended" (2) and sleep until it's free. The+        // kernel only puts us to sleep if the word is *still* 2, so an+        // unlock between our swap and the futex call can't be missed.+        while self.state.swap(2, Ordering::Acquire) != 0 {+            futex_wait(&self.state, 2, None);+        }+    }+    pub fn unlock(&self) {+        // 1 -> 0: nobody waits. 2 -> 0: wake one sleeper.+        if self.state.swap(0, Ordering::Release) == 2 {+            futex_wake(&self.state, 1);+        }+    }+}++/// Thread-local storage, the x86_64 way: each thread's FS register base+/// points at its own block, whose first word points at itself (so code can+/// find the block with `mov rax, fs:[0]`). We keep one u64 per thread in+/// the second word.+#[repr(C)]+pub struct TlsBlock {+    pub self_ptr: u64,+    pub value: u64,+}+pub fn arch_prctl(code: u64, addr: u64) -> i64 {+    syscall3(SYS_ARCH_PRCTL, code, addr, 0)+}+/// This thread's TLS value (fs:[8]).+pub fn tls_value() -> u64 {+    let v: u64;+    unsafe { asm!("mov {}, fs:[8]", out(reg) v, options(nostack, readonly)) };+    v+}++/// A running thread (see `spawn_thread`).+pub struct Thread {+    pub tid: i64,+    /// The kernel writes our tid here (CLONE_PARENT_SETTID) and zeroes it ++    /// futex-wakes it when the thread exits (CLONE_CHILD_CLEARTID): that's+    /// how `join` knows it's done, exactly like pthread_join.+    tid_word: alloc::boxed::Box<AtomicU32>,+    stack: u64,+    tls: alloc::boxed::Box<TlsBlock>,+}++const THREAD_STACK: u64 = 64 * 1024;+type ThreadFn = alloc::boxed::Box<dyn FnOnce()>;++/// The new thread starts here, on its own stack, with the boxed closure.+extern "C" fn thread_entry(arg: u64) -> ! {+    let f = unsafe { alloc::boxed::Box::from_raw(arg as *mut ThreadFn) };+    f();+    // exit (60), not exit_group: only this thread ends.+    syscall3(SYS_EXIT_THREAD, 0, 0, 0);+    loop {}+}++/// Start `f` in a new thread of this process (same memory, same fds),+/// with `tls_value` in its TLS block. Like pthread_create.+pub fn spawn_thread(tls_value: u64, f: impl FnOnce() + Send + 'static) -> Result<Thread, i64> {+    let stack = mmap(0, THREAD_STACK, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);+    if stack < 0 {+        return Err(stack);+    }+    let stack = stack as u64;+    let boxed: alloc::boxed::Box<ThreadFn> = alloc::boxed::Box::new(alloc::boxed::Box::new(f));+    let arg = alloc::boxed::Box::into_raw(boxed) as u64;+    let mut tls = alloc::boxed::Box::new(TlsBlock { self_ptr: 0, value: tls_value });+    tls.self_ptr = &*tls as *const TlsBlock as u64;+    let tid_word = alloc::boxed::Box::new(AtomicU32::new(0));+    // The child's stack starts with [entry, arg] for the asm below.+    let sp = stack + THREAD_STACK - 16;+    unsafe {+        *(sp as *mut u64) = thread_entry as usize as u64;+        *((sp + 8) as *mut u64) = arg;+    }+    let flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM+        | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID;+    let tid_ptr = tid_word.as_ptr() as u64;+    let ret: i64;+    // clone returns twice: the parent gets the tid, the child 0 -- on its+    // new stack, so it must not touch the parent's stack frame. It pops+    // [entry, arg] and calls entry(arg) straight from assembly (after the+    // two pops rsp is 16-byte aligned, as `call` wants).+    unsafe {+        asm!(+            "syscall",+            "test rax, rax",+            "jnz 2f",+            "pop rax",+            "pop rdi",+            "call rax",+            "ud2",+            "2:",+            inlateout("rax") SYS_CLONE as i64 => ret,+            in("rdi") flags,+            in("rsi") sp,+            in("rdx") tid_ptr,+            in("r10") tid_ptr,+            in("r8") tls.self_ptr,+            lateout("rcx") _,+            lateout("r11") _,+            options(nostack)+        );+    }+    if ret < 0 {+        unsafe { drop(alloc::boxed::Box::from_raw(arg as *mut ThreadFn)) };+        munmap(stack, THREAD_STACK);+        return Err(ret);+    }+    Ok(Thread { tid: ret, tid_word, stack, tls })+}++impl Thread {+    /// Wait for the thread to finish, then free its stack.+    pub fn join(self) {+        loop {+            let tid = self.tid_word.load(Ordering::Acquire);+            if tid == 0 {+                break; // the kernel cleared it: the thread is gone+            }+            futex_wait(&self.tid_word, tid, None);         }+        munmap(self.stack, THREAD_STACK);+        drop(self.tls);     } }