rust-kernel

//! Processes: a thread + an address space (`Mm`) + a file descriptor table
//! (+ arguments, environment, a current directory, a parent, a process
//! group and signal state).
//!
//! **How Unix makes processes: fork + exec.** There's no "start program X"
//! call. Instead:
//!   * `fork()` duplicates the calling process. Both copies return from the
//!     same fork call: the parent gets the child's pid, the child gets 0.
//!     The child has a copy of everything: memory, open files, cwd, signal
//!     handlers.
//!   * `execve(path, argv, envp)` replaces the *running* program with
//!     another one: new code, new memory, same process (pid, open files,
//!     cwd, parent).
//! Why split it? Because between the two the child is still running the
//! *parent's* code, and can set itself up with all the normal syscalls:
//! `dup2` a pipe onto stdout, `chdir`, `setpgid`, reset signal handlers,
//! close files. No "spawn options" structure has to anticipate every
//! possible setup: a shell does `cmd > file` as fork, open, dup2, exec.
//! Copying all memory just to throw it away at exec would be slow; that's
//! what copy-on-write (mm.rs) is for: fork only copies page tables.
//!
//! `spawn` (our syscall 500 from stage 9, and `spawn_path` for the kernel)
//! stays as a convenience: new process + exec in one step, like
//! `posix_spawn`.
//!
//! **Exit and wait.** A process's exit status is stored in it, its parent
//! gets SIGCHLD and is woken; `wait4` returns the status and removes the
//! child from the process table ("reaps" it -- until then it's a *zombie*).
//! If the parent exits first, the children become orphans (parent =
//! ORPHAN) and remove themselves when they exit; Unix hands them to
//! `init` instead. Since stage 16 we do too when there's someone to hand
//! them to: an ancestor that called prctl(PR_SET_CHILD_SUBREAPER), or a
//! pid 1 named "init" (see `adopter`).
//!
//! **Process groups** (job control): every process is in a group, named by
//! the pid of its first member. A shell puts each pipeline in its own group
//! and tells the terminal which group is in the *foreground*; Ctrl+C then
//! sends SIGINT to that whole group -- every program of the pipeline, but
//! not the shell (tty.rs).
//!
//! **Threads** (stage 15). A process can run several threads: each is a
//! `Task` with its own registers, kernel stack, thread id, TLS pointer and
//! signal mask, but they all share the process's memory, files and signal
//! handlers. `clone(CLONE_VM | CLONE_THREAD | ...)` makes one (that's what
//! pthread_create does). The first thread's id equals the pid. `exit` ends
//! one thread; `exit_group` (what C's exit() calls) ends them all. The
//! process itself ends when its last thread does.
//!
//! **Memory lifetime.** A process's frames and page tables are freed when
//! the last `Arc<Process>` goes away (dropping its `Mm`). That's always
//! after its threads are gone: each holds an Arc (through its Task) until
//! the scheduler frees it, which happens only once the CPU has switched to
//! another thread and address space. So no CPU can still be using the page
//! tables.

use crate::elf::{self, ElfError};
use crate::errno::Errno;
use crate::fpu::FpuState;
use crate::fs::{self, file::File};
use crate::mm::Mm;
use crate::signal::{self, SigInfo, SignalState, ThreadSig};
use crate::thread::{self, WaitQueue};
use crate::trap::TrapFrame;
use crate::serial_println;
use alloc::{collections::BTreeMap, string::String, sync::Arc, vec, vec::Vec};
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use spin::Mutex;
use x86_64::instructions::interrupts::without_interrupts;

pub use crate::mm::{in_user_stack_guard, USER_STACK_GUARD, USER_STACK_SIZE, USER_STACK_TOP, USER_STACK_ZONE_START};

/// `parent` of a process whose parent exited first.
pub const ORPHAN: u64 = u64::MAX;
/// Max open files per process.
pub const MAX_FDS: usize = 64;
/// Max total size of a program's arguments + environment strings.
pub const MAX_ARGS_SIZE: usize = 64 * 1024;
/// Max processes at once: fork() fails with EAGAIN beyond that, which is
/// what stops a fork bomb from eating all memory.
pub const MAX_PROCESSES: usize = 64;
/// Max threads in one process (clone fails with EAGAIN beyond that).
pub const MAX_THREADS: usize = 64;
/// Environment of processes the kernel starts.
pub const DEFAULT_ENV: [&str; 3] = ["PATH=/bin:/mnt/disk0/bin", "HOME=/", "TERM=linux"];

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExitStatus {
    /// Called exit(code).
    Exited(i64),
    /// Killed by signal number .0; .1 says why (for messages): the signal
    /// name, or e.g. "page fault" / "stack overflow" for SIGSEGV.
    Killed(u32, &'static str),
}

impl ExitStatus {
    /// The number a shell shows: the exit code, or 128 + the signal number
    /// (so 139 = 128 + SIGSEGV, 130 = 128 + SIGINT).
    pub fn code(self) -> i64 {
        match self {
            ExitStatus::Exited(code) => code,
            ExitStatus::Killed(sig, _) => 128 + sig as i64,
        }
    }

    /// The `int status` of wait4 (decoded by the W* macros, see ulib.rs):
    /// exit code in bits 8-15, or the killing signal in bits 0-6.
    pub fn wait_status(self) -> i32 {
        match self {
            ExitStatus::Exited(code) => ((code & 0xff) << 8) as i32,
            ExitStatus::Killed(sig, _) => sig as i32,
        }
    }
}

/// Open files: fd number -> file, plus "close on exec" flags.
#[derive(Clone)]
pub struct FdTable {
    files: Vec<Option<Arc<dyn File>>>,
    /// Bit n: close fd n when the process execs (FD_CLOEXEC / O_CLOEXEC).
    cloexec: u64,
    /// RLIMIT_NOFILE's soft limit (stage 16): fds are < this. At most MAX_FDS.
    limit: usize,
}

impl FdTable {
    fn new(stdio: [Arc<dyn File>; 3]) -> FdTable {
        FdTable { files: stdio.into_iter().map(Some).collect(), cloexec: 0, limit: MAX_FDS }
    }

    fn get(&self, fd: u64) -> Result<Arc<dyn File>, Errno> {
        self.files.get(fd as usize).and_then(|f| f.clone()).ok_or(Errno::EBADF)
    }

    /// Put `file` into the lowest free slot >= `min` (Unix's rule).
    fn add(&mut self, file: Arc<dyn File>, min: usize, cloexec: bool) -> Result<u64, Errno> {
        let fd = match (min..self.limit).find(|&i| self.files.get(i).is_none_or(|f| f.is_none())) {
            Some(fd) => fd,
            None => return Err(Errno::EMFILE),
        };
        self.set(fd, file, cloexec);
        Ok(fd as u64)
    }

    /// Put `file` at `fd` (whatever was there is returned, to be closed).
    fn set(&mut self, fd: usize, file: Arc<dyn File>, cloexec: bool) -> Option<Arc<dyn File>> {
        if self.files.len() <= fd {
            self.files.resize(fd + 1, None);
        }
        self.set_cloexec(fd, cloexec);
        self.files[fd].replace(file)
    }

    fn set_cloexec(&mut self, fd: usize, on: bool) {
        if on { self.cloexec |= 1 << fd } else { self.cloexec &= !(1 << fd) }
    }

    /// execve: close the close-on-exec ones. Returns them (close them
    /// after unlocking).
    fn exec(&mut self) -> Vec<Arc<dyn File>> {
        let mut closed = Vec::new();
        for (fd, slot) in self.files.iter_mut().enumerate() {
            if self.cloexec & (1 << fd) != 0 {
                closed.extend(slot.take());
            }
        }
        self.cloexec = 0;
        closed
    }
}

/// One thread of a process (what Linux calls a "task"). The scheduler's
/// kernel thread that runs it holds an `Arc<Task>` (thread.rs).
pub struct Task {
    /// Thread id (gettid). The first thread's is the pid.
    tid: AtomicU64,
    pub process: Arc<Process>,
    /// The scheduler thread running it (set when it starts).
    kthread: AtomicUsize,
    /// Its FS segment base: the thread-local-storage pointer (arch_prctl,
    /// clone's CLONE_SETTLS). Loaded into the CPU on every switch.
    pub fs_base: AtomicU64,
    /// set_tid_address / CLONE_CHILD_CLEARTID: when the thread exits, write
    /// 0 there and futex-wake it (that's how pthread_join waits).
    pub clear_child_tid: AtomicU64,
    /// Registers it starts user mode with (taken when it starts).
    start: Mutex<Option<TrapFrame>>,
    /// Told to exit (another thread called exit_group or execve).
    pub killed: AtomicBool,
    /// Its own signal mask and pending signals. Locked with interrupts off,
    /// after the process's `signals` if both are needed.
    pub sig: Mutex<ThreadSig>,
    /// Its FPU/SSE/AVX registers while it isn't running (fpu.rs). Only the
    /// scheduler touches it, with interrupts off: saved when the thread is
    /// switched out, loaded when it's switched in.
    pub fpu: Mutex<crate::fpu::FpuState>,
    /// Its name, prctl(PR_SET_NAME) (stage 16): up to 15 bytes + NUL.
    /// Starts as the program's name; threads inherit their creator's.
    pub comm: Mutex<[u8; 16]>,
    /// set_robust_list (stage 16): user address of its `struct
    /// robust_list_head` (0 = none). When the thread dies, the kernel walks
    /// that list and marks every futex lock it still held "owner died".
    pub robust_list: AtomicU64,
    /// CPU time it used (stage 17), in timer ticks (`CPU_TICK_US` each):
    /// for CLOCK_THREAD_CPUTIME_ID.
    pub cpu_ticks: AtomicU64,
}

/// One APIC timer tick (apic.rs programs ~100 a second), in microseconds:
/// the unit of CPU-time accounting (stage 17).
pub const CPU_TICK_US: u64 = 10_000;

/// CPU time, Linux style (stage 17): every timer tick is charged to
/// whatever thread it interrupted, as "user" time if the CPU was running
/// the program and "system" time if it was in the kernel for it. That's
/// what Linux does without fine-grained accounting too: a sample, not a
/// stopwatch. Children's times are added to `child_*` when they're reaped
/// with wait4, which is what times() and getrusage(RUSAGE_CHILDREN) show.
#[derive(Default)]
pub struct CpuTimes {
    pub user: AtomicU64,
    pub system: AtomicU64,
    pub child_user: AtomicU64,
    pub child_system: AtomicU64,
}

/// User and group ids (stage 17). Linux keeps three of each: real (who
/// you are), effective (what permission checks use) and saved (what a
/// setuid program may switch back to), plus "fs" ids and the
/// supplementary groups. We store them all and report them truthfully,
/// but nothing checks them yet: permissions aren't enforced.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Creds {
    pub ruid: u32,
    pub euid: u32,
    pub suid: u32,
    pub fsuid: u32,
    pub rgid: u32,
    pub egid: u32,
    pub sgid: u32,
    pub fsgid: u32,
    pub groups: [u32; NGROUPS_MAX],
    pub ngroups: usize,
}

/// How many supplementary groups a process can have (Linux allows 65536;
/// 32 is plenty here).
pub const NGROUPS_MAX: usize = 32;

impl Creds {
    pub const ROOT: Creds = Creds { ruid: 0, euid: 0, suid: 0, fsuid: 0, rgid: 0, egid: 0, sgid: 0, fsgid: 0, groups: [0; NGROUPS_MAX], ngroups: 0 };
}

/// A thread name as prctl(PR_GET_NAME) returns it: at most 15 bytes, then NULs.
pub fn comm_from(name: &str) -> [u8; 16] {
    let mut comm = [0u8; 16];
    let n = name.len().min(15);
    comm[..n].copy_from_slice(&name.as_bytes()[..n]);
    comm
}

impl Task {
    pub fn tid(&self) -> u64 {
        self.tid.load(Ordering::Relaxed)
    }

    /// Wake it if it's blocked (it then checks for signals / being killed).
    pub fn wake(&self) {
        let kthread = self.kthread.load(Ordering::Acquire);
        if kthread != usize::MAX {
            thread::wake(kthread);
        }
    }
}

pub struct Process {
    pub pid: u64,
    /// Parent pid; 0 = started by the kernel itself, ORPHAN = parent exited.
    pub parent: AtomicU64,
    /// Process group (job control; see the top of this file).
    pgid: AtomicU64,
    /// Session (setsid; a login session = the processes of one terminal).
    sid: AtomicU64,
    /// Controlling terminal (tty.rs): what /dev/tty opens, whose Ctrl+C
    /// reaches this process's group.
    pub ctty: Mutex<Option<Arc<crate::tty::Tty>>>,
    /// Path of the running program (/proc/self/exe).
    pub exe: Mutex<String>,
    /// Program name (changes with execve).
    name: Mutex<String>,
    /// Print every syscall this process makes (`run --trace`; survives exec).
    pub trace: AtomicBool,
    /// Current directory (always a normalized absolute path).
    pub cwd: Mutex<String>,
    /// The address space.
    pub mm: Mutex<Mm>,
    /// Its live threads. Locked with interrupts off.
    tasks: Mutex<Vec<Arc<Task>>>,
    /// Scheduler threads that ran its threads and may not be freed yet
    /// (reaping waits for them). Locked with interrupts off.
    kthreads: Mutex<Vec<thread::Tid>>,
    /// exit_group was called (or a fatal signal arrived): every thread
    /// exits, and the process ends with this status. Interrupts off.
    exiting: Mutex<Option<ExitStatus>>,
    /// A thread of it exited (execve waits for the others to go).
    thread_exit: WaitQueue,
    /// CPUs its threads have run on (bit per CPU; kept up by the scheduler).
    pub ran_on: AtomicU32,
    /// The environment it was started with (`spawn` passes it on).
    env: Mutex<Vec<String>>,
    fds: Mutex<FdTable>,
    /// Locked with interrupts off only (it's read in a wait_until condition).
    status: Mutex<Option<ExitStatus>>,
    exited: WaitQueue,
    /// Its own wait4 sleeps here; children wake it when they change state.
    pub child_wait: WaitQueue,
    /// Signal handlers, process-wide pending signals, stop state. Locked
    /// with interrupts off. (Masks are per thread: `Task::sig`.)
    pub signals: Mutex<SignalState>,
    /// vfork: the parent waits until this child execs or exits.
    vfork_done: AtomicBool,
    /// umask (stage 16): permission bits taken away from new files'
    /// modes. Inherited by children, kept across exec.
    pub umask: AtomicU32,
    /// getrlimit/setrlimit/prlimit64 (stage 16): [soft, hard] per resource.
    pub rlimits: Mutex<[[u64; 2]; RLIM_NLIMITS]>,
    /// prctl(PR_SET_PDEATHSIG): signal to get when the parent exits (0 = none).
    pub pdeathsig: AtomicU32,
    /// prctl(PR_SET_DUMPABLE) (remembered only: we never write core files).
    pub dumpable: AtomicU32,
    /// prctl(PR_SET_NO_NEW_PRIVS): once set, never cleared (we have no
    /// setuid programs, so it changes nothing yet).
    pub no_new_privs: AtomicBool,
    /// prctl(PR_SET_CHILD_SUBREAPER) (stage 16): this process adopts its
    /// descendants' orphans and promises to reap them (see `adopter`).
    /// Not inherited.
    pub child_subreaper: AtomicBool,
    /// CPU time used (stage 17), see `CpuTimes`.
    pub cpu: CpuTimes,
    /// User and group ids (stage 17): inherited by fork, kept by exec.
    pub creds: Mutex<Creds>,
    /// The nice value (stage 17), -20..19: stored for get/setpriority,
    /// inherited; the scheduler doesn't use it (every thread is equal).
    pub nice: core::sync::atomic::AtomicI32,
}

// --- Resource limits (stage 16) ------------------------------------------
//
// Linux's RLIMIT_* numbers. Each limit has a soft value (what's enforced)
// and a hard one (the ceiling the soft one may be raised to). We enforce
// RLIMIT_NOFILE (the fd table); the others report what the kernel really
// does (STACK = the fixed stack size, NPROC = the process table) or are
// remembered without being enforced.
pub const RLIMIT_CPU: usize = 0;
pub const RLIMIT_FSIZE: usize = 1;
pub const RLIMIT_DATA: usize = 2;
pub const RLIMIT_STACK: usize = 3;
pub const RLIMIT_CORE: usize = 4;
pub const RLIMIT_NPROC: usize = 6;
pub const RLIMIT_NOFILE: usize = 7;
pub const RLIMIT_MEMLOCK: usize = 8;
pub const RLIMIT_SIGPENDING: usize = 11;
pub const RLIMIT_MSGQUEUE: usize = 12;
pub const RLIMIT_NLIMITS: usize = 16;
pub const RLIM_NLIMITS: usize = RLIMIT_NLIMITS;
pub const RLIM_INFINITY: u64 = u64::MAX;

/// What a process the kernel starts gets (Linux's usual defaults, with our
/// real numbers where the kernel has a fixed limit).
pub fn default_rlimits() -> [[u64; 2]; RLIM_NLIMITS] {
    let mut r = [[RLIM_INFINITY; 2]; RLIM_NLIMITS];
    r[RLIMIT_STACK] = [USER_STACK_SIZE, USER_STACK_SIZE];
    r[RLIMIT_CORE] = [0, RLIM_INFINITY]; // no core files
    r[RLIMIT_NPROC] = [MAX_PROCESSES as u64; 2];
    r[RLIMIT_NOFILE] = [MAX_FDS as u64; 2];
    r[RLIMIT_MEMLOCK] = [8 << 20; 2];
    r[RLIMIT_SIGPENDING] = [1024; 2];
    r[RLIMIT_MSGQUEUE] = [819_200; 2];
    r[13] = [0, 0]; // RLIMIT_NICE
    r[14] = [0, 0]; // RLIMIT_RTPRIO
    r
}

/// The most a hard limit can be raised to (EPERM above): where the kernel
/// has a fixed size, that.
fn rlimit_ceiling(resource: usize) -> u64 {
    match resource {
        RLIMIT_NOFILE => MAX_FDS as u64,
        RLIMIT_STACK => USER_STACK_SIZE,
        RLIMIT_NPROC => MAX_PROCESSES as u64,
        _ => RLIM_INFINITY,
    }
}

static NEXT_PID: AtomicU64 = AtomicU64::new(1);
/// All processes that haven't been reaped yet, by pid. Locked with
/// interrupts off only (exit_current takes it with interrupts off). Lock
/// order: this table before any process's `signals`.
static PROCESSES: Mutex<BTreeMap<u64, Arc<Process>>> = Mutex::new(BTreeMap::new());
/// vfork parents wait here.
static VFORK_WAIT: WaitQueue = WaitQueue::new();

/// Everything needed to start a process besides the program itself.
pub struct SpawnArgs {
    pub args: Vec<String>, // argv; args[0] is the program name by convention
    pub env: Vec<String>,
    pub stdio: [Arc<dyn File>; 3],
    pub cwd: String,
    pub parent: u64,
    /// Process group: 0 = a new group of its own.
    pub pgid: u64,
    /// Session: 0 = a new session of its own.
    pub sid: u64,
    /// Controlling terminal.
    pub ctty: Option<Arc<crate::tty::Tty>>,
    pub trace: bool,
}

impl SpawnArgs {
    /// Defaults for a process started by the kernel: console stdio, cwd "/".
    pub fn kernel(name: &str) -> SpawnArgs {
        let console: Arc<dyn File> = crate::tty::console_file();
        SpawnArgs {
            args: vec![String::from(name)],
            env: DEFAULT_ENV.iter().map(|s| String::from(*s)).collect(),
            stdio: [console.clone(), console.clone(), console],
            cwd: String::from("/"),
            parent: 0,
            pgid: 0,
            sid: 0,
            ctty: Some(crate::tty::console()),
            trace: false,
        }
    }
}

fn elf_errno(e: ElfError) -> Errno {
    match e {
        ElfError::OutOfMemory => Errno::ENOMEM,
        _ => Errno::ENOEXEC, // "Exec format error"
    }
}

/// Load the program at `path` (absolute) from the VFS and start it.
pub fn spawn_path(path: &str, spawn: SpawnArgs) -> Result<Arc<Process>, Errno> {
    let image = fs::read_all(path)?; // ENOENT, EISDIR, EIO, ...
    if count() >= MAX_PROCESSES {
        return Err(Errno::EAGAIN);
    }
    spawn_image(path, &image, spawn).map_err(elf_errno)
}

/// Start ELF program `elf_image` from the kernel (console stdio, no args).
pub fn spawn(name: &str, elf_image: &[u8]) -> Result<Arc<Process>, ElfError> {
    spawn_image(name, elf_image, SpawnArgs::kernel(name))
}

/// Load `elf_image` into a new address space and start it. `path` is
/// where it came from (AT_EXECFN); the process is named after its last part.
pub fn spawn_image(path: &str, elf_image: &[u8], spawn: SpawnArgs) -> Result<Arc<Process>, ElfError> {
    let name = path.rsplit('/').next().unwrap_or(path);
    let mut mm = Mm::new().ok_or(ElfError::OutOfMemory)?;
    // (On error, dropping `mm` gives its memory back.)
    let args: Vec<&[u8]> = spawn.args.iter().map(|a| a.as_bytes()).collect();
    let env: Vec<&[u8]> = spawn.env.iter().map(|a| a.as_bytes()).collect();
    let start = load_program(&mut mm, elf_image, &args, &env, path)?;
    let [stdin, stdout, stderr] = spawn.stdio;
    let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed);
    let process = Process {
        pid,
        parent: AtomicU64::new(spawn.parent),
        pgid: AtomicU64::new(if spawn.pgid == 0 { pid } else { spawn.pgid }),
        sid: AtomicU64::new(if spawn.sid == 0 { pid } else { spawn.sid }),
        ctty: Mutex::new(spawn.ctty),
        exe: Mutex::new(String::from(path)),
        name: Mutex::new(String::from(name)),
        trace: AtomicBool::new(spawn.trace),
        cwd: Mutex::new(spawn.cwd),
        mm: Mutex::new(mm),
        tasks: Mutex::new(Vec::new()),
        kthreads: Mutex::new(Vec::new()),
        exiting: Mutex::new(None),
        thread_exit: WaitQueue::new(),
        ran_on: AtomicU32::new(0),
        env: Mutex::new(spawn.env),
        fds: Mutex::new(FdTable::new([stdin, stdout, stderr])),
        status: Mutex::new(None),
        exited: WaitQueue::new(),
        child_wait: WaitQueue::new(),
        signals: Mutex::new(SignalState::new()),
        vfork_done: AtomicBool::new(false),
        umask: AtomicU32::new(0o022),
        rlimits: Mutex::new(default_rlimits()),
        pdeathsig: AtomicU32::new(0),
        dumpable: AtomicU32::new(1),
        no_new_privs: AtomicBool::new(false),
        child_subreaper: AtomicBool::new(false),
        cpu: CpuTimes::default(),
        creds: Mutex::new(Creds::ROOT),
        nice: core::sync::atomic::AtomicI32::new(0),
    };
    serial_println!("[process] started pid {} ({}), entry {:#x}", pid, name, start.rip);
    Ok(start_process(process, start, 0, 0, crate::fpu::initial(), signal::AltStack::DISABLED, comm_from(name)))
}

/// Put the process in the table and start its first thread (tid = pid).
fn start_process(process: Process, start: TrapFrame, fs_base: u64, blocked: u64, fpu: FpuState, altstack: signal::AltStack, comm: [u8; 16]) -> Arc<Process> {
    let process = Arc::new(process);
    without_interrupts(|| PROCESSES.lock().insert(process.pid, process.clone()));
    start_task(&process, process.pid, start, fs_base, blocked, fpu, altstack, comm);
    process
}

/// Start a thread `tid` of `process` that enters user mode with `start`.
/// `fpu`: its FPU/SSE registers (a copy of the creator's, or clean).
/// `altstack`: its sigaltstack (fork keeps the caller's); `comm`: its name.
#[allow(clippy::too_many_arguments)]
fn start_task(process: &Arc<Process>, tid: u64, start: TrapFrame, fs_base: u64, blocked: u64, fpu: FpuState, altstack: signal::AltStack, comm: [u8; 16]) -> Arc<Task> {
    let task = Arc::new(Task {
        tid: AtomicU64::new(tid),
        process: process.clone(),
        kthread: AtomicUsize::new(usize::MAX),
        fs_base: AtomicU64::new(fs_base),
        clear_child_tid: AtomicU64::new(0),
        start: Mutex::new(Some(start)),
        killed: AtomicBool::new(false),
        sig: Mutex::new(ThreadSig::with_altstack(blocked, altstack)),
        fpu: Mutex::new(fpu),
        comm: Mutex::new(comm),
        robust_list: AtomicU64::new(0),
        cpu_ticks: AtomicU64::new(0),
    });
    without_interrupts(|| process.tasks.lock().push(task.clone()));
    let l4 = process.mm.lock().l4;
    thread::spawn_with(&process.name(), user_thread_entry, l4, Some(task.clone()));
    task
}

/// First code of every user thread (still ring 0, but already running
/// with the process's CR3 and kernel stack): enter user mode with the
/// prepared registers.
fn user_thread_entry() {
    let task = thread::current_task().expect("user thread without task");
    let me = thread::current_id();
    task.kthread.store(me, Ordering::Release);
    without_interrupts(|| task.process.kthreads.lock().push(me)); // reap() joins it
    let frame = task.start.lock().take().expect("no start registers");
    let doomed = task.killed.load(Ordering::Acquire) || task.process.exiting().is_some();
    drop(task); // return_to_user never returns: don't leak the Arc
    if doomed {
        exit_thread(ExitStatus::Exited(0)); // the group is going away already
    }
    crate::trap::return_to_user(frame)
}

/// Random bytes for AT_RANDOM (virtio-rng; the TSC if there's none).
fn random_bytes(buf: &mut [u8]) {
    if !crate::virtio::rng::fill(buf) {
        let mut x = unsafe { core::arch::x86_64::_rdtsc() } | 1;
        for b in buf {
            x ^= x << 13;
            x ^= x >> 7;
            x ^= x << 17;
            *b = x as u8;
        }
    }
}

// Auxiliary vector keys (Linux's AT_* values, `man getauxval`).
const AT_NULL: u64 = 0;
const AT_PHDR: u64 = 3;
const AT_PHENT: u64 = 4;
const AT_PHNUM: u64 = 5;
const AT_PAGESZ: u64 = 6;
const AT_BASE: u64 = 7;
const AT_FLAGS: u64 = 8;
const AT_ENTRY: u64 = 9;
const AT_UID: u64 = 11;
const AT_EUID: u64 = 12;
const AT_GID: u64 = 13;
const AT_EGID: u64 = 14;
const AT_PLATFORM: u64 = 15;
const AT_HWCAP: u64 = 16;
const AT_CLKTCK: u64 = 17;
const AT_SECURE: u64 = 23;
const AT_RANDOM: u64 = 25;
const AT_EXECFN: u64 = 31;

/// Load a program into the empty address space `mm` and build its initial
/// stack the way Linux does, so a C library's `_start` finds everything
/// where it expects it. Returns the registers to start with.
///
/// The stack, from USER_STACK_TOP downwards:
///
///   strings: execfn, environment, arguments, "x86_64", 16 random bytes,
///            (a copy of the program headers if no segment contains them)
///   (padding to 16 bytes)
///   auxv:  pairs of [key, value] ..., [AT_NULL, 0]
///   envp:  pointers to the environment strings ..., NULL
///   argv:  pointers to the argument strings ..., NULL
///   argc                                          <- rsp (16-byte aligned)
pub fn load_program(mm: &mut Mm, image: &[u8], argv: &[&[u8]], envp: &[&[u8]], execfn: &str) -> Result<TrapFrame, ElfError> {
    use crate::mm::{Kind, Vma, PROT_READ, PROT_WRITE};
    let info = elf::load(image, mm)?;
    // The heap starts on the page after the program; the stack is a region
    // that fills in as it's used.
    mm.brk_start = info.end.next_multiple_of(4096);
    mm.brk = mm.brk_start;
    mm.insert(Vma::new(USER_STACK_TOP - USER_STACK_SIZE, USER_STACK_TOP, PROT_READ | PROT_WRITE, Kind::Stack));

    let mut sp = USER_STACK_TOP;
    let mut push = |mm: &mut Mm, bytes: &[u8], nul: bool| -> Result<u64, ElfError> {
        sp -= bytes.len() as u64 + nul as u64;
        mm.write_bytes(sp, bytes).map_err(|_| ElfError::OutOfMemory)?;
        if nul {
            mm.write_bytes(sp + bytes.len() as u64, &[0]).map_err(|_| ElfError::OutOfMemory)?;
        }
        Ok(sp)
    };
    let execfn_ptr = push(mm, execfn.as_bytes(), true)?;
    let env_ptrs: Vec<u64> = envp.iter().map(|e| push(mm, e, true)).collect::<Result<_, _>>()?;
    let arg_ptrs: Vec<u64> = argv.iter().map(|a| push(mm, a, true)).collect::<Result<_, _>>()?;
    let platform = push(mm, b"x86_64", true)?;
    let mut random = [0u8; 16];
    random_bytes(&mut random);
    let random_ptr = push(mm, &random, false)?;
    let phdr = match info.phdr {
        Some(addr) => addr,
        None => {
            let start = info.phoff as usize;
            push(mm, &image[start..start + (info.phent * info.phnum) as usize], false)?
        }
    };

    let mut words: Vec<u64> = vec![arg_ptrs.len() as u64];
    words.extend(&arg_ptrs);
    words.push(0);
    words.extend(&env_ptrs);
    words.push(0);
    #[rustfmt::skip]
    words.extend([
        AT_PHDR, phdr, AT_PHENT, info.phent, AT_PHNUM, info.phnum,
        AT_PAGESZ, 4096, AT_BASE, 0, AT_FLAGS, 0, AT_ENTRY, info.entry,
        AT_UID, 0, AT_EUID, 0, AT_GID, 0, AT_EGID, 0,
        AT_PLATFORM, platform, AT_HWCAP, 0, AT_CLKTCK, 100, AT_SECURE, 0,
        AT_RANDOM, random_ptr, AT_EXECFN, execfn_ptr, AT_NULL, 0,
    ]);
    let rsp = (sp - 8 * words.len() as u64) & !0xf;
    let bytes: Vec<u8> = words.iter().flat_map(|w| w.to_le_bytes()).collect();
    mm.write_bytes(rsp, &bytes).map_err(|_| ElfError::OutOfMemory)?;
    // The ABI also says: rdx = function for atexit (0 = none), others zero.
    Ok(TrapFrame::new_user(info.entry, rsp))
}

impl Process {
    /// The open file behind `fd`.
    pub fn file(&self, fd: u64) -> Result<Arc<dyn File>, Errno> {
        self.fds.lock().get(fd)
    }

    /// Put `file` into the lowest free fd slot (like Unix).
    pub fn add_file(&self, file: Arc<dyn File>) -> Result<u64, Errno> {
        self.fds.lock().add(file, 0, false)
    }

    /// open(O_CLOEXEC), pipe2, fcntl(F_DUPFD...): lowest free fd >= `min`.
    pub fn add_file_with(&self, file: Arc<dyn File>, min: usize, cloexec: bool) -> Result<u64, Errno> {
        self.fds.lock().add(file, min, cloexec)
    }

    /// dup2/dup3: make `new` refer to the same file as `old`.
    pub fn dup_to(&self, old: u64, new: u64, cloexec: bool) -> Result<u64, Errno> {
        if new as usize >= MAX_FDS {
            return Err(Errno::EBADF);
        }
        let replaced = {
            let mut fds = self.fds.lock();
            if new as usize >= fds.limit {
                return Err(Errno::EBADF);
            }
            let file = fds.get(old)?;
            fds.set(new as usize, file, cloexec)
        };
        if let Some(f) = &replaced {
            self.drop_locks(&**f);
        }
        drop(replaced); // closing the old file may block: after unlocking
        Ok(new)
    }

    pub fn cloexec(&self, fd: u64) -> Result<bool, Errno> {
        let fds = self.fds.lock();
        fds.get(fd)?;
        Ok(fds.cloexec & (1 << fd) != 0)
    }

    pub fn set_cloexec(&self, fd: u64, on: bool) -> Result<(), Errno> {
        let mut fds = self.fds.lock();
        fds.get(fd)?;
        fds.set_cloexec(fd as usize, on);
        Ok(())
    }

    pub fn close(&self, fd: u64) -> Result<(), Errno> {
        let file = {
            let mut fds = self.fds.lock();
            match fds.files.get_mut(fd as usize) {
                Some(slot @ Some(_)) => slot.take(),
                _ => return Err(Errno::EBADF),
            }
        };
        if let Some(f) = &file {
            self.drop_locks(&**f);
        }
        drop(file); // dropping the last Arc closes the file (after unlocking)
        Ok(())
    }

    /// POSIX: closing any fd of a file drops my record locks on it (fs/lock.rs).
    fn drop_locks(&self, file: &dyn File) {
        fs::lock::file_closed(fs::lock::key_of(file), self.pid);
    }

    /// The fd limit (RLIMIT_NOFILE's soft value).
    /// Every open fd and its file, lowest first (stage 18: /proc/self/fd).
    pub fn open_files(&self) -> Vec<(u64, Arc<dyn File>)> {
        let fds = self.fds.lock();
        fds.files.iter().enumerate().filter_map(|(fd, f)| f.clone().map(|f| (fd as u64, f))).collect()
    }

    pub fn fd_limit(&self) -> usize {
        self.fds.lock().limit
    }

    /// getrlimit / prlimit64: [soft, hard] of `resource`. EINVAL if unknown.
    pub fn rlimit(&self, resource: u64) -> Result<[u64; 2], Errno> {
        let r = usize::try_from(resource).ok().filter(|&r| r < RLIM_NLIMITS).ok_or(Errno::EINVAL)?;
        Ok(self.rlimits.lock()[r])
    }

    /// setrlimit / prlimit64. The soft limit can't be above the hard one
    /// (EINVAL); a hard limit above what the kernel can do is EPERM. (We're
    /// root, so raising a hard limit is otherwise fine, like Linux's
    /// CAP_SYS_RESOURCE.)
    pub fn set_rlimit(&self, resource: u64, new: [u64; 2]) -> Result<(), Errno> {
        let r = usize::try_from(resource).ok().filter(|&r| r < RLIM_NLIMITS).ok_or(Errno::EINVAL)?;
        if new[0] > new[1] {
            return Err(Errno::EINVAL);
        }
        if new[1] > rlimit_ceiling(r) {
            return Err(Errno::EPERM);
        }
        self.rlimits.lock()[r] = new;
        if r == RLIMIT_NOFILE {
            // Fds already open above the new limit stay open (like Linux);
            // only new ones must be below it.
            self.fds.lock().limit = new[0] as usize;
        }
        Ok(())
    }

    pub fn status(&self) -> Option<ExitStatus> {
        without_interrupts(|| *self.status.lock())
    }

    /// Make `path` absolute, relative to this process's current directory.
    pub fn resolve(&self, path: &str) -> Result<String, Errno> {
        let cwd = self.cwd.lock().clone();
        fs::normalize(&cwd, path)
    }

    pub fn name(&self) -> String {
        self.name.lock().clone()
    }

    /// prctl(PR_SET_NAME) on the main thread renames the process too.
    pub fn set_name(&self, name: &str) {
        *self.name.lock() = String::from(name);
    }

    pub fn parent(&self) -> u64 {
        self.parent.load(Ordering::Relaxed)
    }

    pub fn pgid(&self) -> u64 {
        self.pgid.load(Ordering::Relaxed)
    }

    pub fn tracing(&self) -> bool {
        self.trace.load(Ordering::Relaxed)
    }

    pub fn env(&self) -> Vec<String> {
        self.env.lock().clone()
    }

    pub fn sid(&self) -> u64 {
        self.sid.load(Ordering::Relaxed)
    }

    /// Wake its threads if they're blocked (they then check for signals).
    pub fn wake(&self) {
        for task in self.tasks() {
            task.wake();
        }
    }

    /// Its live threads.
    pub fn tasks(&self) -> Vec<Arc<Task>> {
        without_interrupts(|| self.tasks.lock().clone())
    }

    /// The thread with id `tid`, if it's one of ours and alive.
    pub fn task(&self, tid: u64) -> Option<Arc<Task>> {
        without_interrupts(|| self.tasks.lock().iter().find(|t| t.tid() == tid).cloned())
    }

    /// Set once exit_group has been called (all threads are on their way out).
    pub fn exiting(&self) -> Option<ExitStatus> {
        without_interrupts(|| *self.exiting.lock())
    }
}

/// Look up a live (or not yet reaped) process.
pub fn get(pid: u64) -> Option<Arc<Process>> {
    without_interrupts(|| PROCESSES.lock().get(&pid).cloned())
}

/// How many processes are in the table (running or not yet reaped).
pub fn count() -> usize {
    without_interrupts(|| PROCESSES.lock().len())
}

/// Every process in the table.
pub fn all() -> Vec<Arc<Process>> {
    without_interrupts(|| PROCESSES.lock().values().cloned().collect())
}

/// The (not exited) members of process group `pgid`.
pub fn group_members(pgid: u64) -> Vec<Arc<Process>> {
    without_interrupts(|| {
        PROCESSES.lock().values().filter(|p| p.pgid() == pgid && p.status().is_none()).cloned().collect()
    })
}

/// End the whole current process: every thread of it (exit_group, a
/// fatal signal). The first status given wins.
pub fn exit_group(status: ExitStatus) -> ! {
    if let Some(p) = thread::current_process() {
        without_interrupts(|| {
            p.exiting.lock().get_or_insert(status);
        });
        // The other threads notice at their next return to user mode, or
        // right away if they're sleeping (signal::interrupted).
        p.wake();
        signal::wake_stopped();
    }
    exit_thread(status)
}

/// The old name: end the current process.
pub fn exit_current(status: ExitStatus) -> ! {
    exit_group(status)
}

/// End the calling thread (the exit syscall). If it's the last one, the
/// process ends too -- with `status`, unless exit_group already set one.
pub fn exit_thread(status: ExitStatus) -> ! {
    if let Some(task) = thread::current_task() {
        let p = task.process.clone();
        // Robust futexes (stage 16): locks it died holding get "owner died".
        exit_robust_list(&p, &task);
        // pthread_join sleeps on this word with a futex until it's 0.
        let ctid = task.clear_child_tid.swap(0, Ordering::Relaxed);
        if ctid != 0 && ctid % 4 == 0 && crate::mm::user_access_ok(ctid, 4, true) {
            unsafe { core::ptr::write_volatile(ctid as *mut u32, 0) };
            crate::futex::wake(&p, ctid, 1);
        }
        let last = without_interrupts(|| {
            let mut tasks = p.tasks.lock();
            tasks.retain(|t| !Arc::ptr_eq(t, &task));
            tasks.is_empty()
        });
        if !last {
            serial_println!("[process] pid {} thread {} exited", p.pid, task.tid());
        }
        drop(task);
        p.thread_exit.wake_all();
        if last {
            let status = p.exiting().unwrap_or(status);
            process_exit(p, status);
        }
    }
    thread::exit_current()
}

// --- Robust futex lists (set_robust_list, stage 16) ------------------------
//
// A "robust" mutex survives its owner dying. The lock word holds the
// owner's thread id; each thread keeps a linked list of the robust locks it
// holds, in its own memory, and tells the kernel where the list starts:
//
//   struct robust_list_head {
//       struct robust_list *next;   // first entry (the head itself = empty)
//       long futex_offset;          // entry address + this = the lock word
//       struct robust_list *list_op_pending; // lock being taken/released now
//   };
//
// When the thread exits, the kernel walks the list. Every lock word that
// still has the thread's id gets FUTEX_OWNER_DIED set (and the id cleared),
// and if somebody waits (FUTEX_WAITERS) one waiter is woken -- so the next
// pthread_mutex_lock returns EOWNERDEAD instead of hanging forever.

pub const ROBUST_LIST_HEAD_SIZE: u64 = 24;
const FUTEX_WAITERS: u32 = 0x8000_0000;
const FUTEX_OWNER_DIED: u32 = 0x4000_0000;
const FUTEX_TID_MASK: u32 = 0x3fff_ffff;
/// Linux's limit on list entries (a loop in a corrupted list can't hang us).
const ROBUST_LIST_LIMIT: usize = 2048;

fn read_user_u64(addr: u64) -> Option<u64> {
    let b = crate::syscall::user_slice(addr, 8).ok()?;
    Some(u64::from_le_bytes(b.try_into().ok()?))
}

/// Mark one lock word "owner died" if `tid` holds it.
fn handle_futex_death(p: &Process, uaddr: u64, tid: u32) {
    if uaddr % 4 != 0 || !crate::mm::user_access_ok(uaddr, 4, true) {
        return;
    }
    let word = unsafe { &*(uaddr as *const core::sync::atomic::AtomicU32) };
    let mut val = word.load(Ordering::SeqCst);
    loop {
        if val & FUTEX_TID_MASK != tid {
            return; // not (or no longer) ours
        }
        let new = (val & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
        match word.compare_exchange(val, new, Ordering::SeqCst, Ordering::SeqCst) {
            Ok(_) => break,
            Err(now) => val = now,
        }
    }
    if val & FUTEX_WAITERS != 0 {
        crate::futex::wake(p, uaddr, 1);
    }
}

/// Walk the exiting thread's robust list (see above).
fn exit_robust_list(p: &Process, task: &Task) {
    let head = task.robust_list.swap(0, Ordering::Relaxed);
    if head == 0 {
        return;
    }
    let tid = task.tid() as u32;
    let (Some(mut entry), Some(offset), Some(pending)) = (read_user_u64(head), read_user_u64(head + 8), read_user_u64(head + 16)) else {
        return;
    };
    let offset = offset as i64;
    for _ in 0..ROBUST_LIST_LIMIT {
        if entry == head {
            break;
        }
        // (Bit 0 of an entry pointer is a PI-futex flag: ignore it.)
        let next = read_user_u64(entry & !1);
        if entry != pending {
            handle_futex_death(p, (entry & !1).wrapping_add(offset as u64), tid);
        }
        match next {
            Some(n) => entry = n,
            None => break,
        }
    }
    if pending != 0 {
        handle_futex_death(p, (pending & !1).wrapping_add(offset as u64), tid);
    }
}

/// The last thread of `process` is exiting: close everything, become a
/// zombie, tell the parent.
fn process_exit(process: Arc<Process>, status: ExitStatus) {
    // Close all files (a pipe's reader sees EOF once its writers are gone).
    let files = core::mem::take(&mut process.fds.lock().files);
    drop(files);
    fs::lock::process_exited(process.pid); // its fcntl record locks
    crate::timer::process_exited(process.pid); // its alarm and POSIX timers (stage 17)
    // prctl(PR_SET_PDEATHSIG): children that asked to hear of our death.
    for child in all().into_iter().filter(|c| c.parent() == process.pid && c.status().is_none()) {
        let sig = child.pdeathsig.load(Ordering::Relaxed);
        if sig != 0 {
            signal::send(&child, sig, SigInfo::from_pid(sig, process.pid));
        }
    }
    crate::tty::process_exited(&process); // (raw mode, controlling terminal)
    let name = process.name();
    match status {
        // Serial only: on screen the shell reports exit codes itself.
        ExitStatus::Exited(code) => {
            serial_println!("[process] pid {} ({}) exited with code {}", process.pid, name, code)
        }
        // A crash (SIGSEGV...) is worth a line on screen; a signal
        // somebody sent on purpose (Ctrl+C, kill) isn't.
        ExitStatus::Killed(sig, why) if why != signal::name(sig) => {
            crate::kprintln!("[process] pid {} ({}) killed: {} ({})", process.pid, name, why, signal::name(sig))
        }
        ExitStatus::Killed(sig, _) => {
            serial_println!("[process] pid {} ({}) killed by {}", process.pid, name, signal::name(sig))
        }
    }
    // No preemption from here on: we're about to disappear anyway.
    x86_64::instructions::interrupts::disable();
    let removed = {
        // The table lock makes "set my status" and "look at my parent"
        // one step, so a parent exiting at the same time on another CPU
        // either sees our status (and reaps us) or orphans us first (and
        // we reap ourselves). Never both, never neither.
        let mut table = PROCESSES.lock();
        *process.status.lock() = Some(status);
        let mut removed = Vec::new();
        let children: Vec<Arc<Process>> = table.values().filter(|c| c.parent() == process.pid).cloned().collect();
        let adopter = adopter(&table, &process);
        for child in children {
            if child.status().is_some() {
                removed.extend(table.remove(&child.pid)); // exited: reap it
            } else {
                // Still running: a reaper adopts it, or it reaps itself.
                child.parent.store(adopter.unwrap_or(ORPHAN), Ordering::Relaxed);
            }
        }
        // Nobody will wait for us if we're an orphan, or if our parent
        // said so by ignoring SIGCHLD explicitly (POSIX): reap ourselves.
        let parent_ignores = table.get(&process.parent()).is_some_and(|p| {
            let action = p.signals.lock().action(signal::SIGCHLD);
            action.handler == signal::SIG_IGN || action.flags & signal::SA_NOCLDWAIT != 0
        });
        if process.parent() == ORPHAN || parent_ignores {
            removed.extend(table.remove(&process.pid));
        }
        removed
    };
    drop(removed); // (frees their memory if they're completely gone)
    process.exited.wake_all(); // the kernel may be blocked in wait()
    process.vfork_done.store(true, Ordering::Release);
    VFORK_WAIT.wake_all();
    let code = match status {
        ExitStatus::Exited(_) => signal::CLD_EXITED,
        ExitStatus::Killed(sig, _) if signal::default_action(sig) == signal::DefaultAction::Core => signal::CLD_DUMPED,
        ExitStatus::Killed(..) => signal::CLD_KILLED,
    };
    let si_status = match status {
        ExitStatus::Exited(c) => c as i32,
        ExitStatus::Killed(sig, _) => sig as i32,
    };
    signal::notify_parent(&process, code, si_status);
    drop(process);
}

/// Who adopts the running children of `dying` (stage 16)? Like Linux: the
/// nearest living ancestor that said prctl(PR_SET_CHILD_SUBREAPER), else
/// pid 1 -- but only if pid 1 is a real `init` (named "init", or a
/// subreaper), because one that never calls wait(-1) would collect
/// zombies forever. Today pid 1 is usually our own shell, which only waits
/// for its jobs, so most orphans get None: they reap themselves on exit
/// (parent = ORPHAN), which frees their memory just the same.
fn adopter(table: &BTreeMap<u64, Arc<Process>>, dying: &Process) -> Option<u64> {
    let alive = |pid: u64| table.get(&pid).filter(|p| p.status().is_none() && p.pid != dying.pid);
    let mut pid = dying.parent();
    for _ in 0..table.len() {
        match alive(pid) {
            Some(p) if p.child_subreaper.load(Ordering::Relaxed) => return Some(p.pid),
            Some(p) => pid = p.parent(),
            None => break,
        }
    }
    alive(1).filter(|init| init.child_subreaper.load(Ordering::Relaxed) || init.name() == "init").map(|init| init.pid)
}

/// Block (the calling thread) until `process` has ended and its thread is
/// gone, then remove it from the process table. Used by the kernel (tests,
/// the shell supervisor). Once the caller drops its own `Arc`, all of the
/// process's memory is free again.
pub fn wait(process: &Process) -> ExitStatus {
    let status = process.exited.wait_until(|| *process.status.lock());
    reap(process);
    status
}

/// Wait for the (exited) process's threads to be freed, then remove the
/// process from the table.
fn reap(process: &Process) {
    // Every thread registers itself before it can exit, and the status is
    // only set when the last one exits: so the list is complete here.
    let kthreads = without_interrupts(|| core::mem::take(&mut *process.kthreads.lock()));
    for tid in kthreads {
        thread::join(tid);
    }
    let removed = without_interrupts(|| PROCESSES.lock().remove(&process.pid));
    drop(removed);
}

// wait4 options.
pub const WNOHANG: u64 = 1;
pub const WUNTRACED: u64 = 2;
pub const WCONTINUED: u64 = 8;
/// Linux's "wait for threads too" flags: accepted, nothing to do here.
const WALL_FLAGS: u64 = 0x2000_0000 | 0x4000_0000 | 0x8000_0000;

/// Kernel-side wait for one child (the tests use it): EAGAIN with WNOHANG
/// while it runs.
pub fn wait_child(parent: u64, pid: u64, flags: u64) -> Result<ExitStatus, Errno> {
    let child = get(pid).filter(|c| c.parent() == parent).ok_or(Errno::ECHILD)?;
    if flags & WNOHANG != 0 {
        let status = child.status().ok_or(Errno::EAGAIN)?;
        reap(&child);
        return Ok(status);
    }
    Ok(wait(&child))
}

/// What wait4 found.
enum Found {
    Event(Arc<Process>, i32, bool), // child, status word, exited (reap it)
    Nothing,                        // WNOHANG and no news
}

/// wait4(pid, *status, options): wait for a child to exit (or, with
/// WUNTRACED / WCONTINUED, to stop / continue). `pid` > 0: that child;
/// -1: any child; 0: any child in my process group; < -1: any child in
/// group -pid. Returns (pid, status), or pid 0 with WNOHANG and no news.
pub fn wait4(me: &Process, pid: i64, options: u64) -> Result<(u64, i32), Errno> {
    if options & !(WNOHANG | WUNTRACED | WCONTINUED | WALL_FLAGS) != 0 {
        return Err(Errno::EINVAL);
    }
    let matches = |c: &Process| {
        c.parent() == me.pid
            && match pid {
                -1 => true,
                0 => c.pgid() == me.pgid(),
                p if p < -1 => c.pgid() == p.unsigned_abs(),
                p => c.pid == p as u64,
            }
    };
    let found = me.child_wait.wait_until_intr(|| {
        let table = PROCESSES.lock();
        let mut any = false;
        for c in table.values().filter(|c| matches(c)) {
            any = true;
            if let Some(status) = *c.status.lock() {
                return Some(Ok(Found::Event(c.clone(), status.wait_status(), true)));
            }
            let mut s = c.signals.lock();
            match s.report {
                Some(signal::Report::Stopped(sig)) if options & WUNTRACED != 0 => {
                    s.report = None;
                    return Some(Ok(Found::Event(c.clone(), 0x7f | (sig as i32) << 8, false)));
                }
                Some(signal::Report::Continued) if options & WCONTINUED != 0 => {
                    s.report = None;
                    return Some(Ok(Found::Event(c.clone(), 0xffff, false)));
                }
                _ => {}
            }
        }
        if !any {
            Some(Err(Errno::ECHILD))
        } else if options & WNOHANG != 0 {
            Some(Ok(Found::Nothing))
        } else {
            None // sleep until a child changes state (or a signal)
        }
    })??;
    match found {
        Found::Nothing => Ok((0, 0)),
        Found::Event(child, status, exited) => {
            if exited {
                reap(&child);
                // Its CPU time (and its reaped children's) now counts as
                // our children's time (times(), getrusage).
                let c = &child.cpu;
                let user = c.user.load(Ordering::Relaxed) + c.child_user.load(Ordering::Relaxed);
                let system = c.system.load(Ordering::Relaxed) + c.child_system.load(Ordering::Relaxed);
                me.cpu.child_user.fetch_add(user, Ordering::Relaxed);
                me.cpu.child_system.fetch_add(system, Ordering::Relaxed);
            }
            Ok((child.pid, status))
        }
    }
}

/// fork (and vfork / basic clone): a new process, a copy of `parent`, that
/// starts by returning from the same syscall with rax = 0. `stack`: a new
/// stack pointer for the child (clone), or 0 = the same.
pub fn fork(parent: &Arc<Process>, frame: &TrapFrame, stack: u64, vfork: bool) -> Result<u64, Errno> {
    if count() >= MAX_PROCESSES {
        return Err(Errno::EAGAIN); // fork bombs stop here
    }
    // The heart of it: copy the page tables, sharing the frames
    // copy-on-write (mm.rs).
    let mm = parent.mm.lock().fork().ok_or(Errno::ENOMEM)?;
    let mut start = *frame;
    start.rax = 0; // "you're the child"
    if stack != 0 {
        start.rsp = stack;
    }
    let pid = NEXT_PID.fetch_add(1, Ordering::Relaxed);
    let signals = without_interrupts(|| parent.signals.lock().fork());
    // Only the calling thread is copied, with its TLS pointer and mask.
    let me = thread::current_task();
    let fs_base = me.as_ref().map_or(0, |t| t.fs_base.load(Ordering::Relaxed));
    let blocked = me.as_ref().map_or(0, |t| without_interrupts(|| t.sig.lock().blocked));
    // ... and its sigaltstack and name (stage 16).
    let altstack = me.as_ref().map_or(signal::AltStack::DISABLED, |t| without_interrupts(|| t.sig.lock().altstack));
    let comm = me.as_ref().map_or_else(|| comm_from(&parent.name()), |t| *t.comm.lock());
    let child = Process {
        pid,
        parent: AtomicU64::new(parent.pid),
        pgid: AtomicU64::new(parent.pgid()),
        sid: AtomicU64::new(parent.sid()),
        ctty: Mutex::new(parent.ctty.lock().clone()),
        exe: Mutex::new(parent.exe.lock().clone()),
        name: Mutex::new(parent.name()),
        trace: AtomicBool::new(parent.tracing()),
        cwd: Mutex::new(parent.cwd.lock().clone()),
        mm: Mutex::new(mm),
        tasks: Mutex::new(Vec::new()),
        kthreads: Mutex::new(Vec::new()),
        exiting: Mutex::new(None),
        thread_exit: WaitQueue::new(),
        ran_on: AtomicU32::new(0),
        env: Mutex::new(parent.env()),
        fds: Mutex::new(parent.fds.lock().clone()), // same open files (Arc clones)
        status: Mutex::new(None),
        exited: WaitQueue::new(),
        child_wait: WaitQueue::new(),
        signals: Mutex::new(signals),
        vfork_done: AtomicBool::new(false),
        umask: AtomicU32::new(parent.umask.load(Ordering::Relaxed)),
        rlimits: Mutex::new(*parent.rlimits.lock()),
        pdeathsig: AtomicU32::new(0), // (not inherited, like Linux)
        dumpable: AtomicU32::new(parent.dumpable.load(Ordering::Relaxed)),
        no_new_privs: AtomicBool::new(parent.no_new_privs.load(Ordering::Relaxed)),
        child_subreaper: AtomicBool::new(false),
        cpu: CpuTimes::default(),
        creds: Mutex::new(*parent.creds.lock()),
        nice: core::sync::atomic::AtomicI32::new(parent.nice.load(Ordering::Relaxed)),
    };
    serial_println!("[process] pid {} forked pid {}", parent.pid, pid);
    // The child's FPU/SSE registers start as a copy of ours (we're running,
    // so ours are in the CPU right now).
    let child = start_process(child, start, fs_base, blocked, crate::fpu::save_live(), altstack, comm);
    if vfork {
        // The parent sleeps until the child has exec'd or exited (real
        // vfork shares the memory meanwhile; we just use copy-on-write).
        VFORK_WAIT.wait_until(|| child.vfork_done.load(Ordering::Acquire).then_some(()));
    }
    Ok(pid)
}

/// clone(CLONE_THREAD ...): a new thread in process `p`, starting like the
/// caller returning from clone with rax = 0, on stack `stack`, with TLS
/// pointer `tls` (if Some). Returns its tid, which is also written to the
/// user addresses in `set_tid` (CLONE_PARENT_SETTID / CHILD_SETTID) before
/// the thread can run.
pub fn clone_thread(p: &Arc<Process>, frame: &TrapFrame, stack: u64, tls: Option<u64>, clear_tid: u64, set_tid: &[u64]) -> Result<u64, Errno> {
    let me = thread::current_task().ok_or(Errno::EINVAL)?;
    if p.tasks().len() >= MAX_THREADS || p.exiting().is_some() {
        return Err(Errno::EAGAIN);
    }
    let mut start = *frame;
    start.rax = 0; // "you're the new thread"
    if stack != 0 {
        start.rsp = stack;
    }
    for &ptr in set_tid {
        crate::syscall::user_slice_mut(ptr, 4)?; // check first
    }
    let tid = NEXT_PID.fetch_add(1, Ordering::Relaxed);
    for &ptr in set_tid {
        crate::syscall::user_slice_mut(ptr, 4)?.copy_from_slice(&(tid as u32).to_le_bytes());
    }
    let fs_base = tls.unwrap_or_else(|| me.fs_base.load(Ordering::Relaxed));
    let blocked = without_interrupts(|| me.sig.lock().blocked);
    // Forget scheduler threads of threads that are completely gone.
    without_interrupts(|| p.kthreads.lock().retain(|&k| thread::exists(k)));
    // A new thread has no sigaltstack (its creator's stack isn't its own)
    // and starts with its creator's name.
    let comm = *me.comm.lock();
    let task = start_task(p, tid, start, fs_base, blocked, crate::fpu::save_live(), signal::AltStack::DISABLED, comm);
    task.clear_child_tid.store(clear_tid, Ordering::Relaxed);
    serial_println!("[process] pid {} new thread {}", p.pid, tid);
    Ok(tid)
}

/// execve in a process with several threads: the others must go first
/// (the new program starts with one thread). Linux calls this de_thread.
fn kill_other_threads(p: &Arc<Process>) {
    let Some(me) = thread::current_task() else { return };
    for t in p.tasks() {
        if !Arc::ptr_eq(&t, &me) {
            t.killed.store(true, Ordering::Release);
            t.wake();
        }
    }
    signal::wake_stopped();
    p.thread_exit.wait_until(|| (p.tasks.lock().len() == 1).then_some(()));
    // The survivor takes over the main thread id, like Linux.
    me.tid.store(p.pid, Ordering::Relaxed);
}

/// execve, after the arguments have been copied in: replace the current
/// program. `frame` gets the new program's registers.
pub fn execve(p: &Arc<Process>, frame: &mut TrapFrame, path: &str, argv: &[Vec<u8>], envp: &[Vec<u8>]) -> Result<(), Errno> {
    let size: usize = argv.iter().chain(envp).map(|s| s.len() + 1 + 8).sum();
    if size > MAX_ARGS_SIZE {
        return Err(Errno::E2BIG);
    }
    let image = fs::read_all(path)?;
    // Check the whole file *before* the point of no return: once the old
    // program's memory is gone, there's nobody left to report an error to.
    elf::check(&image).map_err(elf_errno)?;
    let name = path.rsplit('/').next().unwrap_or(path);
    let args: Vec<&[u8]> = argv.iter().map(|a| a.as_slice()).collect();
    let env: Vec<&[u8]> = envp.iter().map(|a| a.as_slice()).collect();
    // --- point of no return ---
    kill_other_threads(p);
    let loaded = {
        let mut mm = p.mm.lock();
        mm.clear();
        load_program(&mut mm, &image, &args, &env, path)
    };
    let Ok(start) = loaded else {
        exit_current(ExitStatus::Killed(signal::SIGSEGV, "exec failed (out of memory)"));
    };
    *frame = start;
    *p.name.lock() = String::from(name);
    *p.exe.lock() = String::from(path);
    crate::timer::exec(p.pid); // POSIX timers don't survive exec (the alarm does)
    if let Some(me) = thread::current_task() {
        me.fs_base.store(0, Ordering::Relaxed); // no TLS until the new libc sets one
        crate::fpu::reset_live(); // clean FPU/SSE/AVX registers for the new program
        me.clear_child_tid.store(0, Ordering::Relaxed);
        me.robust_list.store(0, Ordering::Relaxed); // the list was in the old program's memory
        *me.comm.lock() = comm_from(name);
        without_interrupts(|| me.sig.lock().altstack = signal::AltStack::DISABLED);
    }
    *p.env.lock() = envp.iter().map(|e| String::from_utf8_lossy(e).into_owned()).collect();
    without_interrupts(|| p.signals.lock().exec());
    let closed = p.fds.lock().exec();
    drop(closed);
    p.vfork_done.store(true, Ordering::Release);
    VFORK_WAIT.wake_all();
    serial_println!("[process] pid {} exec {}", p.pid, path);
    Ok(())
}

/// setpgid(pid, pgid): move process `pid` (0 = me; myself or a child of
/// mine) into group `pgid` (0 = a new group named after it). The group
/// must exist unless it's the new one.
pub fn setpgid(me: &Process, pid: u64, pgid: u64) -> Result<(), Errno> {
    let pid = if pid == 0 { me.pid } else { pid };
    let target = get(pid).ok_or(Errno::ESRCH)?;
    if target.pid != me.pid && target.parent() != me.pid {
        return Err(Errno::ESRCH);
    }
    let pgid = if pgid == 0 { pid } else { pgid };
    // (Exited-but-not-reaped members count: a pipeline's first program may
    // already be done when the shell puts the second one in its group.)
    if pgid != pid && !without_interrupts(|| PROCESSES.lock().values().any(|p| p.pgid() == pgid)) {
        return Err(Errno::EPERM);
    }
    target.pgid.store(pgid, Ordering::Relaxed);
    Ok(())
}

/// setsid(): start a new session (and process group) led by `p`, with no
/// controlling terminal yet (the next terminal it opens becomes one, see
/// tty.rs). A process-group leader can't: its group would straddle two
/// sessions -- which is why getty forks first.
pub fn setsid(p: &Process) -> Result<u64, Errno> {
    if without_interrupts(|| PROCESSES.lock().values().any(|q| q.pgid() == p.pid)) {
        return Err(Errno::EPERM);
    }
    p.sid.store(p.pid, Ordering::Relaxed);
    p.pgid.store(p.pid, Ordering::Relaxed);
    *p.ctty.lock() = None;
    Ok(p.pid)
}

/// Send `sig` from the kernel to group `pgid` (Ctrl+C, Ctrl+Z).
pub fn signal_group(pgid: u64, sig: u32) {
    let _ = signal::send_group(pgid, sig, SigInfo::from_pid(sig, 0));
}