rust-kernel

@@ -261,7 +261,7 @@ dependencies = [  [[package]] name = "rustbelt"-version = "0.18.0"+version = "0.19.0" dependencies = [  "bootloader",  "conquer-once",

@@ -1,6 +1,6 @@ [package] name = "rustbelt"-version = "0.18.0"+version = "0.19.0" edition = "2024"  [dependencies]

@@ -1005,6 +1005,10 @@ heap_allocation::simple_allocation...               [ok] | `tests/libc_batch1.rs` | **libc batch 1** (stage 16): `libctest` runs one group per test, twice, and nothing (threads, processes, frames, file locks) may be left over: `wait` (dash's `wait`: SIGCHLD blocked, fork, `rt_sigsuspend` returns `EINTR` once the handler ran and puts the old mask back; a SIGCHLD already pending ends it at once); `signals` (`rt_sigpending`, `rt_sigtimedwait` with and without a timeout and from another process, real-time signals queued 3 times with their values, a standard signal only once, `EPERM`/`ESRCH` for `rt_sigqueueinfo`, `sigaltstack` + `SA_ONSTACK`, `EPERM` on the stack, unknown `sa_flags` dropped); `locks` (`F_GETFL` access modes, `F_SETFL`, `F_GETLK`/`F_SETLK`/`F_SETLKW` between two processes, `EDEADLK`, locks dropped on close and on exit, `RLIMIT_NOFILE` vs `dup` and `F_DUPFD`); `files` (`truncate`/`ftruncate`/`fsync` on ramfs, ext2 and FAT, `umask` on `open` and `mkdir` and across fork, `chmod`/`fchmod`/`fchmodat`, `utimensat` with `UTIME_NOW`/`UTIME_OMIT`, `EROFS` on btrfs); `misc` (rlimits, `prctl`, `sched_getaffinity`, `madvise(MADV_DONTNEED)` reads zeros, a robust futex gets `FUTEX_OWNER_DIED`); `orphans` (running and zombie orphans are freed, a `PR_SET_CHILD_SUBREAPER` process adopts and reaps a grandchild); `--trace` names unknown calls `syscall_N` | | `tests/coreutils_batch2.rs` | **coreutils batch 2** (stage 17): `coretest` runs one group per test, twice, and nothing (threads, processes, frames, file locks, timers) may be left over: `links` (`link`/`linkat`/`symlink`/`symlinkat` on ramfs and ext2 with `st_nlink` and `st_ino`, a dangling symlink, renaming a full directory and over a second name, `EPERM` on FAT/exFAT, `EROFS` on btrfs, `EXDEV`); `owners` (the `chown` family on ramfs and ext2, `-1`, `lchown` vs `chown` on a symlink, FAT's `EPERM` unless nothing changes); `statfs` (`statfs`/`fstatfs` magic numbers and sizes per mount, `ST_RDONLY`, pipes, `statx` vs `stat`, `st_dev` per mount); `info` (`sysinfo` 99 and 502, `times`, `getrusage` for self and children, the CPU-time clocks, `clock_getres`); `sockets` (`getsockname`/`getpeername`, `SO_REUSEADDR` & co., `SO_TYPE`, `SO_ERROR` after a refused connect, `SHUT_WR` = end of file for the peer, `EPIPE` after it, `socketpair` refused for `AF_INET` and unknown families; since stage 18 it works for `AF_UNIX`); `timers` (`alarm`, one-shot and periodic `setitimer`, `pause`, POSIX timers with `SI_TIMER` + `si_value`, `TIMER_ABSTIME`, `SIGEV_NONE`, not inherited by fork); `ids` (`setuid` & co. with the saved id, groups, new files owned by their creator, nice values, `sethostname` -> `uname`); `misc` (`flock` between open files and processes, `preadv`/`pwritev`, `sendfile`, mask sizes for `ppoll`/`pselect6`, a `ppoll` mask letting a signal in, `getdents64` while deleting on ramfs and ext2); `--trace` names (`link`, `statx`, `rb_sysinfo`) | | `tests/unix_sockets.rs` | **Unix sockets and /proc/self** (stage 18): `unixtest` runs one group per test, twice, and threads, processes, frames (so no open file stuck "in flight") and bound socket names must be back where they were: `stream` (a server on a path in `/tmp` and on ext2 echoes for a forked client; `S_IFSOCK` with mode 0777 minus the umask, `EADDRINUSE`, `ENXIO` from `open`, `unlink`, `ENOENT`/`ECONNREFUSED`, `EPERM` on FAT; abstract names and autobind (two sockets, two different names); `ECONNRESET` for unread data and for a connection the server closed before `accept`; `EPIPE`; `SO_PEERCRED`, `getsockname`/`getpeername`); `pair` (`socketpair` both ways and across fork, `SHUT_WR`, `MSG_PEEK`, seqpacket message borders, dgram pairs, `SOCK_CLOEXEC`/`SOCK_NONBLOCK`); `dgram` (sender names, `MSG_TRUNC`, `connect` and `EPERM` for strangers, `AF_UNSPEC`, `EPROTOTYPE`, `EMSGSIZE`, `ECONNREFUSED`); `rights` (a file and a pipe end read through received fds, `MSG_CTRUNC`, `MSG_CMSG_CLOEXEC`, `EBADF` for a bad fd, fds still in flight closed with their socket, a stream read taking the bytes before an fd message together with it and stopping after it (stage 19), `SCM_CREDENTIALS`); `nonblock` (`EAGAIN`, `SO_SNDBUF` 4096 reading back 8192, the buffer limit, poll bits, a blocked writer finishing in one `write`, `EPOLLRDHUP`/`EPOLLHUP`, nonblocking `accept`/`accept4`, a full backlog, `SO_RCVTIMEO`, `FIONREAD`); `proc` (`/proc/self/fd` listing and links, `pipe:[N]`/`socket:[N]` matching `st_ino`, opening an entry, `/proc/self/exe`); `--trace` names (`sendmsg`, `recvmsg`) |+| `tests/threads_stress.rs` | **the flake fix** (stage 19): the `threads` program 40 times in a row; a watchdog prints every thread and futex waiter if a run hangs (it used to, about once in 40 runs) |+| `tests/vdso.rs` | **the vDSO** (stage 19): the embedded image is a one-page `ET_DYN` with `__vdso_*` symbols; the vvar page is published (even `seq`, TSC flag set); the kernel clock counts in nanoseconds; `vdsotest` passes twice with no leaks (auxv entries, vDSO vs syscall ordering, 100,000 monotonic reads, sub-microsecond steps, `gettimeofday`/`time`/`sched_getcpu`, the vvar page can't be made writable or written, it all works after fork and after execve, a dynamic program is `ENOEXEC`); `--trace` shows **no** clock syscalls for 1000 rounds of clock calls, and with the fast path switched off (`vdso::force_syscall`) they're all there; `--trace` names (`gettimeofday`, `time`, `getcpu`) |+| `tests/static_pie.rs` | **static PIE** (stage 19): `piehello` (a `musl-gcc -static-pie` program: relocations applied, `AT_PHDR`/`AT_ENTRY`/`AT_BASE`, RELRO, no INTERP), a different page-aligned base on every run, `piethreads` (pthreads, `__thread` variables, stdio to a file, `ttyname`) twice with no leaks, `dynhello` (PT_INTERP) refused with `ENOEXEC`, ET_EXEC programs still load |+| `tests/mremap.rs` | **mremap** (stage 19): `remaptest` twice with no leaks: grow in place, shrink, same size, `ENOMEM` without `MREMAP_MAYMOVE`, a move keeps the bytes and empties the old place, `MREMAP_FIXED`, the `EINVAL`/`EFAULT` cases, a moved file mapping still reads the file, copy-on-write after a move in a forked child, musl's `realloc` of big blocks |  How it works: `#![feature(custom_test_frameworks)]` collects every `#[test_case]` function and passes them to `rustbelt::test_runner` (in `src/lib.rs`), which@@ -1112,7 +1116,7 @@ user/{ls,cat,wc,grep,mkdir,rm,touch,cp,sleep}.rs   the shell's tools user/{hello,evil,spin,demo,child}.rs  test/demo programs (all programs go to /mnt/disk0/bin) user/sysinfo.rs           CPUs, per-CPU load bars, threads, processes, memory (the sysinfo syscall) user/overflow.rs          recurses forever: killed with "stack overflow" by the user stack guard zone-src/time.rs               microsecond clock: the TSC, calibrated against the PIT at boot+src/time.rs               the clock: the TSC, calibrated against the PIT at boot (nanoseconds since stage 19) src/net/mod.rs            networking: the smoltcp Interface, DHCP + DNS, the `netd` thread, IRQ handling src/net/virtio_net.rs     legacy virtio-net driver: RX/TX virtqueues, MAC src/net/device.rs         smoltcp's Device: frames for our own MAC/IP (127.x) loop back, the rest go to the NIC@@ -1133,6 +1137,11 @@ src/syscall_ext.rs        stage 17: links, chown, statfs/statx, CPU times, socke src/timer.rs              stage 17: alarm, setitimer and POSIX timers; the timerd thread that sends their signals user/unixtest.rs          stage 18: Unix socket and /proc/self checks, one group per argument (tests/unix_sockets.rs) src/fs/procfs.rs          stage 18: /proc/self/fd and /proc/self/exe, made up from the asking process+vdso/vdso.c, vdso/vdso.lds  stage 19: the vDSO (clock_gettime, gettimeofday, time, getcpu without a+                          system call) and its linker script; build.rs compiles it into the kernel+src/vdso.rs               stage 19: the vDSO's pages, the vvar page and its seqlock, the TSC trust rule+user/{vdsotest,piehello,piethreads,dynhello,remaptest}.c  stage 19: C programs built with musl-gcc+                          (needs musl-tools; tests/vdso.rs, static_pie.rs and mremap.rs run them) src/fs/lock.rs            stage 16: fcntl record locks (F_GETLK/F_SETLK/F_SETLKW), deadlock detection src/fpu.rs                stage 15.1: FPU/SSE/AVX on per CPU (CR0/CR4/XCR0), XSAVE or FXSAVE save/restore scripts/hvc0-session.txt  the commands run-headless.sh feeds into hvc0 (for `getty -1 /dev/hvc0`)@@ -1564,8 +1573,8 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 7 | `poll` | *pollfd, n, timeout_ms | **stage 15**: -1 = forever; pipes, sockets, ttys, files, epoll fds | | 8 | `lseek` | fd, offset, whence | `SEEK_SET/CUR/END` = 0/1/2; `ESPIPE` on console/pipes | | 9 | `mmap` | addr, len, prot, flags, fd, offset | MAP_PRIVATE/SHARED, MAP_ANONYMOUS, MAP_FIXED; files: private only |-| 10 | `mprotect` | addr, len, prot | PROT_READ 1, PROT_WRITE 2, PROT_EXEC 4 |-| 11 | `munmap` | addr, len |  |+| 10 | `mprotect` | addr, len, prot | PROT_READ 1, PROT_WRITE 2, PROT_EXEC 4; **stage 19**: the vvar page can't be made writable (`EACCES`) |+| 11 | `munmap` | addr, len | **stage 19**: also above the mmap area (the stack, the vDSO), like `mprotect` | | 12 | `brk` | addr | returns the (new) end of the heap; brk(0) = current | | 13 | `rt_sigaction` | sig, *new, *old, 8 | kernel `struct sigaction`; handlers need SA_RESTORER | | 14 | `rt_sigprocmask` | how, *set, *old, 8 | SIG_BLOCK 0 / SIG_UNBLOCK 1 / SIG_SETMASK 2 |@@ -1579,6 +1588,7 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 22 | `pipe` | *[u32; 2] | [read end, write end]; `pipe2` (293) also takes `O_CLOEXEC` | | 23 | `select` | n, *rd, *wr, *ex, *timeval | **stage 15**: fd bitmaps (n <= 1024) | | 24 | `sched_yield` | | |+| 25 | `mremap` | addr, old_len, new_len, flags, new_addr | **stage 19**: grow (in place or, with `MREMAP_MAYMOVE` 1, moved), shrink, `MREMAP_FIXED` 2; mmap regions only; `MREMAP_DONTUNMAP` and old_len 0 are `EINVAL` | | 28 | `madvise` | addr, len, advice | **stage 16**: `MADV_DONTNEED` 4 / `MADV_FREE` 8 free private anonymous pages (they read back as zeros); other known hints return 0; unknown ones `EINVAL`, unmapped memory `ENOMEM` | | 32 | `dup` | fd | lowest free fd | | 33 | `dup2` | old, new | closes `new` first if open |@@ -1631,6 +1641,7 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 91 | `fchmod` | fd, mode | **stage 16** | | 92 | `chown` | path, uid, gid | **stage 17**: -1 = unchanged; stored on ramfs and ext2 (FAT/exFAT: `EPERM` unless nothing changes); not enforced; `fchown` 93, `lchown` 94 | | 95 | `umask` | mask | **stage 16**: per process (default 022), inherited by fork; returns the old one |+| 96 | `gettimeofday` | *timeval, *timezone | **stage 19**: microseconds of `CLOCK_REALTIME`; the time zone is always 0 (UTC). Usually answered by the vDSO | | 97 | `getrlimit` | resource, *[cur, max] | **stage 16**: = `prlimit64(0, ...)`; `setrlimit` 160 too | | 98 | `getrusage` | who, *rusage | **stage 17**: user/system CPU time (10 ms samples) for `RUSAGE_SELF`, `RUSAGE_CHILDREN`, `RUSAGE_THREAD`; `ru_maxrss` | | 99 | `sysinfo` | *sysinfo | **stage 17**: Linux's 112-byte struct: uptime, memory, process count, "load" = busy fraction since boot |@@ -1662,6 +1673,7 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 170 | `sethostname` | name, len | **stage 17**: what `uname` reports; root only, at most 64 bytes; `setdomainname` 171 | | 186 | `gettid` |  | **stage 15**: thread id (= pid for the first thread) | | 200 | `tkill` | tid, sig | **stage 15** |+| 201 | `time` | *time_t or 0 | **stage 19**: seconds since 1970. Usually answered by the vDSO | | 202 | `futex` | *u32, op, val, *timespec, *u32, val3 | **stage 15**: FUTEX_WAIT 0, WAKE 1, WAIT_BITSET 9, WAKE_BITSET 10 (+ PRIVATE 128) | | 204 | `sched_getaffinity` | tid, len, *mask | **stage 16**: one bit per CPU (all of them: we don't pin threads); returns the bytes written (8). `sched_setaffinity` 203 accepts any mask that includes a CPU | | 213 | `epoll_create` | size | **stage 15**: = epoll_create1(0) |@@ -1669,8 +1681,8 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 218 | `set_tid_address` | *u32 | **stage 15**: cleared + futex-woken when this thread exits; returns the tid | | 222 | `timer_create` | clock, *sigevent, *id | **stage 17**: `CLOCK_REALTIME`/`MONOTONIC`/`BOOTTIME`; `SIGEV_SIGNAL`, `SIGEV_NONE`, `SIGEV_THREAD_ID`; signals carry `SI_TIMER` and `si_value` | | 223 | `timer_settime` | id, flags, *new, *old | **stage 17**: `TIMER_ABSTIME`; periodic; `timer_gettime` 224, `timer_getoverrun` 225, `timer_delete` 226 |-| 228 | `clock_gettime` | clock, *timespec | microsecond resolution (TSC calibrated against the PIT). `CLOCK_MONOTONIC` (1) counts from boot; since stage 15 `CLOCK_REALTIME` (0) is real Unix time (the CMOS clock read once at boot, plus uptime); **stage 17**: `CLOCK_PROCESS_CPUTIME_ID` (2) and `CLOCK_THREAD_CPUTIME_ID` (3), in 10 ms steps |-| 229 | `clock_getres` | clock, *timespec | **stage 17**: 1 µs for the wall clocks, 10 ms for the CPU-time ones |+| 228 | `clock_gettime` | clock, *timespec | microsecond resolution (**stage 19**: nanoseconds, and usually answered by the vDSO without a system call) (TSC calibrated against the PIT). `CLOCK_MONOTONIC` (1) counts from boot; since stage 15 `CLOCK_REALTIME` (0) is real Unix time (the CMOS clock read once at boot, plus uptime); **stage 17**: `CLOCK_PROCESS_CPUTIME_ID` (2) and `CLOCK_THREAD_CPUTIME_ID` (3), in 10 ms steps |+| 229 | `clock_getres` | clock, *timespec | **stage 17**: 1 µs for the wall clocks (1 ns since stage 19), 10 ms for the CPU-time ones | | 230 | `clock_nanosleep` | clock, flags, *timespec, *remaining | **stage 15.1**: `CLOCK_REALTIME`, `MONOTONIC` or `BOOTTIME`; flag `TIMER_ABSTIME` (1) sleeps until that time. musl's `nanosleep` and `sleep` use this | | 231 | `exit_group` | code | ends the process: every thread (stage 15; `exit`, 60, ends one thread) | | 232 | `epoll_wait` | epfd, *epoll_event, max, timeout_ms | **stage 15**: level-triggered; EPOLLONESHOT supported |@@ -1698,6 +1710,7 @@ Numbers are Linux's wherever Linux has the same call, so man pages and | 293 | `pipe2` | *[u32; 2], flags | flags: O_CLOEXEC | | 295 | `preadv` | fd, *iovec, n, offset | **stage 17**: the file position doesn't move; `pwritev` 296 | | 302 | `prlimit64` | pid, resource, *new, *old | **stage 16**: limits are per process and inherited. Real numbers where the kernel has a real limit: `RLIMIT_NOFILE` 64 (the fd table), `RLIMIT_STACK` (the user stack), `RLIMIT_NPROC`; the rest unlimited. Lowering works; raising a hard limit past the real one is `EPERM` |+| 309 | `getcpu` | *cpu, *node, 0 | **stage 19**: the CPU we're on (node 0); `sched_getcpu()`. Usually answered by the vDSO with `rdtscp` | | 318 | `getrandom` | buf, len, flags | random bytes from virtio-rng (the host's `/dev/urandom`), up to 64 KiB per call; never blocks, flags ignored. Without virtio-rng: a TSC-seeded xorshift (not secure) | | 332 | `statx` | dirfd, path, flags, mask, *statx | **stage 17**: the 256-byte `struct statx`, from the same data as `stat` | | 500 | `spawn` | path, argv, argv_len, *SpawnOptions | **ours**: start a program, returns its pid (fork + exec in one step, like `posix_spawn`; the kernel's first shell and `getty` use it, the shell uses fork + exec) |@@ -4700,6 +4713,255 @@ Doesn't (yet):   `/proc/mounts` (the `mounts` call 501 still does that job), and   `/proc/self` is a directory, not a symlink. +## The vDSO, static PIE and mremap (stage 19)++Stage 19 is about running ordinary C programs, built with a normal C+library (**musl**), faster and with fewer special tricks. Three things:++1. the **vDSO**, so asking for the time doesn't need a system call;+2. **static PIE** programs, which is what `musl-gcc -static-pie` makes;+3. **mremap**, so `realloc` of a big block doesn't copy it.++Before those, two commits fixed old problems: the `threads_and_futexes`+flake and a list from Clarke's review of 0.18.0. They're described at+the end ("The flake" and "Small fixes from Clarke's probe").++The C programs are compiled by `build.rs` with `musl-gcc`, which comes+in the `musl-tools` package: `sudo apt install musl-tools`.++### Why a system call for the time is too slow++A program asks "what time is it?" all the time: for every log line,+every timeout, every frame of a game. Until now each of those was a+system call: switch to the kernel, save registers, look up the call,+read the clock, switch back. Reading the clock is the cheap part.++The CPU has a counter that anyone may read with one instruction,+`rdtsc` (the **time stamp counter**, TSC). The kernel already turns it+into time (`src/time.rs`): at boot it measures how many TSC ticks there+are per millisecond, and from then on++    nanoseconds = (tsc_now - tsc_at_boot) * mult >> 32++where `mult` is "nanoseconds per tick" as a fixed-point number (a whole+number scaled up by 2^32, because the kernel avoids floating point). A+program could do this sum itself, *if* it knew `tsc_at_boot` and `mult`.+That's what the vDSO is for.++### The vDSO: a library the kernel gives every program++**vDSO** means "virtual dynamic shared object": a tiny shared library+(`.so`) that isn't a file anywhere. Its source is `vdso/vdso.c`;+`build.rs` compiles it (with the linker script `vdso/vdso.lds`) into a+one-page ELF file, and the kernel embeds that with `include_bytes!`+(`src/vdso.rs`). Every process gets it mapped into its address space,+and the kernel tells the program where with an entry in the auxiliary+vector (the list of facts on the initial stack, stage 14):+`AT_SYSINFO_EHDR` = the address of the vDSO's ELF header.++musl looks there at start-up, finds the function `__vdso_clock_gettime`+in the vDSO's symbol table (with the version name `LINUX_2.6`, which is+what Linux uses), and from then on `clock_gettime()` calls it instead of+making a system call. The vDSO has four functions, the same as Linux's:++| Function | What it does | Without the fast path |+|----------|--------------|-----------------------|+| `__vdso_clock_gettime` | `CLOCK_REALTIME`, `CLOCK_MONOTONIC` and their `_COARSE`/`_RAW` versions from the TSC | system call 228; other clocks (CPU time, `BOOTTIME`) always use it |+| `__vdso_gettimeofday` | the wall clock in microseconds | system call 96 |+| `__vdso_time` | the wall clock in seconds | system call 201 |+| `__vdso_getcpu` | which CPU am I on? (`sched_getcpu()`) via `rdtscp` | system call 309 |++The three system calls 96, 201 and 309 are new too: the vDSO needs+something to fall back on, and old programs call them directly.++### The vvar page and its seqlock++The numbers the vDSO needs (`tsc_at_boot`, `mult`, the wall-clock time+at boot, and a "you may use the TSC" flag) live in the **vvar page**:+one 4 KiB page that the kernel writes and every process can only+*read*. It sits right in front of the vDSO's code, so the vDSO finds it+at "my own address minus 4096" without any relocation. Its layout is+`struct vvar` in `vdso/vdso.c`, and the same thing as `VVar` in+`src/vdso.rs`; the two must match.++But what if the kernel changes the numbers (say, after calibrating the+clock again) while a program is halfway through reading them? It could+get a new `mult` with an old `tsc_at_boot` and compute nonsense. The+fix is a **seqlock** ("sequence lock"), which costs readers almost+nothing:++- The page starts with a counter, `seq`. The kernel makes it **odd**+  before it changes anything and **even** again afterwards.+- A reader reads `seq`, then the numbers, then `seq` again. If the+  first value was odd, or the two differ, a write happened in between:+  it simply tries again.++Readers never block and never write anything, which is why the page can+be read-only. `mprotect` refuses to make it writable (`EACCES`), and a+program that writes to it anyway gets SIGSEGV.++The vDSO's code and the vvar page are the same physical frames in every+process. They're mapped at a fixed place just below the stack's guard+zone (`VVAR_ADDR`, then `VDSO_ADDR` 4 KiB above it; 64 KiB are kept for+them), and `mmap` never hands out that range. They survive `fork` like+any other page and are mapped again by `execve`.++### Can we trust the TSC?++On some older CPUs the TSC changes speed with the CPU's clock (to save+power), so ticks don't mean time. Newer CPUs promise an **invariant**+TSC, shown by a CPUID bit, and Linux only uses the vDSO fast path then.++Surprise: QEMU without KVM (TCG, what the tests run on) **doesn't offer+that bit**, not even with `-cpu max,+invtsc`; it prints a warning and+leaves it off. Its TSC is still steady, because it's just QEMU's own+clock. So Rustbelt trusts the TSC when it's invariant **or** when CPUID+says "you're running under a hypervisor". The rule is+`tsc_trusted()` in `src/vdso.rs`; on a real old CPU without the bit the+vDSO just makes the system call. For the tests,+`vdso::force_syscall(true)` switches the fast path off, so the fallback+gets tested too.++While at it, the kernel clock went from microseconds to **nanoseconds**+(`clock_getres` now says 1 ns), so the vDSO and the system call give the+same answer to the nanosecond.++### Static PIE: programs that can live anywhere++Until now every program was an **ET_EXEC** file: linked to run at one+fixed address, `0x80_0040_0000`, which Rustbelt's linker script+(`user/link.ld`) chooses. A **PIE** ("position-independent executable",+ELF type **ET_DYN**) can be loaded anywhere. It carries a list of+*relocations*: places in its data that hold addresses and must be+adjusted by "where I actually got loaded". Normally the dynamic loader+(`ld.so`) does that, but a **static PIE** fixes itself up: musl's start+code (`_dlstart`) applies its own relocations before `main`.++What the kernel does for one (`src/elf.rs`):++- It picks a **random base**, a whole number of pages above+  `USER_START + 4 MiB` (up to 1 GiB higher), and loads every segment at+  "base + the address in the file". A different base each run makes+  attacks that guess addresses harder (ASLR).+- It tells the program where it ended up: `AT_PHDR` (its program+  headers), `AT_ENTRY` (its entry point) and `AT_BASE` = 0 (no dynamic+  loader, as on Linux).+- `PT_GNU_STACK` says whether the stack may be executable (almost never;+  respected now), `PT_TLS` is the template for thread-local variables+  (musl finds it through `AT_PHDR`), and `PT_GNU_RELRO` marks data a+  dynamic loader would make read-only after relocating; both are+  accepted.+- A program that needs a dynamic loader (**PT_INTERP**, made by plain+  `musl-gcc` without `-static`) can't run here: `execve` says `ENOEXEC`.++`user/piehello.c` checks its own relocations and auxv entries;+`user/piethreads.c` runs pthreads, `__thread` variables, stdio and+`ttyname` as a static PIE.++**Why not a plain `musl-gcc -static` program?** That makes an ET_EXEC+linked at `0x400000`, the usual Linux address. Rustbelt's user space is+only one 512 GiB slot of the address space, starting at `0x80_0000_0000`+(the rest is the kernel's), so `0x400000` is simply not available, and+an ET_EXEC can't be moved. To build a classic ET_EXEC anyway,+`build.rs` links it at our address: `-static -no-pie -nostartfiles`+with musl's `crt1.o`/`crti.o`/`crtn.o` named by hand,+`-Wl,-Ttext-segment=0x8000400000`, `-Wl,--defsym=_DYNAMIC=0x8000400000`+and `-Wl,--no-relax`. (Debian's musl start files are compiled as+position-independent code, which is why the extra flags are needed.)+`vdsotest` is built that way. For your own programs, `-static-pie` is+much easier and just works.++### mremap: growing memory without copying++`mremap(old, old_len, new_len, flags, new_addr)` makes an `mmap`+region bigger or smaller. musl's `realloc` uses it for big blocks+(which get their own `mmap`): without it, growing a 4 MiB block meant+allocating a new one and copying all 4 MiB.++- **Smaller**: the end is unmapped.+- **Bigger, with free space right behind it**: the region just grows.+- **Bigger, no room**: with `MREMAP_MAYMOVE` the kernel finds a new+  place, otherwise it's `ENOMEM`. Moving copies nothing: the page-table+  entries are taken out at the old address and put back at the new one+  (`Mm::remap` in `src/mm.rs`), so the very same frames, still shared+  copy-on-write with a forked child if they were, just show up somewhere+  else.+- `MREMAP_FIXED` says exactly where to move it.++Only `mmap` regions can be resized (not the program, the heap, the+stack, the vvar page or the vDSO). `user/remaptest.c` checks all of+this, plus `realloc` up to 32 MiB.++### The flake++For a few stages, about one run in 30 of `tests/libc.rs`'s+`threads_and_futexes` failed, and rerunning passed. Running the+`threads` program 150 times per boot made it hang every time, which+gave something to look at. It was three separate races between CPUs:++1. **clone set up "wake my joiner when I exit" too late.** The new+   thread was already running when `clone` stored that address+   (`CLONE_CHILD_CLEARTID`). A very short thread could finish first, so+   nobody woke the thread waiting in `pthread_join`.+2. **A deadlock between a lock and a TLB shootdown.** One CPU held a+   process's memory lock and waited for every other CPU to say "I've+   forgotten those pages" (a TLB shootdown, stage 14). Another CPU of+   the same process wanted that lock and spun for it *with interrupts+   off*, so it never heard the request. Each waited for the other. Now+   a CPU waiting for that lock answers shootdowns while it waits.+3. **A lost futex wake.** `futex_wait` read the futex value and *then*+   joined the waiting list; a wake in between was lost. It now reads the+   value while holding the list's lock, as Linux does.++`tests/threads_stress.rs` runs `threads` 40 times per boot and prints+every thread and futex waiter if one hangs.++### Small fixes from Clarke's probe++Clarke's tests of 0.18.0 found a few differences from Linux:++- **Device numbers.** Every `/dev` file now has Linux's major/minor+  numbers (`st_rdev`: `/dev/null` is 1:3, `/dev/zero` (new) 1:5,+  `/dev/pts/N` 136:N, ...), and `fstat` of an open device gives the same+  answer as `stat` of its `/dev` path. musl's `ttyname()` (and so `tty`)+  compares exactly those numbers, so it works now.+- **Unix sockets and fds**: a stream read takes the bytes in front of+  an fd-carrying message together with it, like Linux (see the note in+  the stage 18 chapter).+- **`/proc/self/cwd`** is a link to the current directory.+- **Autobind** starts at a random name, like Linux.++### Testing it++| Test file | Program it runs | What it shows |+|-----------|-----------------|---------------|+| `tests/vdso.rs` | `vdsotest` | the vDSO and the system calls agree and never go backwards; `--trace` shows no clock system calls at all; it works after fork and exec |+| `tests/static_pie.rs` | `piehello`, `piethreads`, `dynhello` | static PIEs load at random bases, threads and stdio work, PT_INTERP is refused |+| `tests/mremap.rs` | `remaptest` | every `mremap` case, and `realloc` |+| `tests/threads_stress.rs` | `threads` | the flake stays fixed |++From the shell: `vdsotest`, `piehello`, `remaptest`.++### What stage 19 does and doesn't do++Does: everything above, with Linux's numbers and names.++Doesn't (yet):++- **No dynamic loader**: programs with `PT_INTERP` (dynamically linked)+  get `ENOEXEC`. Use `-static-pie`.+- **Stock `musl-gcc -static`** (ET_EXEC at `0x400000`) can't run; see+  above for the flags that make an ET_EXEC that can.+- **`timerfd`** isn't there yet: it needs a new kind of file that `poll`+  and `epoll` can wait on, and the timers (stage 17) only tick every+  ~55 ms anyway. Next on the list.+- **`mremap`**: no `MREMAP_DONTUNMAP`, no `old_len` 0 (a second view of+  a shared mapping), and only `mmap` regions.+- **The clock isn't adjusted**: no NTP, no `settimeofday`; the vvar page+  is written once at boot.+- **`/dev/tty`** opened as a file `fstat`s as the terminal it stands for+  (like `/dev/pts/0`), not as `/dev/tty` itself.+ ## Roadmap  Done: all 12 chapters of the 2nd edition, plus preemptive kernel threads,@@ -4755,6 +5017,7 @@ timers, user ids), and Unix sockets with `/proc/self` (stage 18). - [x] libc batch 1 (stage 16): `rt_sigsuspend`/`rt_sigpending`/`rt_sigtimedwait`/`rt_sigqueueinfo`, queued real-time signals, `sigaltstack`, per-process `umask` and rlimits (`prlimit64`), `fcntl` record locks with deadlock detection and honest `F_GETFL`, `truncate`/`ftruncate`/`fsync`/`chmod`/`utimensat` with real file times, `prctl`, `sched_getaffinity`, `madvise`, robust futex lists, subreapers and self-reaping orphans, `--trace` naming unknown calls by number — [man 2 sigsuspend](https://man7.org/linux/man-pages/man2/sigsuspend.2.html), [man 2 fcntl](https://man7.org/linux/man-pages/man2/fcntl.2.html), [man 2 sigaltstack](https://man7.org/linux/man-pages/man2/sigaltstack.2.html), [man 2 prlimit](https://man7.org/linux/man-pages/man2/prlimit.2.html), [man 2 get_robust_list](https://man7.org/linux/man-pages/man2/get_robust_list.2.html) - [x] coreutils batch 2 (stage 17): `link`/`symlink` and their `*at` forms, native `rename` of full directories and `RENAME_NOREPLACE`, the `chown` family with owners on ramfs and ext2, `statfs`/`fstatfs`/`statx` with Linux's magic numbers and a real `st_dev`/`st_nlink`, Linux's `sysinfo` (99), `times`/`getrusage`/CPU-time clocks/`clock_getres`, `getsockname`/`getpeername`/`getsockopt`/`shutdown` and remembered socket options, `alarm`/`setitimer`/POSIX timers, real (unenforced) user and group ids, nice values, `sethostname`, `flock`, `preadv`/`pwritev`, `sendfile`, and a `getdents64` that `rm -r` can trust — [man 2 link](https://man7.org/linux/man-pages/man2/link.2.html), [man 2 statfs](https://man7.org/linux/man-pages/man2/statfs.2.html), [man 2 timer_create](https://man7.org/linux/man-pages/man2/timer_create.2.html), [man 7 credentials](https://man7.org/linux/man-pages/man7/credentials.7.html), [man 2 flock](https://man7.org/linux/man-pages/man2/flock.2.html) - [x] Unix sockets (stage 18): `AF_UNIX` stream/dgram/seqpacket with path, abstract and unnamed names, `socketpair`, `sendmsg`/`recvmsg` with `SCM_RIGHTS` (in-flight fds, `MSG_CTRUNC`, `MSG_CMSG_CLOEXEC`) and `SCM_CREDENTIALS`, `SO_PEERCRED`/`SO_PASSCRED`, real buffer limits with backpressure, poll/epoll with `POLLRDHUP`/`POLLHUP`, and `/proc/self/fd` + `/proc/self/exe` — [man 7 unix](https://man7.org/linux/man-pages/man7/unix.7.html), [man 3 cmsg](https://man7.org/linux/man-pages/man3/cmsg.3.html), [man 2 sendmsg](https://man7.org/linux/man-pages/man2/sendmsg.2.html), [man 5 proc](https://man7.org/linux/man-pages/man5/proc.5.html), [Wayland protocol: wire format](https://wayland.freedesktop.org/docs/html/ch04.html)+- [x] The vDSO, static PIE and mremap (stage 19): `__vdso_clock_gettime`/`gettimeofday`/`time`/`getcpu` with a seqlocked vvar page and `AT_SYSINFO_EHDR`, a nanosecond kernel clock, ET_DYN programs without PT_INTERP at a random base (`AT_PHDR`/`AT_ENTRY`/`AT_BASE`, `PT_GNU_STACK`/`PT_TLS`/`PT_GNU_RELRO`), musl-gcc C programs in the build, `mremap` with `MREMAP_MAYMOVE`/`MREMAP_FIXED`, the threads flake fixed, Linux device numbers and `ttyname` — [man 7 vdso](https://man7.org/linux/man-pages/man7/vdso.7.html), [man 2 mremap](https://man7.org/linux/man-pages/man2/mremap.2.html) - [ ] The kernel/OS split and a musl port (next; waits for Tyler's go; see the missing-syscall list in "musl prerequisites") - [ ] A Hyprland-style tiling compositor on the framebuffer (later) 

@@ -1,6 +1,9 @@ //! Build script, two jobs: //!-//! 1. Compile the user programs in user/ into ELF executables in OUT_DIR.+//! 0. Compile the vDSO (vdso/, stage 19) into OUT_DIR/vdso.so.+//!+//! 1. Compile the user programs in user/ into ELF executables in OUT_DIR+//!    (stage 19: also C programs against musl, see `MUSL_PROGRAMS`). //!    They're compiled with plain `rustc` for the built-in //!    `x86_64-unknown-none` target (prebuilt `core`, so no build-std //!    needed), as static executables (`relocation-model=static`) linked at a@@ -87,6 +90,27 @@ const PROGRAMS: &[&str] = &[ /// plain Linux binaries that check Rustbelt speaks the Linux ABI. const C_PROGRAMS: &[&str] = &["rawabi"]; // stage 15.1 +/// Stage 19: C programs built against a real C library, musl, with the+/// `musl-gcc` wrapper (Debian/Ubuntu: `sudo apt install musl-tools`).+#[derive(Clone, Copy)]+enum Musl {+    /// Stock `musl-gcc -static-pie`: a position-independent executable.+    StaticPie,+    /// A classic static ET_EXEC. It must live at Rustbelt's user base, so+    /// the plain `musl-gcc -static` recipe (linked at 0x400000) needs help:+    /// see `build_musl_program`.+    StaticExec,+    /// Plain dynamic linking (PT_INTERP): Rustbelt must refuse it.+    Dynamic,+}+const MUSL_PROGRAMS: &[(&str, Musl)] = &[+    ("vdsotest", Musl::StaticExec),+    ("piehello", Musl::StaticPie),+    ("piethreads", Musl::StaticPie),+    ("dynhello", Musl::Dynamic),+    ("remaptest", Musl::StaticPie),+];+ const DISK_SECTORS: u64 = 64 * 1024 * 1024 / 512; const PART_START: u64 = 2048; /// Leave 1 MiB at the end too (the backup GPT lives in the last 33 sectors).@@ -103,13 +127,18 @@ fn main() {     println!("cargo:rerun-if-changed=disk4");     println!("cargo:rerun-if-changed=build");     println!("cargo:rerun-if-changed=build.rs");+    println!("cargo:rerun-if-changed=vdso"); +    build_vdso(&manifest, &out_dir);     for program in PROGRAMS {         build_program(&user_dir, &out_dir, program);     }     for program in C_PROGRAMS {         build_c_program(&user_dir, &out_dir, program);     }+    for &(program, kind) in MUSL_PROGRAMS {+        build_musl_program(&user_dir, &out_dir, program, kind);+    }     build_disk(&manifest, &out_dir);     build_ext2_disks(&manifest, &out_dir);     build_exfat_disks(&manifest, &out_dir);@@ -150,6 +179,68 @@ fn build_c_program(user_dir: &Path, out_dir: &Path, program: &str) {     assert!(status.success(), "building C program {program} failed"); } +/// The vDSO (stage 19, see vdso/vdso.c): a tiny shared library the kernel+/// maps into every process; src/vdso.rs embeds the result with+/// `include_bytes!`. -fPIC and no libc: it has to work at any address and+/// without anything else loaded. The linker script vdso/vdso.lds lays it+/// out as one page (ELF header first) and names the version LINUX_2.6;+/// --hash-style=both writes the old SysV hash table too, which musl needs.+fn build_vdso(manifest: &Path, out_dir: &Path) {+    let cc = env::var("CC").unwrap_or_else(|_| "cc".into());+    let dir = manifest.join("vdso");+    let status = Command::new(&cc)+        .args(["-O2", "-fPIC", "-shared", "-nostdlib", "-ffreestanding", "-fno-stack-protector"])+        .args(["-fno-builtin", "-fno-asynchronous-unwind-tables", "-Wall", "-Wextra", "-s"])+        .arg(format!("-Wl,-T,{}", dir.join("vdso.lds").display()))+        .args(["-Wl,--hash-style=both", "-Wl,-soname=linux-vdso.so.1", "-Wl,--no-undefined"])+        .args(["-Wl,-z,max-page-size=4096", "-Wl,--build-id=none", "-Wl,-z,noexecstack"])+        .arg("-o")+        .arg(out_dir.join("vdso.so"))+        .arg(dir.join("vdso.c"))+        .status()+        .expect("failed to run the C compiler (install gcc)");+    assert!(status.success(), "building the vDSO failed");+}++/// A C program linked with musl (stage 19).+///+/// StaticExec: the ELF type the stock recipe makes (ET_EXEC), but moved to+/// Rustbelt's user base (0x80_0000_0000; the kernel owns the low addresses)+/// with `-Ttext-segment`. Debian's musl start files are compiled as+/// position-independent code, which then needs three more tricks: musl's+/// own crt1.o instead of gcc's crtbegin (which can't reach a far-away+/// `__cxa_finalize`), `--no-relax`, and a dummy `_DYNAMIC` (crt1's `_start`+/// takes its address, which is 0 -- too far away -- in a static program).+/// Static PIEs need none of this, which is why stage 19 added them.+fn build_musl_program(user_dir: &Path, out_dir: &Path, program: &str, kind: Musl) {+    let musl = Path::new("/usr/lib/x86_64-linux-musl");+    let mut cmd = Command::new("musl-gcc");+    cmd.args(["-O2", "-Wall", "-Wextra", "-pthread", "-Wl,--build-id=none"]);+    let source = user_dir.join(format!("{program}.c"));+    match kind {+        Musl::StaticPie => {+            cmd.arg("-static-pie").arg(&source);+        }+        Musl::Dynamic => {+            cmd.args(["-fPIE", "-pie"]).arg(&source);+        }+        Musl::StaticExec => {+            cmd.args(["-static", "-no-pie", "-nostartfiles"])+                .args(["-Wl,-Ttext-segment=0x8000400000", "-Wl,--defsym=_DYNAMIC=0x8000400000", "-Wl,--no-relax"])+                .arg(musl.join("crt1.o"))+                .arg(musl.join("crti.o"))+                .arg(&source)+                .arg(musl.join("crtn.o"));+        }+    }+    let status = cmd+        .arg("-o")+        .arg(out_dir.join(program))+        .status()+        .expect("failed to run musl-gcc (install it: `sudo apt install musl-tools`)");+    assert!(status.success(), "building musl program {program} failed");+}+ /// Find a tool in PATH or the sbin directories (often not in a user's PATH). fn tool(name: &str) -> PathBuf {     let path = env::var("PATH").unwrap_or_default();@@ -201,7 +292,8 @@ fn build_disk(manifest: &Path, out_dir: &Path) {     }     // ...and the user programs to /bin.     mtools("mmd", &["::/bin"]);-    for program in PROGRAMS.iter().chain(C_PROGRAMS) {+    let musl = MUSL_PROGRAMS.iter().map(|(name, _)| name);+    for program in PROGRAMS.iter().chain(C_PROGRAMS).chain(musl) {         mtools("mcopy", &[out_dir.join(program).to_str().unwrap(), &format!("::/bin/{program}")]);     } 

@@ -88,7 +88,7 @@ set -e # Lines the shell self-test must have printed (only for a plain boot). if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     expect=(-        "Rustbelt 0.18.0 (stage 18: Unix sockets) x86_64" # ver+        "Rustbelt 0.19.0 (stage 19: vDSO and static PIE) x86_64" # ver         "[smp] 4 CPUs online"                          # smp.rs started 3 more cores         "[fpu] "                                       # XSAVE (-cpu max) or FXSAVE (qemu64)         "B:  /mnt/disk0   fat32"                       # drives@@ -142,8 +142,8 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         "[virtio] hvc0: virtio-console at 00:05.0"         "[virtio] gpu: virtio-gpu 2D at 00:03.0, display 1280x800, framebuffer 1024x768"         "[virtio] fbcon: text console on the framebuffer, 128x48 characters"-        "crw-rw-rw-      <DEV>  fb0"                   # ls -l /dev-        "crw-rw-rw-      <DEV>  hvc0"+        "crw-rw----      <DEV>  fb0"                   # ls -l /dev (Linux modes since stage 19)+        "crw-------      <DEV>  hvc0"         "tile: 4 windows on 1024x768, console 61x46"   # the tiling demo         "console: full screen, 128x48 characters"      # tile -r         "[forktest] all checks passed"                 # stage 14: processes@@ -159,7 +159,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         "[nbtest] all checks passed"         "[fputest] all checks passed"                  # stage 15.1: musl prerequisites         "[pathtest] all checks passed"-        "rawabi: uname: Rustbelt 0.18.0"               # a no-libc gcc binary+        "rawabi: uname: Rustbelt 0.19.0"               # a no-libc gcc binary         "rawabi: all checks passed"         "sh: nosuchprogram: command not found"         "cd: there is no drive Q:"@@ -169,11 +169,11 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     for line in "${expect[@]}"; do         grep -qF -- "$line" "$LOG" || { echo "[run-headless] MISSING: $line"; missing=1; }     done-    # Pipe results: `... | grep -i EXT | wc -l` -> 1, `dir bin | wc -l` -> 39 (the programs),+    # Pipe results: `... | grep -i EXT | wc -l` -> 1, `dir bin | wc -l` -> 44 (the programs),     # `cat C:/chunks.txt | wc -l` (copied from E:) -> 3,     # `type D:/COW.TXT | wc -l` (copied from E:) -> 4, and     # `cat D:/seq.txt | wc -l` (downloaded by wget) -> 20000.-    for n in 1 39 3 4 20000; do+    for n in 1 44 3 4 20000; do         grep -qx -- "$n" "$LOG" || { echo "[run-headless] MISSING pipe result: $n"; missing=1; }     done     # No memory leaks: once the shell and all ~60 programs it ran have@@ -184,7 +184,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         echo "[run-headless] MISSING: frames back at the baseline after the shell"; missing=1     fi     # The other direction: the host fetched a page from httpd in the guest.-    if grep -qF "Hello from Rustbelt! Rustbelt 0.18.0" "$CURL_OUT" 2>/dev/null; then+    if grep -qF "Hello from Rustbelt! Rustbelt 0.19.0" "$CURL_OUT" 2>/dev/null; then         echo "[run-headless] host curl got httpd's page: $(head -1 "$CURL_OUT")"     else         echo "[run-headless] MISSING: host curl through hostfwd to httpd"; missing=1@@ -196,7 +196,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     # drop the '\r's before comparing whole lines.     tr -d '\r' < "$HVC_OUT" > "$HVC_OUT.txt" 2>/dev/null && mv "$HVC_OUT.txt" "$HVC_OUT"     # The shell on hvc0 ran the host's commands and answered over hvc0.-    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.18.0 (stage 18" \+    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.19.0 (stage 19" \                 "$ echo hi over hvc0" "hi over hvc0" "fb0" "hvc0"; do         grep -qF -- "$line" "$HVC_OUT" 2>/dev/null || { echo "[run-headless] MISSING on hvc0: $line"; missing=1; }     done

@@ -45,7 +45,7 @@ fputest | grep passed pathtest | grep passed rawabi | grep -v "ok   " TXT-    set -- "Rustbelt 0.18.0 (stage 18: Unix sockets) x86_64" "tile: 4 windows on 1024x768, console 61x46" \+    set -- "Rustbelt 0.19.0 (stage 19: vDSO and static PIE) x86_64" "tile: 4 windows on 1024x768, console 61x46" \         "^C" "(interrupted: SIGINT)" "threads: 4 threads summed 1..=40000000 = 800000020000000" \         "echoserver: epoll: 3 clients, 9 echoes" "[threads] all checks passed" \         "[fputest] all checks passed" "[pathtest] all checks passed" \

@@ -1,5 +1,5 @@ //! A tiny ELF64 loader: just enough to load a statically linked user program-//! (execve and spawn use it).+//! (execve and spawn use it) -- classic or position independent. //! //! An ELF file starts with a header saying what it is (64-bit, little endian, //! x86_64, executable) and where the *program headers* are. Each program@@ -8,6 +8,19 @@ //! the rest (that's .bss), with permissions R/W/X". That's all a loader for a //! static executable has to do; then jump to `entry`. //!+//! Stage 19: **static PIE** ("position-independent executable"). Modern+//! toolchains make PIE files by default (e_type ET_DYN): their addresses+//! start at 0 and the program can run anywhere, because it fixes up its own+//! pointers at start-up (musl's `rcrt1.o` does that). We pick a *load+//! bias* -- a random page-aligned base, so every run lands somewhere else+//! (address space layout randomization) -- add it to every address in the+//! file, and tell the program in its auxiliary vector (AT_PHDR, AT_ENTRY).+//! A PIE file that names a *dynamic loader* (PT_INTERP, e.g.+//! /lib/ld-musl-x86_64.so.1) needs shared libraries we don't have: ENOEXEC.+//! Other program headers we look at: PT_GNU_STACK (may the stack be+//! executable?), PT_TLS and PT_GNU_RELRO (just accepted: the C library+//! sets up thread-local storage and makes RELRO read-only by itself).+//! //! Every number we read comes from an untrusted file, so all offsets and sizes //! are bounds-checked. Reference: `man 5 elf`, https://wiki.osdev.org/ELF @@ -30,9 +43,17 @@ pub enum ElfError {     /// give a page one set of permissions). A linker script problem.     SharedPage,     OutOfMemory,+    /// A dynamically linked program (PT_INTERP): needs a dynamic loader.+    NeedsInterpreter, }  const PT_LOAD: u32 = 1;+const PT_INTERP: u32 = 3;+const PT_TLS: u32 = 7;+const PT_GNU_STACK: u32 = 0x6474_e551;+const PT_GNU_RELRO: u32 = 0x6474_e552;+const ET_EXEC: u16 = 2;+const ET_DYN: u16 = 3; const PF_X: u32 = 1; const PF_W: u32 = 2; @@ -58,8 +79,19 @@ struct Segment {     memsz: u64, } -/// Check the ELF header; return (entry point, PT_LOAD segments).-fn parse(data: &[u8]) -> Result<(u64, impl Iterator<Item = Result<Segment, ElfError>> + '_), ElfError> {+/// What the ELF header and the program headers say.+struct Parsed {+    /// ET_DYN (PIE): may be loaded anywhere.+    relocatable: bool,+    entry: u64,+    /// The PT_LOAD segments, sorted by address.+    segments: Vec<Segment>,+    /// PT_GNU_STACK asks for an executable stack.+    exec_stack: bool,+}++/// Check the ELF header and read the program headers.+fn parse(data: &[u8]) -> Result<Parsed, ElfError> {     if data.len() < 64 {         return Err(ElfError::TooShort);     }@@ -72,9 +104,11 @@ fn parse(data: &[u8]) -> Result<(u64, impl Iterator<Item = Result<Segment, ElfEr     if read_u16(data, 18)? != 0x3e {         return Err(ElfError::NotX86_64); // e_machine = EM_X86_64     }-    if read_u16(data, 16)? != 2 {-        return Err(ElfError::NotExecutable); // e_type = ET_EXEC (static, non-PIE)-    }+    let relocatable = match read_u16(data, 16)? {+        ET_EXEC => false, // a classic static executable: fixed addresses+        ET_DYN => true,   // PIE (stage 19)+        _ => return Err(ElfError::NotExecutable),+    };     let entry = read_u64(data, 24)?;     let phoff = read_u64(data, 32)? as usize;     let phentsize = read_u16(data, 54)? as usize;@@ -83,24 +117,25 @@ fn parse(data: &[u8]) -> Result<(u64, impl Iterator<Item = Result<Segment, ElfEr         return Err(ElfError::BadProgramHeader);     } -    let segments = (0..phnum).filter_map(move |i| {-        let ph = phoff.checked_add(i.checked_mul(phentsize)?)?;-        let segment = (|| {-            Ok(Segment {-                flags: read_u32(data, ph + 4)?,+    let mut parsed = Parsed { relocatable, entry, segments: Vec::new(), exec_stack: false };+    for i in 0..phnum {+        let ph = i.checked_mul(phentsize).and_then(|o| o.checked_add(phoff)).ok_or(ElfError::BadProgramHeader)?;+        let flags = read_u32(data, ph + 4)?;+        match read_u32(data, ph)? {+            PT_LOAD => parsed.segments.push(Segment {+                flags,                 offset: read_u64(data, ph + 8)?,                 vaddr: read_u64(data, ph + 16)?,                 filesz: read_u64(data, ph + 32)?,                 memsz: read_u64(data, ph + 40)?,-            })-        })();-        match read_u32(data, ph) {-            Ok(PT_LOAD) => Some(segment),-            Ok(_) => None, // other kinds (GNU_STACK, ...) we ignore-            Err(e) => Some(Err(e)),+            }),+            PT_INTERP => return Err(ElfError::NeedsInterpreter),+            PT_GNU_STACK => parsed.exec_stack = flags & PF_X != 0,+            PT_TLS | PT_GNU_RELRO => {} // the C library handles these+            _ => {}                     // others (NOTE, PHDR, ...) we ignore         }-    });-    Ok((entry, segments))+    }+    Ok(parsed) }  /// What `load` tells execve about the program (for the auxiliary vector).@@ -115,19 +150,51 @@ pub struct LoadInfo {     pub phoff: u64,     pub phent: u64,     pub phnum: u64,+    /// Load bias: what was added to every address in the file (0 for a+    /// classic ET_EXEC, the random base for a PIE).+    pub bias: u64,+    /// PT_GNU_STACK said the stack may hold code (old programs need it).+    pub exec_stack: bool,+}++/// Where a PIE goes: 4 MiB above the start of user space plus a random+/// number of pages (up to 1 GiB), so the heap after it and mmap from the+/// top still have hundreds of GiB between them.+const PIE_BASE: u64 = USER_START + 0x40_0000;+const PIE_RANDOM_PAGES: u64 = 0x4_0000;++/// A random load bias for a PIE whose lowest segment starts at `low`.+fn pie_bias(low: u64) -> u64 {+    let mut random = [0u8; 8];+    crate::process::random_bytes(&mut random);+    PIE_BASE + (u64::from_le_bytes(random) % PIE_RANDOM_PAGES) * 4096 - (low & !0xfff) }  /// Parse and check everything (headers, segment bounds, permissions) without /// mapping anything: execve calls this *before* it destroys the old program, /// so a bad file can still be reported as an error.-fn plan(data: &[u8]) -> Result<(LoadInfo, Vec<Segment>), ElfError> {-    let (entry, segments) = parse(data)?;+///+/// `random`: pick a random bias for a PIE (else the lowest one, enough to+/// check the file).+fn plan(data: &[u8], random: bool) -> Result<(LoadInfo, Vec<Segment>), ElfError> {+    let Parsed { relocatable, entry, mut segments, exec_stack } = parse(data)?;+    let mut bias = 0;+    if relocatable {+        let low = segments.iter().map(|s| s.vaddr).min().unwrap_or(0);+        if low >= USER_START {+            return Err(ElfError::OutsideUserSpace); // not really position independent+        }+        bias = if random { pie_bias(low) } else { PIE_BASE - (low & !0xfff) };+        for seg in &mut segments {+            seg.vaddr = seg.vaddr.wrapping_add(bias);+        }+    }+    let entry = entry.wrapping_add(bias);     if !(USER_START..USER_END).contains(&entry) {         return Err(ElfError::OutsideUserSpace);     }-    let segments = segments.collect::<Result<Vec<Segment>, ElfError>>()?;     let (phoff, phent, phnum) = (read_u64(data, 32)?, read_u16(data, 54)? as u64, read_u16(data, 56)? as u64);-    let mut info = LoadInfo { entry, end: USER_START, phdr: None, phoff, phent, phnum };+    let mut info = LoadInfo { entry, end: USER_START, phdr: None, phoff, phent, phnum, bias, exec_stack };     // The previous segment's last page and its flags.     let mut previous: Option<(u64, u32)> = None;     for seg in &segments {@@ -137,7 +204,7 @@ fn plan(data: &[u8]) -> Result<(LoadInfo, Vec<Segment>), ElfError> {             return Err(ElfError::BadProgramHeader);         }         // Segments must stay below the user stack and its guard zone.-        if seg.vaddr < USER_START || mem_end > crate::mm::USER_STACK_ZONE_START {+        if seg.vaddr < USER_START || mem_end > crate::mm::MMAP_TOP {             return Err(ElfError::OutsideUserSpace);         }         if seg.memsz == 0 {@@ -166,7 +233,7 @@ fn plan(data: &[u8]) -> Result<(LoadInfo, Vec<Segment>), ElfError> { /// fill every segment (eagerly -- programs are small), and record each as /// a region. pub fn load(data: &[u8], mm: &mut Mm) -> Result<LoadInfo, ElfError> {-    let (info, segments) = plan(data)?;+    let (info, segments) = plan(data, true)?;     let nx_enabled = memory::nx_enabled();     for seg in segments.iter().filter(|s| s.memsz > 0) {         let mem_end = seg.vaddr + seg.memsz;@@ -222,5 +289,5 @@ pub fn load(data: &[u8], mm: &mut Mm) -> Result<LoadInfo, ElfError> { /// Just validate the file (execve before the point of no return, tests). /// Returns the entry point. pub fn check(data: &[u8]) -> Result<u64, ElfError> {-    plan(data).map(|(info, _)| info.entry)+    plan(data, false).map(|(info, _)| info.entry) }

@@ -49,10 +49,11 @@ pub mod syscall; // syscall/sysret entry + system call implementations pub mod syscall_ext; // stage 17's syscalls for coreutils/busybox: links, owners, statfs, ids, timers... pub mod task; // cooperative multitasking: async tasks + executor pub mod trap; // TrapFrame: saved user registers, the common return to ring 3-pub mod time; // microsecond clock (TSC calibrated against the PIT)+pub mod time; // nanosecond clock (TSC calibrated against the PIT) pub mod timer; // alarm, setitimer, POSIX timers: signals at a deadline (stage 17) pub mod thread; // threads + the multi-core scheduler (sleep/wake, wait queues) pub mod tty; // terminals: line discipline, termios, console/hvc0/ptys+pub mod vdso; // stage 19: the vDSO + vvar page, time without a syscall pub mod vga_buffer; // the console: VGA text mode, or fbcon on a GPU pub mod virtio; // modern virtio transport + virtio-rng, -console, -gpu 

@@ -148,6 +148,7 @@ pub fn init_frame_allocator(frame_allocator: BootInfoFrameAllocator) {     EARLY_FRAMES.store(frame_allocator.next as u64, Ordering::Relaxed);     without_interrupts(|| *FRAME_ALLOCATOR.lock() = Some(frame_allocator));     init_kernel_stack_area(); // must happen before any process exists+    crate::vdso::init(); // stage 19: the vDSO's frames, before any process too }  /// Frames given back with `free_frame`, as a linked list threaded through

@@ -8,6 +8,8 @@ //!   heap      from the end of the program up to `brk` (grows with brk()) //!   mmap      what mmap() created: anonymous memory, or a file's bytes //!   stack     1 MiB under USER_STACK_TOP, with a 64 KiB guard zone below+//!   vvar/vdso the kernel's read-only clock page and the vDSO library+//!             (stage 19, vdso.rs): the same frames in every process //! //! **Demand paging.** A region is just a promise. The first access to one //! of its pages finds no page-table entry, the CPU raises a page fault, and@@ -61,8 +63,13 @@ pub const USER_STACK_SIZE: u64 = 1024 * 1024; pub const USER_STACK_GUARD: u64 = 64 * 1024; /// Lowest address of stack + guard: program segments must end below this. pub const USER_STACK_ZONE_START: u64 = USER_STACK_TOP - USER_STACK_SIZE - USER_STACK_GUARD;-/// mmap hands out addresses downwards from here (the heap grows up towards it).-pub const MMAP_TOP: u64 = USER_STACK_ZONE_START;+/// Stage 19: 64 KiB below the guard zone are reserved for the vvar page and+/// the vDSO (see vdso.rs), at the same address in every process.+pub const VDSO_AREA_SIZE: u64 = 64 * 1024;+pub const VVAR_ADDR: u64 = USER_STACK_ZONE_START - VDSO_AREA_SIZE;+/// mmap hands out addresses downwards from here (the heap grows up towards+/// it). Program segments must end below it too.+pub const MMAP_TOP: u64 = VVAR_ADDR;  /// Is user address `addr` in the guard zone under the user stack? pub fn in_user_stack_guard(addr: u64) -> bool {@@ -81,6 +88,10 @@ pub enum Kind {     Heap,     Mmap,     Stack,+    /// The vvar page (stage 19): read-only, mprotect can't make it writable.+    Vvar,+    /// The vDSO's code (stage 19).+    Vdso, }  /// One region: `start..end` (page aligned) with the same permissions and@@ -271,6 +282,10 @@ impl Mm {         if covered < end {             return Err(Errno::ENOMEM);         }+        // Only the kernel writes the vvar page (Linux says EACCES too).+        if prot & PROT_WRITE != 0 && self.vmas.iter().any(|v| v.kind == Kind::Vvar && v.end > start && v.start < end) {+            return Err(Errno::EACCES);+        }         self.split_at(start, end);         for v in self.vmas.iter_mut().filter(|v| v.start >= start && v.end <= end) {             v.prot = prot;@@ -328,6 +343,85 @@ impl Mm {         if covered < end { Err(Errno::ENOMEM) } else { Ok(()) }     } +    /// mremap (stage 19): resize the region `old..old+old_len` to+    /// `new_len` bytes, moving it if `may_move` and it can't grow where it+    /// is, or to `fixed` if given (MREMAP_FIXED). Returns the new address.+    ///+    /// Moving doesn't copy anything: the page-table entries are taken out+    /// at the old address and put back at the new one, so the same frames+    /// (and their copy-on-write and sharing state) simply show up+    /// somewhere else. Only mmap regions can be resized; the program, the+    /// heap (brk owns it), the stack, the vvar page and the vDSO can't.+    pub fn remap(&mut self, old: u64, old_len: u64, new_len: u64, may_move: bool, fixed: Option<u64>) -> Result<u64, Errno> {+        let old_end = old + old_len;+        let new_len = page_up(new_len);+        let vma = self.find(old).ok_or(Errno::EFAULT)?;+        if old_end > vma.end {+            return Err(Errno::EFAULT); // must be inside one region+        }+        if vma.kind != Kind::Mmap {+            return Err(Errno::EINVAL);+        }+        let (vma_end, shared_anon) = (vma.end, vma.shared && vma.file.is_none());+        if let Some(to) = fixed {+            if to % PAGE_SIZE != 0 || !Mm::range_ok(to, to + new_len) || (to < old_end && old < to + new_len) {+                return Err(Errno::EINVAL);+            }+            if new_len < old_len {+                self.unmap(old + new_len, old_end);+            }+            self.unmap(to, to + new_len);+            return self.move_region(old, old + old_len.min(new_len), to, new_len, shared_anon);+        }+        if new_len <= old_len {+            self.unmap(old + new_len, old_end); // shrinking (or nothing)+            return Ok(old);+        }+        let new_end = old + new_len;+        if old_end == vma_end && Mm::range_ok(old, new_end) && !self.overlaps(old_end, new_end) {+            let v = self.vmas.iter_mut().find(|v| v.start <= old && old < v.end).unwrap();+            v.end = new_end; // room right behind it: grow in place+            if shared_anon {+                self.populate(old_end, new_end).map_err(|_| Errno::ENOMEM)?;+            }+            return Ok(old);+        }+        if !may_move {+            return Err(Errno::ENOMEM);+        }+        let to = self.find_free(new_len).ok_or(Errno::ENOMEM)?;+        self.move_region(old, old_end, to, new_len, shared_anon)+    }++    /// Move the region part `from..from_end` to `to` and make it `len`+    /// bytes long there (len >= from_end - from; the rest is new).+    fn move_region(&mut self, from: u64, from_end: u64, to: u64, len: u64, shared_anon: bool) -> Result<u64, Errno> {+        self.split_at(from, from_end);+        let i = self.vmas.iter().position(|v| v.start == from).ok_or(Errno::EFAULT)?;+        let mut vma = self.vmas.remove(i);+        let mut lost = Vec::new();+        for off in (0..from_end - from).step_by(PAGE_SIZE as usize) {+            if let Some((frame, flags)) = memory::user_pte(self.l4, from + off) {+                memory::unmap_user_page(self.l4, from + off);+                memory::flush_user_page(from + off);+                if memory::map_user_frame(self.l4, to + off, frame, flags).is_none() {+                    lost.push(frame); // no frame for a page table: the page is lost+                }+            }+        }+        self.shootdown(); // other CPUs must forget the old addresses+        for frame in lost {+            memory::release_frame(frame);+        }+        vma.start = to;+        vma.end = to + len;+        self.insert(vma);+        if shared_anon && len > from_end - from {+            self.populate(to + (from_end - from), to + len).map_err(|_| Errno::ENOMEM)?;+        }+        Ok(to)+    }+     /// A free range of `len` bytes for mmap: the highest gap below MMAP_TOP     /// (and above the heap).     pub fn find_free(&self, len: u64) -> Option<u64> {@@ -387,6 +481,16 @@ impl Mm {         }         let page = addr & !(PAGE_SIZE - 1);         match memory::user_pte(self.l4, page) {+            // The vvar page or the vDSO: map the kernel's one shared frame+            // (never writable here: a write after mprotect copies it).+            None if matches!(vma.kind, Kind::Vvar | Kind::Vdso) => {+                let frame = crate::vdso::frame_at(page).ok_or(Fault::Unmapped)?;+                memory::share_frame(frame);+                if memory::map_user_frame(self.l4, page, frame, vma.pte_flags(false)).is_none() {+                    memory::release_frame(frame);+                    return Err(Fault::OutOfMemory);+                }+            }             // Not there yet: demand paging.             None => {                 let frame = memory::alloc_zeroed_frame().ok_or(Fault::OutOfMemory)?;

@@ -584,6 +584,14 @@ const AT_CLKTCK: u64 = 17; const AT_SECURE: u64 = 23; const AT_RANDOM: u64 = 25; const AT_EXECFN: u64 = 31;+/// Stage 19: where the vDSO's ELF header is (see vdso.rs).+const AT_SYSINFO_EHDR: u64 = 33;++/// AT_HWCAP on x86_64 is CPUID leaf 1's EDX: the basic feature bits (FPU,+/// TSC, SSE, SSE2, ...), what Linux puts there too.+fn hwcap() -> u64 {+    core::arch::x86_64::__cpuid(1).edx as u64+}  /// Load a program into the empty address space `mm` and build its initial /// stack the way Linux does, so a C library's `_start` finds everything@@ -605,7 +613,10 @@ pub fn load_program(mm: &mut Mm, image: &[u8], argv: &[&[u8]], envp: &[&[u8]], e     // that fills in as it's used.     mm.brk_start = info.end.next_multiple_of(4096);     mm.brk = mm.brk_start;-    mm.insert(Vma::new(USER_STACK_TOP - USER_STACK_SIZE, USER_STACK_TOP, PROT_READ | PROT_WRITE, Kind::Stack));+    let stack_prot = PROT_READ | PROT_WRITE | if info.exec_stack { crate::mm::PROT_EXEC } else { 0 };+    mm.insert(Vma::new(USER_STACK_TOP - USER_STACK_SIZE, USER_STACK_TOP, stack_prot, Kind::Stack));+    // Stage 19: the vvar page and the vDSO, in every program.+    let vdso = crate::vdso::map_into(mm);      let mut sp = USER_STACK_TOP;     let mut push = |mm: &mut Mm, bytes: &[u8], nul: bool| -> Result<u64, ElfError> {@@ -639,10 +650,11 @@ pub fn load_program(mm: &mut Mm, image: &[u8], argv: &[&[u8]], envp: &[&[u8]], e     #[rustfmt::skip]     words.extend([         AT_PHDR, phdr, AT_PHENT, info.phent, AT_PHNUM, info.phnum,-        AT_PAGESZ, 4096, AT_BASE, 0, AT_FLAGS, 0, AT_ENTRY, info.entry,+        AT_PAGESZ, 4096, AT_BASE, 0 /* no dynamic loader */, AT_FLAGS, 0, AT_ENTRY, info.entry,         AT_UID, 0, AT_EUID, 0, AT_GID, 0, AT_EGID, 0,-        AT_PLATFORM, platform, AT_HWCAP, 0, AT_CLKTCK, 100, AT_SECURE, 0,-        AT_RANDOM, random_ptr, AT_EXECFN, execfn_ptr, AT_NULL, 0,+        AT_PLATFORM, platform, AT_HWCAP, hwcap(), AT_CLKTCK, 100, AT_SECURE, 0,+        AT_RANDOM, random_ptr, AT_EXECFN, execfn_ptr, AT_SYSINFO_EHDR, vdso,+        AT_NULL, 0,     ]);     let rsp = (sp - 8 * words.len() as u64) & !0xf;     let bytes: Vec<u8> = words.iter().flat_map(|w| w.to_le_bytes()).collect();

@@ -274,6 +274,7 @@ static AP_READY: AtomicBool = AtomicBool::new(false); /// one. Needs the heap and interrupts on. Returns the number of CPUs. pub fn init() -> usize {     apic::init_bsp();+    crate::vdso::init_cpu(0);     CPU_TO_APIC[0].store(apic::id(), Ordering::Relaxed);     APIC_TO_CPU[apic::id() as usize].store(0, Ordering::Relaxed);     thread::init();@@ -359,6 +360,7 @@ extern "C" fn ap_entry(cpu: u64) -> ! {     interrupts::init_idt();     crate::syscall::init();     apic::init_ap();+    crate::vdso::init_cpu(cpu); // stage 19: getcpu() via rdtscp     let stack = without_interrupts(|| AP_STACK.lock().take()).expect("AP boot stack missing");     AP_READY.store(true, Ordering::SeqCst);     thread::run_ap(cpu, stack) // become this CPU's idle thread

@@ -187,6 +187,16 @@ use crate::fs::{self, InodeKind}; use crate::trap::TrapFrame; use crate::{memory, process, signal, thread}; use alloc::{string::String, sync::Arc, vec::Vec};++/// Print one `--trace` line (and keep a copy if a test asked for one).+macro_rules! trace_out {+    ($($arg:tt)*) => {{+        let line = alloc::format!($($arg)*);+        crate::kprintln!("{}", line);+        trace_keep(&line);+    }};+}+ use core::arch::global_asm; use core::sync::atomic::Ordering; @@ -643,7 +653,7 @@ extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {     let process = thread::current_process();     let trace = process.as_ref().is_some_and(|p| p.tracing());     if trace && (nr == SYS_EXIT || nr == SYS_EXIT_GROUP) {-        crate::kprintln!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64);+        trace_out!("[trace {}] exit({})", process.as_ref().unwrap().pid, a1 as i64);     }     drop(process); // don't hold an Arc while blocking/exiting     // A successful execve throws away the memory its path lived in, so the@@ -802,8 +812,8 @@ extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {     if trace && !interrupted {         let pid = current().map(|p| p.pid).unwrap_or(0);         match exec_line {-            Some(line) if ret == 0 => crate::kprintln!("[trace {}] {}", pid, line),-            _ => crate::kprintln!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret)),+            Some(line) if ret == 0 => trace_out!("[trace {}] {}", pid, line),+            _ => trace_out!("[trace {}] {}", pid, trace_line(nr, [a1, a2, a3, a4], ret)),         }     }     // An interrupted call (-512/-514) is traced *after* signal delivery,@@ -815,12 +825,38 @@ extern "C" fn syscall_dispatch(frame: &mut TrapFrame) {     if let Some(line) = line {         let pid = current().map(|p| p.pid).unwrap_or(0);         match outcome {-            Some(true) => crate::kprintln!("[trace {}] {} = ? (interrupted, restarted)", pid, line.rsplit_once(" = ").map(|l| l.0).unwrap_or(&line)),-            _ => crate::kprintln!("[trace {}] {}", pid, line),+            Some(true) => trace_out!("[trace {}] {} = ? (interrupted, restarted)", pid, line.rsplit_once(" = ").map(|l| l.0).unwrap_or(&line)),+            _ => trace_out!("[trace {}] {}", pid, line),         }     } } +/// Stage 19, for tests: collect the `--trace` lines while this is Some+/// (tests/vdso.rs checks that clock_gettime makes no system call).+static TRACE_COPY: spin::Mutex<Option<String>> = spin::Mutex::new(None);++fn trace_keep(line: &str) {+    x86_64::instructions::interrupts::without_interrupts(|| {+        if let Some(copy) = TRACE_COPY.lock().as_mut() {+            copy.push_str(line);+            copy.push('\n');+        }+    });+}++/// Start (true) or stop (false) collecting `--trace` lines; returns what+/// was collected since the last start.+pub fn collect_trace(on: bool) -> String {+    x86_64::instructions::interrupts::without_interrupts(|| {+        let mut copy = TRACE_COPY.lock();+        let old = copy.take().unwrap_or_default();+        if on {+            *copy = Some(String::new());+        }+        old+    })+}+ /// One syscall in strace style: `open("/mnt/disk0/readme.txt", 0) = 3`. /// Strings and buffers are read back from user memory (after the call, so /// `read` shows what it got), shortened to 40 bytes.@@ -1403,7 +1439,7 @@ fn sys_uname(buf: u64) -> Result<u64, Errno> {         "Rustbelt",                     // sysname (`uname -s`)         host.as_str(),                  // nodename (hostname; sethostname changes it)         env!("CARGO_PKG_VERSION"),      // release (`uname -r`)-        "stage 18: Unix sockets",      // version+        "stage 19: vDSO and static PIE", // version         "x86_64",                       // machine         domain.as_str(),                // domainname     ];@@ -1561,14 +1597,15 @@ fn sys_recvfrom(fd: u64, buf: u64, len: u64, addr_ptr: u64, len_ptr: u64) -> Res /// since 1970, from the CMOS clock at boot) or the MONOTONIC family (1, /// 4 RAW, 6 COARSE, 7 BOOTTIME: time since boot). CPU-time clocks: EINVAL. fn sys_clock_gettime(clock: u64, ts_ptr: u64) -> Result<u64, Errno> {-    let us = match clock {-        0 | 5 => crate::time::realtime_us(), // REALTIME, REALTIME_COARSE-        1 | 4 | 6 | 7 => crate::time::now_us(),+    // Stage 19: nanoseconds, computed exactly like the vDSO does.+    let ns = match clock {+        0 | 5 => crate::time::realtime_ns(), // REALTIME, REALTIME_COARSE+        1 | 4 | 6 | 7 => crate::time::now_ns(),         // Stage 17: CPU time used so far (process / this thread).-        2 | 3 => crate::syscall_ext::cpu_clock_us(clock)?,+        2 | 3 => crate::syscall_ext::cpu_clock_us(clock)? * 1000,         _ => return Err(Errno::EINVAL),     };-    copy_to_user(ts_ptr, &[us / 1_000_000, (us % 1_000_000) * 1000])?;+    copy_to_user(ts_ptr, &[ns / 1_000_000_000, ns % 1_000_000_000])?;     Ok(0) } @@ -1925,9 +1962,14 @@ fn sys_mmap(p: &process::Process, addr: u64, len: u64, prot: u64, flags: u64, fd     Ok(start) } +/// The pages of `addr..addr+len` for munmap/mprotect: `addr` page aligned.+///+/// Stage 19: anywhere in user space, not just below MMAP_TOP (where mmap+/// puts things): the vvar page, the vDSO and the stack are above it, and+/// programs may mprotect/munmap those too. fn page_range(addr: u64, len: u64) -> Result<(u64, u64), Errno> {     let end = addr.checked_add(len).ok_or(Errno::EINVAL)?.next_multiple_of(4096);-    if addr % 4096 != 0 || !crate::mm::Mm::range_ok(addr, end.max(addr + 1)) {+    if addr % 4096 != 0 || addr < crate::memory::USER_START || end.max(addr + 1) > crate::memory::USER_END {         return Err(Errno::EINVAL);     }     Ok((addr, end))

@@ -21,6 +21,10 @@ //! | 98  | getrusage            | CPU time used (self, children, thread)             | //! | 100 | times                | CPU time in clock ticks (100 per second)           | //! | 229 | clock_getres         | how fine each clock is                             |+//! | 96  | gettimeofday         | stage 19: wall clock in µs (vDSO fallback)         |+//! | 201 | time                 | stage 19: wall clock in seconds (vDSO fallback)    |+//! | 309 | getcpu               | stage 19: which CPU am I on? (vDSO fallback)       |+//! | 25  | mremap               | stage 19: grow/shrink/move an mmap region          | //! | 54  | setsockopt           | SO_RCVTIMEO; SO_REUSEADDR & co. accepted          | //! | 55  | getsockopt           | SO_ERROR, SO_TYPE, SO_RCVBUF/SNDBUF, ...           | //! | 51  | getsockname          | our own address and port                           |@@ -112,6 +116,11 @@ pub const SYS_SYMLINKAT: u64 = 266; pub const SYS_PREADV: u64 = 295; pub const SYS_PWRITEV: u64 = 296; pub const SYS_STATX: u64 = 332;+// Stage 19: what the vDSO falls back on.+pub const SYS_GETTIMEOFDAY: u64 = 96;+pub const SYS_TIME: u64 = 201;+pub const SYS_GETCPU: u64 = 309;+pub const SYS_MREMAP: u64 = 25;  /// linkat: link what a symlink points to, not the symlink. pub const AT_SYMLINK_FOLLOW: u64 = 0x400;@@ -143,6 +152,10 @@ pub fn dispatch(nr: u64, a: [u64; 6]) -> Option<Result<u64, Errno>> {         SYS_SYSINFO => sys_sysinfo(a[0]),         SYS_GETRUSAGE => sys_getrusage(a[0] as i64, a[1]),         SYS_CLOCK_GETRES => sys_clock_getres(a[0], a[1]),+        SYS_GETTIMEOFDAY => sys_gettimeofday(a[0], a[1]),+        SYS_TIME => sys_time(a[0]),+        SYS_GETCPU => sys_getcpu(a[0], a[1]),+        SYS_MREMAP => sys_mremap(a[0], a[1], a[2], a[3], a[4]),         // 6. Sockets.         SYS_GETSOCKOPT => sys_getsockopt(a[0], a[1], a[2], a[3], a[4]),         SYS_GETSOCKNAME => with_socket(a[0], |s| Ok(s.local())).and_then(|addr| write_sockaddr(a[1], a[2], addr).map(|_| 0)),@@ -233,6 +246,10 @@ pub fn name(nr: u64) -> Option<&'static str> {         SYS_TIMER_GETOVERRUN => "timer_getoverrun",         SYS_TIMER_DELETE => "timer_delete",         SYS_CLOCK_GETRES => "clock_getres",+        SYS_GETTIMEOFDAY => "gettimeofday",+        SYS_TIME => "time",+        SYS_GETCPU => "getcpu",+        SYS_MREMAP => "mremap",         SYS_FCHOWNAT => "fchownat",         SYS_LINKAT => "linkat",         SYS_SYMLINKAT => "symlinkat",@@ -468,7 +485,7 @@ fn sys_times(buf: u64) -> Result<u64, Errno> { /// clocks move in 10 ms ticks. fn sys_clock_getres(clock: u64, res: u64) -> Result<u64, Errno> {     let ns = match clock {-        0 | 1 | 4 | 5 | 6 | 7 => 1000,+        0 | 1 | 4 | 5 | 6 | 7 => 1, // stage 19: nanoseconds (was microseconds)         2 | 3 => process::CPU_TICK_US * 1000,         _ => return Err(Errno::EINVAL),     };@@ -478,6 +495,63 @@ fn sys_clock_getres(clock: u64, res: u64) -> Result<u64, Errno> {     Ok(0) } +/// gettimeofday(*tv, *tz) (stage 19; the vDSO normally answers it): the+/// wall clock in seconds + microseconds. The time zone is always UTC.+fn sys_gettimeofday(tv: u64, tz: u64) -> Result<u64, Errno> {+    let us = crate::time::realtime_us();+    if tv != 0 {+        copy_to_user(tv, &[us / 1_000_000, us % 1_000_000])?;+    }+    if tz != 0 {+        copy_to_user(tz, &[0u32, 0u32])?;+    }+    Ok(0)+}++/// time(*t) (stage 19): Unix time in whole seconds, also stored at `t`.+fn sys_time(t: u64) -> Result<u64, Errno> {+    let secs = crate::time::realtime_ns() / 1_000_000_000;+    if t != 0 {+        copy_to_user(t, &[secs])?;+    }+    Ok(secs)+}++/// getcpu(*cpu, *node, unused) (stage 19): which CPU we're running on+/// (sched_getcpu() in C). One NUMA node, 0.+fn sys_getcpu(cpu: u64, node: u64) -> Result<u64, Errno> {+    let id = crate::smp::cpu_id() as u32;+    if cpu != 0 {+        copy_to_user(cpu, &[id])?;+    }+    if node != 0 {+        copy_to_user(node, &[0u32])?;+    }+    Ok(0)+}++/// mremap(old, old_len, new_len, flags, new_addr) (stage 19): make an+/// mmap region bigger or smaller. realloc() of a big block uses it, so the+/// bytes don't have to be copied. Flags: MREMAP_MAYMOVE (1) lets the kernel+/// move it if it can't grow where it is; MREMAP_FIXED (2, needs MAYMOVE)+/// moves it to `new_addr`. MREMAP_DONTUNMAP (4) and old_len 0 (a second+/// copy of a shared mapping) aren't supported: EINVAL. The real work is+/// Mm::remap in mm.rs.+fn sys_mremap(old: u64, old_len: u64, new_len: u64, flags: u64, new_addr: u64) -> Result<u64, Errno> {+    const MAYMOVE: u64 = 1;+    const FIXED: u64 = 2;+    let fixed = flags & FIXED != 0;+    if old % 4096 != 0 || flags & !(MAYMOVE | FIXED) != 0 || (fixed && flags & MAYMOVE == 0) {+        return Err(Errno::EINVAL);+    }+    let old_len = old_len.next_multiple_of(4096);+    if old_len == 0 || new_len == 0 || new_len > crate::memory::USER_END {+        return Err(Errno::EINVAL);+    }+    old.checked_add(old_len).filter(|&e| e <= crate::memory::USER_END).ok_or(Errno::EFAULT)?;+    current()?.mm().remap(old, old_len, new_len, flags & MAYMOVE != 0, fixed.then_some(new_addr))+}+ /// getrusage(who, *rusage): 0 = RUSAGE_SELF, -1 = RUSAGE_CHILDREN (reaped /// ones), 1 = RUSAGE_THREAD. Fills the two CPU times and ru_maxrss (our /// resident size now, in KiB); the other 14 counters stay 0.

@@ -10,6 +10,10 @@ //! //! Before calibration (or if it never happens, e.g. in some tests) we fall //! back to counting PIT ticks.+//!+//! Stage 19: the clock counts nanoseconds now (`now_ns`), and the numbers+//! behind it are shared with user space through the vDSO's vvar page (see+//! vdso.rs), so programs can read the time without a system call.  use core::sync::atomic::{AtomicU64, Ordering}; @@ -18,6 +22,12 @@ pub const PIT_TICK_US: u64 = 54_925;  static TSC_PER_MS: AtomicU64 = AtomicU64::new(0); static TSC_START: AtomicU64 = AtomicU64::new(0);+/// Stage 19: nanoseconds per TSC tick, as a fixed-point number with 32+/// bits after the point (so 2^32 means "1 ns per tick"). Multiplying and+/// shifting is much cheaper than dividing, and the vDSO (vdso/vdso.c) does+/// the very same arithmetic in user space -- that's why the formula lives+/// in one place, `tsc_to_ns`, and both sides use it. 0 = not calibrated.+static TSC_MULT: AtomicU64 = AtomicU64::new(0);  fn rdtsc() -> u64 {     unsafe { core::arch::x86_64::_rdtsc() }@@ -26,20 +36,40 @@ fn rdtsc() -> u64 { /// Called with a TSC reading at a PIT tick edge (`start`) and the number of /// ticks it took to reach `end`. pub fn calibrate(start: u64, end: u64, pit_ticks: u64) {-    let per_ms = (end - start) * 1000 / (pit_ticks * PIT_TICK_US);+    let per_ms = ((end - start) * 1000 / (pit_ticks * PIT_TICK_US)).max(1);     // Keep the clock monotonic: the TSC clock starts at the PIT uptime.     let uptime_us = crate::interrupts::ticks() * PIT_TICK_US;     TSC_START.store(end.saturating_sub(uptime_us / 1000 * per_ms), Ordering::Relaxed);-    TSC_PER_MS.store(per_ms.max(1), Ordering::Relaxed);+    TSC_PER_MS.store(per_ms, Ordering::Relaxed);+    TSC_MULT.store(((1_000_000u128 << 32) / per_ms as u128) as u64, Ordering::Relaxed);+    crate::vdso::publish(); // tell user space's clock too+}++/// TSC ticks since `TSC_START` -> nanoseconds. Must match `read_clock` in+/// vdso/vdso.c exactly, or the two clocks would disagree.+pub fn tsc_to_ns(delta: u64, mult: u64) -> u64 {+    ((delta as u128 * mult as u128) >> 32) as u64+}++/// Nanoseconds since boot (CLOCK_MONOTONIC).+pub fn now_ns() -> u64 {+    let mult = TSC_MULT.load(Ordering::Relaxed);+    if mult == 0 {+        return crate::interrupts::ticks() * PIT_TICK_US * 1000;+    }+    tsc_to_ns(rdtsc().saturating_sub(TSC_START.load(Ordering::Relaxed)), mult) }  /// Microseconds since boot. pub fn now_us() -> u64 {-    let per_ms = TSC_PER_MS.load(Ordering::Relaxed);-    if per_ms == 0 {-        return crate::interrupts::ticks() * PIT_TICK_US;-    }-    (rdtsc() - TSC_START.load(Ordering::Relaxed)) * 1000 / per_ms+    now_ns() / 1000+}++/// What the vDSO needs to compute the time by itself: (TSC at time 0,+/// ns-per-tick multiplier (0 = not calibrated), Unix time at boot in ns).+pub fn clock_params() -> (u64, u64, u64) {+    let boot = boot_realtime_ns();+    (TSC_START.load(Ordering::Relaxed), TSC_MULT.load(Ordering::Relaxed), boot) }  /// The PIT tick by which `us` microseconds from now will surely have@@ -53,9 +83,9 @@ pub fn deadline_after_us(us: u64) -> u64 { // Wall-clock time (CLOCK_REALTIME) // --------------------------------------------------------------------------- -/// Unix time (microseconds since 1970-01-01 UTC) at boot (now_us = 0),+/// Unix time (nanoseconds since 1970-01-01 UTC) at boot (now_ns = 0), /// read from the CMOS clock on first use. 0 = not read yet.-static BOOT_REALTIME_US: AtomicU64 = AtomicU64::new(0);+static BOOT_REALTIME_NS: AtomicU64 = AtomicU64::new(0);  /// Read a CMOS register (the PC's battery-backed real-time clock). fn cmos(reg: u8) -> u8 {@@ -97,12 +127,21 @@ fn days_from_civil(y: u64, m: u64, d: u64) -> u64 {     era * 146097 + doe - 719468 } -/// Microseconds since 1970-01-01 UTC (clock_gettime(CLOCK_REALTIME)).-pub fn realtime_us() -> u64 {-    let mut boot = BOOT_REALTIME_US.load(Ordering::Relaxed);+fn boot_realtime_ns() -> u64 {+    let mut boot = BOOT_REALTIME_NS.load(Ordering::Relaxed);     if boot == 0 {-        boot = (rtc_unix_seconds() * 1_000_000).saturating_sub(now_us()).max(1);-        BOOT_REALTIME_US.store(boot, Ordering::Relaxed);+        boot = (rtc_unix_seconds() * 1_000_000_000).saturating_sub(now_ns()).max(1);+        BOOT_REALTIME_NS.store(boot, Ordering::Relaxed);     }-    boot + now_us()+    boot+}++/// Nanoseconds since 1970-01-01 UTC (clock_gettime(CLOCK_REALTIME)).+pub fn realtime_ns() -> u64 {+    boot_realtime_ns() + now_ns()+}++/// Microseconds since 1970-01-01 UTC.+pub fn realtime_us() -> u64 {+    realtime_ns() / 1000 }

@@ -0,0 +1,214 @@+//! The vDSO and its vvar page (stage 19).+//!+//! Programs ask for the time *a lot* (every log line, every timeout, every+//! frame of a game). A system call for each is wasteful: entering the+//! kernel and coming back costs far more than reading a clock. Linux's+//! answer, copied here, is the **vDSO**: a tiny shared library (source in+//! vdso/vdso.c, built by build.rs, embedded in the kernel below) that is+//! mapped into every process. Its `__vdso_clock_gettime` reads the CPU's+//! time stamp counter and turns it into nanoseconds itself.+//!+//! To do that it needs the same numbers the kernel uses (time.rs): the TSC+//! value at boot, the nanoseconds-per-tick multiplier and the wall-clock+//! time at boot. They live in the **vvar page**: one physical page, mapped+//! *read-only* into every process right in front of the vDSO's code. Only+//! the kernel writes it (here, in `publish`), under a seqlock: `seq` is odd+//! while a write is in progress, and readers retry if they saw an odd or+//! changed `seq`.+//!+//! How a program finds it: the auxiliary vector on its initial stack gets+//! `AT_SYSINFO_EHDR` = the address of the vDSO's ELF header (process.rs).+//! The C library walks that ELF image's dynamic symbol table to find+//! `__vdso_clock_gettime` (version "LINUX_2.6") and calls it.+//!+//! Layout in every address space (fixed addresses, just below the stack's+//! guard zone, reserved so mmap never hands them out):+//!+//!   VVAR_ADDR              the vvar page (read-only)+//!   VDSO_ADDR = +4 KiB     the vDSO image (read + execute)+//!+//! The pages are the same physical frames everywhere: allocated once at+//! boot (`init`), and mapped on first touch by the page fault handler+//! (mm.rs), which counts each mapping like any shared frame.+//!+//! When the fast path can't be used -- the TSC isn't calibrated yet, or it+//! isn't *invariant* (on some old CPUs it changes speed with the CPU's+//! clock; see `tsc_trusted` for virtual machines) -- `flags` says so and+//! the vDSO just makes the system call.++use crate::memory;+use crate::mm::{Kind, Mm, Vma, PAGE_SIZE, PROT_EXEC, PROT_READ, VVAR_ADDR};+use alloc::vec::Vec;+use core::sync::atomic::{compiler_fence, Ordering};+use x86_64::structures::paging::PhysFrame;++/// Where the vDSO's ELF header is mapped (AT_SYSINFO_EHDR).+pub const VDSO_ADDR: u64 = VVAR_ADDR + PAGE_SIZE;++/// The vDSO itself, compiled by build.rs from vdso/vdso.c.+pub static IMAGE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/vdso.so"));++/// Pages the image needs.+pub fn image_pages() -> u64 {+    (IMAGE.len() as u64).div_ceil(PAGE_SIZE)+}++/// The vvar page's contents. Must match `struct vvar` in vdso/vdso.c.+#[repr(C)]+struct VVar {+    seq: u32,+    flags: u32,+    tsc_start: u64,+    mult: u64,+    boot_realtime_ns: u64,+}++/// The vDSO may compute the time itself.+pub const VVAR_TSC_OK: u32 = 1;+/// `rdtscp` returns the CPU number (we put it in TSC_AUX, see `init_cpu`).+pub const VVAR_RDTSCP_CPU: u32 = 2;++struct Pages {+    vvar: PhysFrame,+    code: Vec<PhysFrame>,+}+static PAGES: spin::Once<Pages> = spin::Once::new();+/// Only one writer of the vvar page at a time.+static WRITER: spin::Mutex<()> = spin::Mutex::new(());++/// Allocate the vvar page and copy the vDSO into frames (once, at boot:+/// memory::init_frame_allocator calls this), then fill in the vvar page.+/// The frames are never freed: we keep our own reference to each, so+/// processes mapping and unmapping them never drop the count to zero.+pub fn init() {+    PAGES.call_once(|| {+        let vvar = memory::alloc_zeroed_frame().expect("no memory for the vvar page");+        let code = IMAGE+            .chunks(PAGE_SIZE as usize)+            .map(|chunk| {+                let frame = memory::alloc_zeroed_frame().expect("no memory for the vDSO");+                let dst = memory::phys_to_virt(frame.start_address()).as_mut_ptr::<u8>();+                unsafe { core::ptr::copy_nonoverlapping(chunk.as_ptr(), dst, chunk.len()) };+                frame+            })+            .collect();+        Pages { vvar, code }+    });+    publish();+}++/// The frame behind user address `page` in the vvar/vDSO area.+pub fn frame_at(page: u64) -> Option<PhysFrame> {+    let pages = PAGES.r#try()?;+    if page == VVAR_ADDR {+        return Some(pages.vvar);+    }+    let index = page.checked_sub(VDSO_ADDR)? / PAGE_SIZE;+    pages.code.get(index as usize).copied()+}++/// Add the two regions to a fresh address space (execve). Their pages+/// appear on first touch. Returns the AT_SYSINFO_EHDR value (0 = none).+pub fn map_into(mm: &mut Mm) -> u64 {+    if PAGES.r#try().is_none() {+        return 0;+    }+    mm.insert(Vma::new(VVAR_ADDR, VVAR_ADDR + PAGE_SIZE, PROT_READ, Kind::Vvar));+    mm.insert(Vma::new(VDSO_ADDR, VDSO_ADDR + image_pages() * PAGE_SIZE, PROT_READ | PROT_EXEC, Kind::Vdso));+    VDSO_ADDR+}++fn cpuid(leaf: u32) -> core::arch::x86_64::CpuidResult {+    core::arch::x86_64::__cpuid(leaf)+}++/// Does the TSC tick at a constant rate, whatever the CPU's speed and+/// power state? (CPUID 0x8000_0007, EDX bit 8: "invariant TSC".)+pub fn tsc_invariant() -> bool {+    cpuid(0x8000_0000).eax >= 0x8000_0007 && cpuid(0x8000_0007).edx & (1 << 8) != 0+}++/// Are we a virtual machine? (CPUID 1, ECX bit 31, "hypervisor present".)+pub fn under_hypervisor() -> bool {+    cpuid(1).ecx & (1 << 31) != 0+}++/// May user space trust the TSC? If it's invariant, yes. A virtual CPU's+/// TSC also ticks at one fixed rate (QEMU without KVM computes it from its+/// own clock), even though QEMU can't *advertise* "invariant TSC" there --+/// so under a hypervisor we trust it too. (The kernel itself uses the TSC+/// either way; this only decides whether the vDSO may.)+pub fn tsc_trusted() -> bool {+    tsc_invariant() || under_hypervisor()+}++/// Tests: turn the fast path off (true) to exercise the syscall fallback.+static FORCE_SYSCALL: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);++pub fn force_syscall(on: bool) {+    FORCE_SYSCALL.store(on, Ordering::Relaxed);+    publish();+}++/// Does the CPU have `rdtscp`? (CPUID 0x8000_0001, EDX bit 27.)+pub fn has_rdtscp() -> bool {+    cpuid(0x8000_0000).eax >= 0x8000_0001 && cpuid(0x8000_0001).edx & (1 << 27) != 0+}++/// Per-CPU setup (every CPU, at start): put the CPU number into the+/// TSC_AUX register, which `rdtscp` returns -- the vDSO's getcpu().+pub fn init_cpu(cpu: usize) {+    if has_rdtscp() {+        unsafe { x86_64::registers::model_specific::Msr::new(0xC000_0103).write(cpu as u64) };+    }+}++/// The flags we'd publish right now (for tests and the boot log).+pub fn flags() -> u32 {+    if FORCE_SYSCALL.load(Ordering::Relaxed) {+        return 0;+    }+    let (_, mult, _) = crate::time::clock_params();+    let mut flags = 0;+    if mult != 0 && tsc_trusted() {+        flags |= VVAR_TSC_OK;+    }+    if has_rdtscp() {+        flags |= VVAR_RDTSCP_CPU;+    }+    flags+}++/// Copy the kernel's current clock numbers into the vvar page (after TSC+/// calibration, and at boot). The seqlock write: make `seq` odd, write the+/// fields, make `seq` even again. x86 never reorders stores with other+/// stores, so the compiler fences (which stop the *compiler* reordering)+/// are all we need for readers on other CPUs to see them in this order.+pub fn publish() {+    let Some(pages) = PAGES.r#try() else { return };+    let (tsc_start, mult, boot_realtime_ns) = crate::time::clock_params();+    let flags = flags();+    let vvar = memory::phys_to_virt(pages.vvar.start_address()).as_mut_ptr::<VVar>();+    x86_64::instructions::interrupts::without_interrupts(|| {+        let _writer = WRITER.lock();+        unsafe {+            let seq = core::ptr::addr_of_mut!((*vvar).seq);+            let start = seq.read_volatile();+            seq.write_volatile(start.wrapping_add(1) | 1); // odd: writing+            compiler_fence(Ordering::SeqCst);+            core::ptr::addr_of_mut!((*vvar).flags).write_volatile(flags);+            core::ptr::addr_of_mut!((*vvar).tsc_start).write_volatile(tsc_start);+            core::ptr::addr_of_mut!((*vvar).mult).write_volatile(mult);+            core::ptr::addr_of_mut!((*vvar).boot_realtime_ns).write_volatile(boot_realtime_ns);+            compiler_fence(Ordering::SeqCst);+            seq.write_volatile((start | 1).wrapping_add(1)); // even: done+        }+    });+}++/// The vvar page's sequence number (tests: it must be even when idle).+pub fn seq() -> u32 {+    let Some(pages) = PAGES.r#try() else { return 0 };+    let vvar = memory::phys_to_virt(pages.vvar.start_address()).as_ptr::<VVar>();+    unsafe { core::ptr::addr_of!((*vvar).seq).read_volatile() }+}

@@ -61,6 +61,11 @@ fn programs_on_disk_are_valid_elf_files() {         let path = alloc::format!("/mnt/disk0/bin/{}", entry.name);         let image = fs::read_all(&path).unwrap();         assert_eq!(image.len(), fs::lookup(&path).unwrap().size());+        if entry.name == "dynhello" {+            // Stage 19: dynamically linked on purpose, to check it's refused.+            assert!(matches!(elf::check(&image), Err(elf::ElfError::NeedsInterpreter)));+            continue;+        }         assert!(elf::check(&image).is_ok(), "{} is not a valid program", entry.name);     } }

@@ -0,0 +1,78 @@+//! Stage 19: mremap, by running `remaptest` (a musl static PIE): growing in+//! place, shrinking, moving (MREMAP_MAYMOVE, MREMAP_FIXED), the errors, a+//! file mapping, copy-on-write after a move in a forked child, and musl's+//! realloc of big blocks.+#![no_std]+#![no_main]+#![feature(custom_test_frameworks)]+#![test_runner(rustbelt::test_runner)]+#![reexport_test_harness_main = "test_main"]++extern crate alloc;++use alloc::string::String;+use alloc::sync::Arc;+use alloc::vec::Vec;+use bootloader::{BootInfo, entry_point};+use core::panic::PanicInfo;+use rustbelt::fs::file::File;+use rustbelt::process::{self, ExitStatus, SpawnArgs};+use rustbelt::{fs, memory, thread};++entry_point!(main);++fn main(boot_info: &'static BootInfo) -> ! {+    rustbelt::init();+    rustbelt::init_memory(boot_info);+    rustbelt::init_smp(); // moves must shoot down other CPUs' TLB entries+    fs::init();+    rustbelt::init_storage();+    test_main();+    rustbelt::hlt_loop();+}++/// Run /mnt/disk0/bin/`name` with `args`; its exit status and stdout.+fn run(name: &str, args: &[&str]) -> (ExitStatus, String) {+    let (reader, writer) = fs::pipe::new();+    let mut spawn = SpawnArgs::kernel(name);+    spawn.args.extend(args.iter().map(|a| String::from(*a)));+    spawn.stdio[1] = Arc::new(writer);+    let p = process::spawn_path(&alloc::format!("/mnt/disk0/bin/{}", name), spawn).expect("spawn failed");+    let mut out = Vec::new();+    let mut buf = [0u8; 256];+    loop {+        match reader.read(&mut buf).unwrap() {+            0 => break,+            n => out.extend_from_slice(&buf[..n]),+        }+    }+    (process::wait(&p), String::from_utf8_lossy(&out).into_owned())+}++fn counts() -> (usize, usize, usize) {+    thread::sleep_ticks(3);+    (thread::thread_count(), process::count(), memory::frames_in_use())+}++/// remaptest passes, twice, and the second run leaves nothing behind.+#[test_case]+fn remaptest_passes() {+    let (status, out) = run("remaptest", &[]);+    assert_eq!(status, ExitStatus::Exited(0), "output: {}", out);+    let before = counts();+    let (status, out) = run("remaptest", &[]);+    assert_eq!(status, ExitStatus::Exited(0), "output: {}", out);+    assert!(out.contains("remaptest: all passed"), "output: {}", out);+    assert_eq!(counts(), before, "remaptest leaked something");+}++/// `--trace` knows the name.+#[test_case]+fn trace_name() {+    assert_eq!(rustbelt::syscall::trace_name(25), "mremap");+}++#[panic_handler]+fn panic(info: &PanicInfo) -> ! {+    rustbelt::test_panic_handler(info)+}

@@ -176,7 +176,7 @@ fn scripted_shell_session() {         "piped text\n",         "wc -l\n2\n",         "saved\nmore\n",-        "Rustbelt 0.18.0",+        "Rustbelt 0.19.0",         "B:/ $ exit 3\n",     ] {         assert!(out.contains(expected), "missing {:?} in:\n{}", expected, out);

@@ -0,0 +1,120 @@+//! Stage 19: static PIE. The ELF loader now also takes ET_DYN files with+//! no PT_INTERP (what stock `musl-gcc -static-pie` makes), loads them at a+//! random base and reports it in the auxiliary vector; dynamically linked+//! programs (PT_INTERP) get ENOEXEC; classic ET_EXEC still works.+#![no_std]+#![no_main]+#![feature(custom_test_frameworks)]+#![test_runner(rustbelt::test_runner)]+#![reexport_test_harness_main = "test_main"]++extern crate alloc;++use alloc::string::String;+use alloc::sync::Arc;+use alloc::vec::Vec;+use bootloader::{BootInfo, entry_point};+use core::panic::PanicInfo;+use rustbelt::errno::Errno;+use rustbelt::fs::file::File;+use rustbelt::process::{self, ExitStatus, SpawnArgs};+use rustbelt::{elf, fs, memory, thread};++entry_point!(main);++fn main(boot_info: &'static BootInfo) -> ! {+    rustbelt::init();+    rustbelt::init_memory(boot_info);+    rustbelt::init_smp(); // piethreads' threads spread over 4 CPUs+    fs::init();+    rustbelt::init_storage();+    test_main();+    rustbelt::hlt_loop();+}++/// Run /mnt/disk0/bin/`name` with `args`; its exit status and stdout.+fn run(name: &str, args: &[&str]) -> (ExitStatus, String) {+    let (reader, writer) = fs::pipe::new();+    let mut spawn = SpawnArgs::kernel(name);+    spawn.args.extend(args.iter().map(|a| String::from(*a)));+    spawn.stdio[1] = Arc::new(writer);+    let p = process::spawn_path(&alloc::format!("/mnt/disk0/bin/{}", name), spawn).expect("spawn failed");+    let mut out = Vec::new();+    let mut buf = [0u8; 256];+    loop {+        match reader.read(&mut buf).unwrap() {+            0 => break,+            n => out.extend_from_slice(&buf[..n]),+        }+    }+    (process::wait(&p), String::from_utf8_lossy(&out).into_owned())+}++fn counts() -> (usize, usize, usize) {+    thread::sleep_ticks(3);+    (thread::thread_count(), process::count(), memory::frames_in_use())+}++/// "hello" from a static PIE: its relocations, AT_PHDR/AT_ENTRY/AT_BASE.+#[test_case]+fn static_pie_hello() {+    let (status, out) = run("piehello", &[]);+    assert!(out.starts_with("hello from a static PIE\n"), "output: {}", out);+    assert_eq!(status, ExitStatus::Exited(0), "output: {}", out);+}++/// Every run lands at a different, page-aligned address in user space.+#[test_case]+fn static_pie_base_is_random() {+    let parse = |s: &str| u64::from_str_radix(s.trim().trim_start_matches("0x"), 16).unwrap();+    let (s1, a) = run("piehello", &["where"]);+    let (s2, b) = run("piehello", &["where"]);+    assert_eq!((s1, s2), (ExitStatus::Exited(0), ExitStatus::Exited(0)));+    let (a, b) = (parse(&a), parse(&b));+    assert_ne!(a, b, "same base twice");+    assert_eq!(a % 4096, b % 4096, "main moved within its page");+    assert!(a > memory::USER_START && a < rustbelt::mm::MMAP_TOP);+}++/// pthreads + thread-local storage (PT_TLS) + stdio + musl's ttyname on a+/// pty, as a static PIE. Nothing leaks.+#[test_case]+fn static_pie_threads_and_stdio() {+    let (status, out) = run("piethreads", &[]);+    assert_eq!(status, ExitStatus::Exited(0), "output: {}", out);+    let before = counts();+    let (status, out) = run("piethreads", &[]);+    assert_eq!(status, ExitStatus::Exited(0), "output: {}", out);+    assert!(out.contains("piethreads: 0 failed"));+    assert_eq!(counts(), before, "piethreads leaked something");+}++/// A dynamically linked program (PT_INTERP) is refused with ENOEXEC.+#[test_case]+fn dynamic_program_is_enoexec() {+    let image = fs::read_all("/mnt/disk0/bin/dynhello").unwrap();+    assert_eq!(elf::check(&image), Err(elf::ElfError::NeedsInterpreter));+    let err = process::spawn_path("/mnt/disk0/bin/dynhello", SpawnArgs::kernel("dynhello")).err();+    assert_eq!(err, Some(Errno::ENOEXEC));+}++/// The loader still takes the classic ET_EXEC files (Rust programs, the+/// no-libc C program, a musl ET_EXEC), and checks PIEs at their base.+#[test_case]+fn exec_files_still_load() {+    for name in ["hello", "rawabi", "vdsotest"] {+        let image = fs::read_all(&alloc::format!("/mnt/disk0/bin/{}", name)).unwrap();+        assert_eq!(u16::from_le_bytes([image[16], image[17]]), 2, "{} is not ET_EXEC", name);+        assert!(elf::check(&image).is_ok(), "{} doesn't load", name);+    }+    let pie = fs::read_all("/mnt/disk0/bin/piehello").unwrap();+    assert_eq!(u16::from_le_bytes([pie[16], pie[17]]), 3);+    let entry = elf::check(&pie).unwrap();+    assert!(entry > memory::USER_START + 0x40_0000, "PIE entry {:#x} not biased", entry);+    assert_eq!(run("rawabi", &["/tmp"]).0, ExitStatus::Exited(0));+}++#[panic_handler]+fn panic(info: &PanicInfo) -> ! {+    rustbelt::test_panic_handler(info)+}

@@ -0,0 +1,145 @@+//! Stage 19: the vDSO. Every process gets a little shared library from the+//! kernel (vdso/vdso.c) plus a read-only "vvar" page with the clock's+//! numbers, so musl's clock_gettime/gettimeofday/time/sched_getcpu don't+//! need a system call. The detailed checks run in user/vdsotest.c (a musl+//! program; exit code = failed checks); here we check the kernel side and+//! use `--trace` to prove no clock syscall happens.+#![no_std]+#![no_main]+#![feature(custom_test_frameworks)]+#![test_runner(rustbelt::test_runner)]+#![reexport_test_harness_main = "test_main"]++extern crate alloc;++use alloc::string::String;+use bootloader::{BootInfo, entry_point};+use core::panic::PanicInfo;+use rustbelt::process::{self, ExitStatus, SpawnArgs};+use rustbelt::{fs, memory, syscall, thread, time, vdso};++entry_point!(main);++fn main(boot_info: &'static BootInfo) -> ! {+    rustbelt::init();+    rustbelt::init_memory(boot_info);+    rustbelt::init_smp(); // calibrates the TSC, which turns the fast path on+    fs::init();+    rustbelt::init_storage();+    test_main();+    rustbelt::hlt_loop();+}++fn run(args: &[&str], trace: bool) -> ExitStatus {+    let mut spawn = SpawnArgs::kernel("vdsotest");+    spawn.args.extend(args.iter().map(|a| String::from(*a)));+    spawn.trace = trace;+    let p = process::spawn_path("/mnt/disk0/bin/vdsotest", spawn).expect("spawn failed");+    process::wait(&p)+}++fn counts() -> (usize, usize, usize, usize) {+    thread::sleep_ticks(3);+    (thread::thread_count(), process::count(), memory::frames_in_use(), memory::shared_frame_count())+}++/// The embedded image is a one-page ELF shared object with the dynamic+/// symbol and SysV hash tables musl looks for.+#[test_case]+fn image_is_a_small_shared_object() {+    let image = vdso::IMAGE;+    assert_eq!(&image[..4], b"\x7fELF");+    assert_eq!(u16::from_le_bytes([image[16], image[17]]), 3, "not ET_DYN");+    assert_eq!(vdso::image_pages(), 1);+    let has = |needle: &[u8]| image.windows(needle.len()).any(|w| w == needle);+    for name in ["__vdso_clock_gettime", "__vdso_gettimeofday", "__vdso_time", "__vdso_getcpu", "LINUX_2.6"] {+        assert!(has(name.as_bytes()), "{} missing from the vDSO", name);+    }+}++/// The vvar page is published (even sequence number) and, after TSC+/// calibration on QEMU, allows the fast path.+#[test_case]+fn vvar_page_is_published() {+    assert_eq!(vdso::seq() % 2, 0);+    assert!(vdso::seq() >= 4, "published at boot and after calibration");+    assert!(vdso::tsc_trusted());+    assert_ne!(vdso::flags() & vdso::VVAR_TSC_OK, 0);+}++/// The kernel's clocks in nanoseconds: monotonic, and REALTIME - MONOTONIC+/// is the boot time (constant).+#[test_case]+fn kernel_clock_in_nanoseconds() {+    let mut prev = time::now_ns();+    for _ in 0..1000 {+        let now = time::now_ns();+        assert!(now >= prev);+        prev = now;+    }+    let (_, mult, boot) = time::clock_params();+    assert_ne!(mult, 0);+    let (r, m) = (time::realtime_ns(), time::now_ns());+    assert!(r >= boot + m - 1_000_000 && r <= boot + m, "realtime {} vs boot {} + now {}", r, boot, m);+    assert!(boot / 1_000_000_000 > 1_577_836_800, "boot time before 2020");+}++/// All of user/vdsotest.c: vDSO vs syscall agreement, monotonicity, other+/// clocks, gettimeofday/time/getcpu, auxv, the read-only vvar page, fork,+/// execve, ENOEXEC for a dynamic program. Nothing leaks.+#[test_case]+fn vdsotest_passes() {+    assert_eq!(run(&[], false), ExitStatus::Exited(0), "vdsotest failed (warm-up)");+    let before = counts();+    assert_eq!(run(&[], false), ExitStatus::Exited(0), "vdsotest failed");+    assert_eq!(counts(), before, "vdsotest leaked something");+}++fn clock_syscalls(trace: &str) -> usize {+    trace+        .lines()+        .filter(|l| ["] clock_gettime(", "] gettimeofday(", "] time(", "] getcpu("].iter().any(|c| l.contains(c)))+        .count()+}++/// 1000 rounds of clock_gettime(MONOTONIC and REALTIME), gettimeofday,+/// time and sched_getcpu under `--trace`: not one of them is a syscall.+#[test_case]+fn clock_reads_make_no_syscalls() {+    syscall::collect_trace(true);+    let status = run(&["calls"], true);+    let trace = syscall::collect_trace(false);+    assert_eq!(status, ExitStatus::Exited(0));+    assert!(trace.contains("exit_group(0)") || trace.contains("exit(0)"), "no trace collected:\n{}", trace);+    assert_eq!(clock_syscalls(&trace), 0, "clock syscalls in the trace:\n{}", trace);+}++/// Without a trustworthy TSC the vDSO makes the system calls instead, and+/// everything still passes.+#[test_case]+fn falls_back_to_syscalls() {+    vdso::force_syscall(true);+    assert_eq!(vdso::flags(), 0);+    syscall::collect_trace(true);+    let calls = run(&["calls"], true);+    let trace = syscall::collect_trace(false);+    let full = run(&[], false);+    vdso::force_syscall(false);+    assert_eq!(calls, ExitStatus::Exited(0));+    assert_eq!(full, ExitStatus::Exited(0), "vdsotest failed on the syscall path");+    assert!(clock_syscalls(&trace) >= 5000, "only {} clock syscalls", clock_syscalls(&trace));+    assert_ne!(vdso::flags() & vdso::VVAR_TSC_OK, 0);+}++/// `--trace` knows the new fallback calls by name.+#[test_case]+fn trace_names() {+    assert_eq!(syscall::trace_name(96), "gettimeofday");+    assert_eq!(syscall::trace_name(201), "time");+    assert_eq!(syscall::trace_name(309), "getcpu");+}++#[panic_handler]+fn panic(info: &PanicInfo) -> ! {+    rustbelt::test_panic_handler(info)+}

@@ -447,7 +447,7 @@ fn info(c: &mut Checks) {     c.check(ru[0] * 1_000_000 + ru[1] + ru[2] * 1_000_000 + ru[3] >= 100_000, "getrusage(RUSAGE_CHILDREN) has it too");     c.check(syscall3(SYS_GETRUSAGE, 5, ru.as_mut_ptr() as u64, 0) == -EINVAL, "getrusage(5): EINVAL");     let mut res = [0u64; 2];-    c.check(syscall3(SYS_CLOCK_GETRES, 1, res.as_mut_ptr() as u64, 0) == 0 && res == [0, 1000], "clock_getres(MONOTONIC): 1 µs");+    c.check(syscall3(SYS_CLOCK_GETRES, 1, res.as_mut_ptr() as u64, 0) == 0 && res == [0, 1], "clock_getres(MONOTONIC): 1 ns (stage 19; was 1 µs)");     c.check(syscall3(SYS_CLOCK_GETRES, 2, res.as_mut_ptr() as u64, 0) == 0 && res == [0, 10_000_000], "clock_getres(PROCESS_CPUTIME): 10 ms");     c.check(syscall3(SYS_CLOCK_GETRES, 0, 0, 0) == 0, "clock_getres with a NULL pointer: 0");     c.check(syscall3(SYS_CLOCK_GETRES, 42, res.as_mut_ptr() as u64, 0) == -EINVAL, "clock_getres(42): EINVAL");

@@ -0,0 +1,13 @@+/*+ * dynhello.c -- stage 19: a *dynamically linked* program (plain+ * `musl-gcc`, no -static). Its ELF file has a PT_INTERP header naming+ * the dynamic loader /lib/ld-musl-x86_64.so.1, which Rustbelt doesn't+ * have, so execve must fail with ENOEXEC ("Exec format error").+ */+#include <stdio.h>++int main(void)+{+	puts("hello from a dynamic program (you shouldn't see this on Rustbelt)");+	return 0;+}

@@ -137,7 +137,7 @@ fn main() -> i64 {     c.check(n > 0 && cwd[0] == b'/', "getcwd gives an absolute path");     let mut ub = [0u8; 6 * 65];     let u = uname(&mut ub);-    c.check(u[0] == "Rustbelt" && u[3].starts_with("stage 18"), "uname: Rustbelt, stage 18");+    c.check(u[0] == "Rustbelt" && u[3].starts_with("stage 19"), "uname: Rustbelt, stage 19");      let (secs, _) = clock_gettime(0);     c.check(secs > 1_600_000_000, "CLOCK_REALTIME is a real date (after 2020)");

@@ -0,0 +1,65 @@+/*+ * piehello.c -- stage 19: "hello" as a static PIE, built with the stock+ * `musl-gcc -static-pie` (no special flags). The file's addresses start+ * at 0; the kernel loads it at a random base and tells us where through+ * the auxiliary vector. musl's start-up code (rcrt1.o) then fixes up the+ * program's own pointers before main runs.+ *+ * Exit code = number of failed checks. `piehello where` prints the load+ * address of main (tests/static_pie.rs runs it twice: different bases).+ */+#define _GNU_SOURCE+#include <elf.h>+#include <link.h>+#include <stdio.h>+#include <string.h>+#include <sys/auxv.h>++static int fails;+#define CHECK(cond, what)                                    \+	do {                                                 \+		if (!(cond)) {                               \+			fails++;                             \+			printf("piehello: FAIL %s\n", what); \+		}                                            \+	} while (0)++/* Pointers in data: these need fixing up (relocating) at start-up. */+static const char *greeting = "hello from a static PIE";+static int (*say)(const char *) = puts;++extern char _start[];+extern const Elf64_Ehdr __ehdr_start; /* the linker's name for our ELF header */++int main(int argc, char **argv)+{+	if (argc > 1 && strcmp(argv[1], "where") == 0) {+		printf("%p\n", (void *)main);+		return 0;+	}+	say(greeting);+	unsigned long phdr = getauxval(AT_PHDR), entry = getauxval(AT_ENTRY);+	CHECK(getauxval(AT_BASE) == 0, "AT_BASE is 0 (no dynamic loader)");+	CHECK(entry == (unsigned long)_start, "AT_ENTRY is where _start is");+	CHECK(getauxval(AT_PHENT) == sizeof(Elf64_Phdr), "AT_PHENT");+	/* Our ELF header is loaded too, at the load bias (the file's lowest+	 * address is 0), and the program headers follow it, where AT_PHDR says. */+	unsigned long bias = (unsigned long)&__ehdr_start;+	CHECK(phdr == bias + __ehdr_start.e_phoff, "AT_PHDR points at the program headers");+	const Elf64_Phdr *ph = (const Elf64_Phdr *)phdr;+	int has_tls = 0, has_relro = 0, has_interp = 0;+	for (unsigned long i = 0; i < getauxval(AT_PHNUM); i++) {+		has_tls |= ph[i].p_type == PT_TLS;+		has_relro |= ph[i].p_type == PT_GNU_RELRO;+		has_interp |= ph[i].p_type == PT_INTERP;+	}+	CHECK(bias != 0 && bias % 4096 == 0, "a page-aligned, non-zero load bias");+	CHECK(bias >= 0x8000000000UL, "loaded in Rustbelt's user space");+	CHECK(has_relro && !has_interp, "PT_GNU_RELRO, no PT_INTERP");+	(void)has_tls;+	/* The relocated pointers point into the loaded image. */+	CHECK((unsigned long)greeting > bias, "relocated string pointer");+	/* RELRO: musl made it read-only after relocating; mprotect worked. */+	printf("argv[0] = %s, %d argument(s), bias %#lx\n", argv[0], argc, bias);+	return fails;+}

@@ -0,0 +1,122 @@+/*+ * piethreads.c -- stage 19: a static PIE (stock `musl-gcc -static-pie`)+ * using pthreads, thread-local storage (PT_TLS), mutexes and stdio, plus+ * musl's ttyname() on a pseudo-terminal. Exit code = failed checks.+ */+#define _GNU_SOURCE+#include <errno.h>+#include <fcntl.h>+#include <pthread.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/stat.h>+#include <sys/sysmacros.h>+#include <time.h>+#include <unistd.h>++static int fails;+#define CHECK(cond, ...)                                     \+	do {                                                 \+		if (!(cond)) {                               \+			fails++;                             \+			printf("piethreads: FAIL ");         \+			printf(__VA_ARGS__);                 \+			printf("\n");                        \+		}                                            \+	} while (0)++#define THREADS 4+#define ROUNDS 10000++static __thread int mine = 7;        /* PT_TLS: initialized ... */+static __thread char scratch[64];    /* ... and zeroed (.tbss) */+static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;+static long shared;+static FILE *out;++static void *worker(void *arg)+{+	long id = (long)arg;+	if (mine != 7 || scratch[0] != 0)+		return (void *)1; /* every thread starts with a fresh TLS copy */+	mine = id;+	snprintf(scratch, sizeof scratch, "thread %ld", id);+	for (int i = 0; i < ROUNDS; i++) {+		pthread_mutex_lock(&lock);+		shared++;+		pthread_mutex_unlock(&lock);+	}+	pthread_mutex_lock(&lock);+	fprintf(out, "%s done\n", scratch);+	pthread_mutex_unlock(&lock);+	/* Nobody else changed our copy meanwhile. */+	return (void *)(long)(mine != id);+}++static void threads(void)+{+	const char *path = "/tmp/piethreads.txt";+	out = fopen(path, "w");+	CHECK(out != NULL, "fopen: errno %d", errno);+	if (!out)+		return;+	pthread_t t[THREADS];+	for (long i = 0; i < THREADS; i++)+		CHECK(pthread_create(&t[i], NULL, worker, (void *)i) == 0, "pthread_create %ld", i);+	for (int i = 0; i < THREADS; i++) {+		void *r;+		pthread_join(t[i], &r);+		CHECK(r == NULL, "thread %d saw wrong TLS", i);+	}+	CHECK(shared == (long)THREADS * ROUNDS, "counter %ld", shared);+	CHECK(mine == 7, "main thread's TLS changed");+	fclose(out);+	/* Read the file back through stdio: one line per thread. */+	FILE *in = fopen(path, "r");+	char line[64];+	int lines = 0;+	while (in && fgets(line, sizeof line, in))+		lines += strstr(line, " done\n") != NULL;+	if (in)+		fclose(in);+	CHECK(lines == THREADS, "%d lines in the file", lines);+	unlink(path);+}++/* musl's ttyname: readlink /proc/self/fd/N, then stat() that name and+ * fstat() the fd -- both must give the same device and inode (stage 19). */+static void tty_names(void)+{+	int master = posix_openpt(O_RDWR | O_NOCTTY);+	CHECK(master >= 0, "posix_openpt: errno %d", errno);+	if (master < 0)+		return;+	CHECK(grantpt(master) == 0 && unlockpt(master) == 0, "grantpt/unlockpt");+	char *name = ptsname(master);+	CHECK(name && strncmp(name, "/dev/pts/", 9) == 0, "ptsname %s", name ? name : "(null)");+	int slave = name ? open(name, O_RDWR | O_NOCTTY) : -1;+	CHECK(slave >= 0, "open %s: errno %d", name, errno);+	if (slave >= 0) {+		char *tn = ttyname(slave);+		CHECK(tn && strcmp(tn, name) == 0, "ttyname(slave) = %s, errno %d", tn ? tn : "(null)", errno);+		struct stat st;+		CHECK(fstat(slave, &st) == 0 && S_ISCHR(st.st_mode) && major(st.st_rdev) == 136, "pts rdev");+		close(slave);+	}+	char *tm = ttyname(master);+	CHECK(tm && strcmp(tm, "/dev/ptmx") == 0, "ttyname(master) = %s", tm ? tm : "(null)");+	close(master);+	if (isatty(0)) {+		char *t0 = ttyname(0);+		CHECK(t0 != NULL, "ttyname(0): errno %d", errno);+	}+}++int main(void)+{+	threads();+	tty_names();+	printf("piethreads: %d failed\n", fails);+	return fails;+}

@@ -0,0 +1,221 @@+/*+ * remaptest.c -- stage 19: mremap, checked from musl.+ *+ * Built with `musl-gcc -static-pie` (see build.rs). mremap makes an mmap+ * region bigger or smaller, moving it if needed; realloc() uses it for big+ * blocks so the bytes don't have to be copied.+ *+ *   remaptest      all checks (exit code = number of failed checks)+ */+#define _GNU_SOURCE+#include <errno.h>+#include <fcntl.h>+#include <signal.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/mman.h>+#include <sys/wait.h>+#include <unistd.h>++#define PG 4096UL++static int fails;+#define CHECK(cond, ...)                                               \+	do {                                                           \+		if (!(cond)) {                                         \+			fails++;                                       \+			printf("remaptest: FAIL line %d: ", __LINE__); \+			printf(__VA_ARGS__);                           \+			printf("\n");                                  \+		}                                                      \+	} while (0)++static char *anon(size_t pages)+{+	char *p = mmap(0, pages * PG, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);+	return p == MAP_FAILED ? 0 : p;+}++static void fill(char *p, size_t pages, int seed)+{+	for (size_t i = 0; i < pages; i++)+		memset(p + i * PG, seed + (int)i, PG);+}++static int same(const char *p, size_t pages, int seed)+{+	for (size_t i = 0; i < pages; i++)+		for (size_t j = 0; j < PG; j += 512)+			if (p[i * PG + j] != (char)(seed + (int)i))+				return 0;+	return 1;+}++static int zero(const char *p, size_t len)+{+	for (size_t i = 0; i < len; i += 256)+		if (p[i])+			return 0;+	return 1;+}++/* Does touching `p` kill a child with SIGSEGV? (= nothing mapped there) */+static int faults(volatile char *p)+{+	pid_t pid = fork();+	if (pid == 0) {+		(void)*p;+		_exit(0);+	}+	int st = 0;+	waitpid(pid, &st, 0);+	return WIFSIGNALED(st) && WTERMSIG(st) == SIGSEGV;+}++static void grow_in_place(void)+{+	char *p = anon(8);+	CHECK(p, "mmap 8 pages");+	munmap(p + 3 * PG, 5 * PG); /* keep 3, leave room behind them */+	fill(p, 3, 'a');+	char *q = mremap(p, 3 * PG, 6 * PG, 0);+	CHECK(q == p, "grow in place: %p, wanted %p", q, p);+	if (q != p)+		return;+	CHECK(same(p, 3, 'a'), "grow in place kept the data");+	CHECK(zero(p + 3 * PG, 3 * PG), "the new part is zero");+	p[6 * PG - 1] = 1; /* and writable */+	q = mremap(p, 6 * PG, 2 * PG, 0);+	CHECK(q == p, "shrink: %p", q);+	CHECK(same(p, 2, 'a'), "shrink kept the data");+	CHECK(faults(p + 2 * PG), "the cut-off part is gone");+	CHECK(mremap(p, 2 * PG, 2 * PG, 0) == p, "same size: nothing to do");+	munmap(p, 2 * PG);+}++static void move(void)+{+	char *a = anon(2);+	char *b = anon(1); /* right below a: can't grow in place */+	CHECK(a && b && b + PG == a, "b (%p) sits right below a (%p)", b, a);+	fill(b, 1, 'm');+	errno = 0;+	CHECK(mremap(b, PG, 4 * PG, 0) == MAP_FAILED && errno == ENOMEM, "no room and no MAYMOVE: ENOMEM (%d)", errno);+	char *c = mremap(b, PG, 4 * PG, MREMAP_MAYMOVE);+	CHECK(c != MAP_FAILED && c != b, "MAYMOVE moved it: %p", c);+	if (c == MAP_FAILED)+		return;+	CHECK(same(c, 1, 'm'), "the moved page kept its bytes");+	CHECK(zero(c + PG, 3 * PG), "the rest is zero");+	CHECK(faults(b), "nothing left at the old address");++	/* MREMAP_FIXED: to an address we pick. */+	char *spot = anon(4);+	munmap(spot, 4 * PG);+	char *d = mremap(c, 4 * PG, 2 * PG, MREMAP_MAYMOVE | MREMAP_FIXED, spot);+	CHECK(d == spot, "MREMAP_FIXED: %p, wanted %p", d, spot);+	if (d == spot)+		CHECK(same(d, 1, 'm'), "MREMAP_FIXED kept the bytes");+	CHECK(faults(c), "and the old place is empty");+	munmap(spot, 2 * PG);+	munmap(a, 2 * PG);+}++static void errors(void)+{+	char *p = anon(2);+	errno = 0;+	CHECK(mremap(p + 1, PG, 2 * PG, MREMAP_MAYMOVE) == MAP_FAILED && errno == EINVAL, "unaligned: EINVAL (%d)", errno);+	errno = 0;+	CHECK(mremap(p, PG, 0, MREMAP_MAYMOVE) == MAP_FAILED && errno == EINVAL, "new_len 0: EINVAL (%d)", errno);+	errno = 0;+	CHECK(mremap(p, PG, 2 * PG, 64) == MAP_FAILED && errno == EINVAL, "unknown flag: EINVAL (%d)", errno);+	errno = 0;+	CHECK(mremap(p, PG, 2 * PG, MREMAP_FIXED, p + 8 * PG) == MAP_FAILED && errno == EINVAL,+	      "FIXED without MAYMOVE: EINVAL (%d)", errno);+	errno = 0;+	CHECK(mremap(p, 2 * PG, 4 * PG, MREMAP_MAYMOVE | MREMAP_FIXED, p + PG) == MAP_FAILED && errno == EINVAL,+	      "FIXED onto itself: EINVAL (%d)", errno);+	munmap(p, 2 * PG);+	errno = 0;+	CHECK(mremap(p, PG, 2 * PG, MREMAP_MAYMOVE) == MAP_FAILED && errno == EFAULT, "nothing mapped: EFAULT (%d)", errno);+}++static void file_backed(void)+{+	int fd = open("/tmp/remap.dat", O_RDWR | O_CREAT | O_TRUNC, 0644);+	CHECK(fd >= 0, "open /tmp/remap.dat");+	char page[PG];+	for (int i = 0; i < 3; i++) {+		memset(page, 'F' + i, PG);+		write(fd, page, PG);+	}+	char *keep = anon(1); /* so the mapping below can't grow in place */+	char *f = mmap(0, PG, PROT_READ, MAP_PRIVATE, fd, 0);+	(void)keep;+	CHECK(f != MAP_FAILED && f[0] == 'F', "file page 0");+	char *g = mremap(f, PG, 3 * PG, MREMAP_MAYMOVE);+	CHECK(g != MAP_FAILED, "mremap a file mapping");+	if (g != MAP_FAILED) {+		CHECK(g[0] == 'F' && g[PG] == 'G' && g[2 * PG + 7] == 'H', "moved file pages read the file: %c %c %c", g[0],+		      g[PG], g[2 * PG + 7]);+		munmap(g, 3 * PG);+	}+	munmap(keep, PG);+	close(fd);+	unlink("/tmp/remap.dat");+}++static void after_fork(void)+{+	char *p = anon(1);+	fill(p, 1, 'p');+	pid_t pid = fork();+	if (pid == 0) {+		char *q = mremap(p, PG, 16 * PG, MREMAP_MAYMOVE);+		if (q == MAP_FAILED || q[0] != 'p')+			_exit(1);+		q[0] = 'c'; /* copy-on-write must still work after the move */+		_exit(q[0] == 'c' ? 0 : 2);+	}+	int st = 0;+	waitpid(pid, &st, 0);+	CHECK(WIFEXITED(st) && WEXITSTATUS(st) == 0, "child moved and wrote: status %#x", st);+	CHECK(p[0] == 'p', "parent's copy unchanged: %c", p[0]);+	munmap(p, PG);+}++static void with_realloc(void)+{+	size_t n = 1 << 20;+	unsigned *v = malloc(n);+	CHECK(v, "malloc 1 MiB");+	for (size_t i = 0; i < n / 4; i++)+		v[i] = (unsigned)i * 2654435761u;+	for (size_t big = 4 << 20; big <= 32 << 20; big *= 2) {+		v = realloc(v, big);+		CHECK(v, "realloc %zu", big);+		if (!v)+			return;+		size_t bad = 0;+		for (size_t i = 0; i < n / 4; i++)+			bad += v[i] != (unsigned)i * 2654435761u;+		CHECK(bad == 0, "realloc to %zu MiB kept the data (%zu wrong)", big >> 20, bad);+	}+	v = realloc(v, 2 << 20);+	CHECK(v && v[12345] == 12345u * 2654435761u, "realloc down");+	free(v);+}++int main(void)+{+	grow_in_place();+	move();+	errors();+	file_backed();+	after_fork();+	with_realloc();+	printf("remaptest: %s (%d failed)\n", fails ? "FAILED" : "all passed", fails);+	return fails;+}

@@ -0,0 +1,251 @@+/*+ * vdsotest.c -- stage 19: the vDSO, checked from a real C library (musl).+ *+ * Built with `musl-gcc -static` as a classic ET_EXEC program (linked at+ * Rustbelt's user base, see build.rs). musl finds the vDSO through+ * AT_SYSINFO_EHDR and then calls __vdso_clock_gettime instead of making a+ * system call. We compare what the vDSO says with what the real system+ * call says, check the clocks never go backwards, and that it all still+ * works in a forked child and after execve.+ *+ *   vdsotest            all checks (exit code = number of failed checks)+ *   vdsotest calls      just call the clock functions a lot, print nothing+ *                       (tests/vdso.rs looks at the --trace output)+ *   vdsotest exec       (used internally: the program after execve)+ */+#define _GNU_SOURCE+#include <elf.h>+#include <errno.h>+#include <sched.h>+#include <signal.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/auxv.h>+#include <sys/mman.h>+#include <sys/syscall.h>+#include <sys/time.h>+#include <sys/wait.h>+#include <time.h>+#include <unistd.h>++static int fails;+#define CHECK(cond, ...)                                              \+	do {                                                          \+		if (!(cond)) {                                        \+			fails++;                                      \+			printf("vdsotest: FAIL line %d: ", __LINE__); \+			printf(__VA_ARGS__);                          \+			printf("\n");                                 \+		}                                                     \+	} while (0)++extern char _start[]; /* musl's entry point (crt1.o) */++static long long ns(struct timespec t) { return t.tv_sec * 1000000000LL + t.tv_nsec; }+static long long us(struct timeval t) { return t.tv_sec * 1000000LL + t.tv_usec; }++/* The real system call, bypassing the C library (and so the vDSO). */+static long long sys_ns(int clock)+{+	struct timespec t;+	if (syscall(SYS_clock_gettime, clock, &t) != 0)+		return -1;+	return ns(t);+}++static long long vdso_ns(int clock)+{+	struct timespec t;+	if (clock_gettime(clock, &t) != 0)+		return -1;+	return ns(t);+}++/* The image behind AT_SYSINFO_EHDR is an ELF shared object, and the page+ * in front of it (the vvar page) is readable but can't be made writable. */+static void image(void)+{+	unsigned long base = getauxval(AT_SYSINFO_EHDR);+	CHECK(base != 0, "no AT_SYSINFO_EHDR");+	if (!base)+		return;+	const Elf64_Ehdr *eh = (const Elf64_Ehdr *)base;+	CHECK(memcmp(eh->e_ident, ELFMAG, SELFMAG) == 0, "vDSO has no ELF magic");+	CHECK(eh->e_type == ET_DYN, "vDSO e_type %d", eh->e_type);+	void *vvar = (void *)(base - 4096);+	unsigned seq = *(volatile unsigned *)vvar;+	CHECK(seq % 2 == 0, "vvar seq %u is odd", seq);+	errno = 0;+	CHECK(mprotect(vvar, 4096, PROT_READ | PROT_WRITE) == -1 && errno == EACCES, "mprotect(vvar, RW): errno %d", errno);+	/* Other auxv entries a C library looks at. */+	CHECK(getauxval(AT_PAGESZ) == 4096, "AT_PAGESZ");+	CHECK(getauxval(AT_CLKTCK) == 100, "AT_CLKTCK");+	CHECK(getauxval(AT_HWCAP) & (1 << 4), "AT_HWCAP has no TSC bit");+	CHECK(getauxval(AT_RANDOM) != 0, "AT_RANDOM");+	CHECK(getauxval(AT_SECURE) == 0, "AT_SECURE");+	CHECK(getauxval(AT_ENTRY) == (unsigned long)_start, "AT_ENTRY %lx", getauxval(AT_ENTRY));+	CHECK(getauxval(AT_PHNUM) > 0 && getauxval(AT_PHENT) == sizeof(Elf64_Phdr), "AT_PHNUM/AT_PHENT");+	CHECK(getauxval(AT_BASE) == 0, "AT_BASE (no dynamic loader)");+	const char *platform = (const char *)getauxval(AT_PLATFORM);+	CHECK(platform && strcmp(platform, "x86_64") == 0, "AT_PLATFORM");+	const char *execfn = (const char *)getauxval(AT_EXECFN);+	CHECK(execfn && strstr(execfn, "vdsotest"), "AT_EXECFN");+}++/* Each fast clock: vDSO, then syscall, then vDSO again must be in order+ * (they compute the same formula from the same numbers); and the clocks+ * never go backwards. */+static void clocks(void)+{+	static const int fast[] = { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_MONOTONIC_RAW, CLOCK_REALTIME_COARSE,+				    CLOCK_MONOTONIC_COARSE };+	for (unsigned i = 0; i < sizeof fast / sizeof fast[0]; i++) {+		int c = fast[i];+		for (int round = 0; round < 100; round++) {+			long long a = vdso_ns(c), b = sys_ns(c), d = vdso_ns(c);+			if (!(a > 0 && a <= b && b <= d)) {+				CHECK(0, "clock %d out of order: vdso %lld, syscall %lld, vdso %lld", c, a, b, d);+				break;+			}+		}+	}+	long long prev = vdso_ns(CLOCK_MONOTONIC);+	for (int i = 0; i < 100000; i++) {+		long long now = vdso_ns(CLOCK_MONOTONIC);+		if (now < prev) {+			CHECK(0, "CLOCK_MONOTONIC went back from %lld to %lld", prev, now);+			break;+		}+		prev = now;+	}+	/* Nanoseconds really: two readings in a row usually differ by less+	 * than a microsecond, but they must not be stuck on whole µs. */+	int sub_us = 0;+	for (int i = 0; i < 100 && !sub_us; i++)+		sub_us = vdso_ns(CLOCK_MONOTONIC) % 1000 != 0;+	CHECK(sub_us, "CLOCK_MONOTONIC only has microseconds");+	/* The wall clock is about now (after 2020), and REALTIME = MONOTONIC + boot. */+	CHECK(vdso_ns(CLOCK_REALTIME) / 1000000000 > 1577836800LL, "CLOCK_REALTIME is before 2020");+	/* Other clocks take the syscall path inside the vDSO, and still work. */+	struct timespec t;+	CHECK(clock_gettime(CLOCK_BOOTTIME, &t) == 0 && ns(t) > 0, "CLOCK_BOOTTIME");+	CHECK(clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &t) == 0, "CLOCK_PROCESS_CPUTIME_ID");+	CHECK(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &t) == 0, "CLOCK_THREAD_CPUTIME_ID");+	errno = 0;+	CHECK(clock_gettime(12345, &t) == -1 && errno == EINVAL, "bad clock: errno %d", errno);++	/* gettimeofday and time: also from the vDSO, also agreeing. */+	struct timeval tv1, tv2, tvs;+	struct timezone tz = { 99, 99 };+	gettimeofday(&tv1, NULL);+	CHECK(syscall(SYS_gettimeofday, &tvs, &tz) == 0, "gettimeofday syscall");+	gettimeofday(&tv2, NULL);+	CHECK(us(tv1) <= us(tvs) && us(tvs) <= us(tv2), "gettimeofday out of order: %lld %lld %lld", us(tv1), us(tvs),+	      us(tv2));+	CHECK(tz.tz_minuteswest == 0 && tz.tz_dsttime == 0, "timezone not UTC");+	CHECK(tv1.tv_usec >= 0 && tv1.tv_usec < 1000000, "tv_usec %ld", (long)tv1.tv_usec);+	time_t t1 = time(NULL), ts = syscall(SYS_time, NULL), t2;+	time(&t2);+	CHECK(t1 <= ts && ts <= t2 && t2 - t1 < 2, "time out of order: %ld %ld %ld", (long)t1, (long)ts, (long)t2);+	CHECK(t1 == tv1.tv_sec || t1 == tv1.tv_sec + 1, "time %ld vs gettimeofday %ld", (long)t1, (long)tv1.tv_sec);++	/* sched_getcpu (getcpu in the vDSO): one of our 4 CPUs. */+	int cpu = sched_getcpu();+	CHECK(cpu >= 0 && cpu < 64, "sched_getcpu %d", cpu);+	unsigned scpu = 999, node = 999;+	CHECK(syscall(SYS_getcpu, &scpu, &node, NULL) == 0 && scpu < 64 && node == 0, "getcpu syscall %u %u", scpu, node);+	/* Pin ourselves? No sched_setaffinity here: just check both agree+	 * at least once in a few tries (the scheduler may move us between). */+	int same = 0;+	for (int i = 0; i < 50 && !same; i++) {+		cpu = sched_getcpu();+		syscall(SYS_getcpu, &scpu, NULL, NULL);+		same = (unsigned)cpu == scpu;+	}+	CHECK(same, "sched_getcpu %d never matched getcpu %u", cpu, scpu);+}++/* Writing to the vvar page is a SIGSEGV (it's mapped read-only). */+static void vvar_is_read_only(void)+{+	unsigned long base = getauxval(AT_SYSINFO_EHDR);+	if (!base)+		return;+	pid_t pid = fork();+	if (pid == 0) {+		*(volatile unsigned *)(base - 4096) = 1;+		_exit(0);+	}+	int status = 0;+	waitpid(pid, &status, 0);+	CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV, "write to vvar: status %#x", status);+}++/* A forked child has the vDSO too (same pages, same answers). */+static void after_fork(void)+{+	long long before = vdso_ns(CLOCK_MONOTONIC);+	pid_t pid = fork();+	if (pid == 0) {+		long long a = vdso_ns(CLOCK_MONOTONIC), b = sys_ns(CLOCK_MONOTONIC), c = vdso_ns(CLOCK_MONOTONIC);+		_exit(getauxval(AT_SYSINFO_EHDR) && a >= before && a <= b && b <= c ? 0 : 1);+	}+	int status = 0;+	waitpid(pid, &status, 0);+	CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0, "vDSO after fork: status %#x", status);+}++/* And a program started by execve gets a fresh one. */+static void after_exec(const char *self)+{+	pid_t pid = fork();+	if (pid == 0) {+		char *argv[] = { (char *)self, "exec", NULL };+		execv(self, argv);+		_exit(99);+	}+	int status = 0;+	waitpid(pid, &status, 0);+	CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0, "vdsotest after execve: status %#x", status);+}++/* A dynamically linked program (PT_INTERP) can't run here: ENOEXEC. */+static void dynamic_is_enoexec(void)+{+	char *argv[] = { "dynhello", NULL };+	errno = 0;+	execv("/mnt/disk0/bin/dynhello", argv);+	CHECK(errno == ENOEXEC, "execve of a dynamic program: errno %d", errno);+}++int main(int argc, char **argv)+{+	if (argc > 1 && strcmp(argv[1], "calls") == 0) {+		/* Lots of clock reads; tests/vdso.rs checks none is a syscall. */+		struct timespec t;+		struct timeval tv;+		long long sum = 0;+		for (int i = 0; i < 1000; i++) {+			clock_gettime(CLOCK_MONOTONIC, &t);+			sum += t.tv_nsec;+			clock_gettime(CLOCK_REALTIME, &t);+			gettimeofday(&tv, NULL);+			sum += time(NULL) + sched_getcpu();+		}+		return sum == 42; /* use the result; 0 */+	}+	if (argc > 1 && strcmp(argv[1], "exec") == 0) {+		image();+		clocks();+		return fails;+	}+	image();+	clocks();+	vvar_is_read_only();+	after_fork();+	after_exec((const char *)getauxval(AT_EXECFN));+	dynamic_is_enoexec();+	printf("vdsotest: %d failed\n", fails);+	return fails;+}

@@ -0,0 +1,180 @@+/*+ * vdso.c -- Rustbelt's vDSO ("virtual dynamic shared object", stage 19).+ *+ * A tiny shared library that the kernel maps into *every* process (it+ * isn't a file anywhere: its bytes are built into the kernel). It answers+ * "what time is it?" without a system call: a syscall means switching+ * into the kernel and back, which costs far more than reading a clock.+ *+ * How it knows the time: the kernel keeps a read-only page, the "vvar"+ * page, just in front of this library's code. In it are the numbers the+ * kernel itself uses to turn the CPU's time stamp counter (TSC, read with+ * `rdtsc`) into nanoseconds. We do the same arithmetic here, in user+ * space, and get exactly the answer the clock_gettime syscall would give.+ *+ * The kernel may change those numbers while we read them, so they are+ * guarded by a *seqlock*: the kernel makes `seq` odd, writes, makes it+ * even again. We read `seq`, copy the numbers, read `seq` again: if it was+ * odd or has changed, our copy may be half old/half new, so we retry.+ *+ * When the TSC can't be trusted (it must tick at a constant rate: an+ * "invariant TSC"), and for every clock we don't handle, we simply make+ * the real system call, so callers always get an answer.+ *+ * C library calls: musl and glibc look for these functions by name *and+ * version* ("LINUX_2.6", see vdso.lds) through the AT_SYSINFO_EHDR entry+ * of the auxiliary vector, which points at our ELF header.+ *+ * Rules for this code: no libc (we *are* below libc), no global writable+ * data (the page is shared by every process and read-only), and nothing+ * that needs relocating (it can be mapped at any address).+ */++typedef unsigned long u64;+typedef unsigned int u32;++/* Must match `VVar` in src/vdso.rs. */+struct vvar {+	volatile u32 seq;       /* odd while the kernel is writing */+	u32 flags;              /* VVAR_TSC_OK | VVAR_RDTSCP_CPU */+	u64 tsc_start;          /* TSC value at "boot" (time 0) */+	u64 mult;               /* nanoseconds per TSC tick, times 2^32 */+	u64 boot_realtime_ns;   /* Unix time at boot, in nanoseconds */+};+#define VVAR_TSC_OK 1u      /* the fast path may be used */+#define VVAR_RDTSCP_CPU 2u  /* rdtscp's TSC_AUX holds the CPU number */++/* The vvar page sits one page below our ELF header (vdso.lds). */+extern const struct vvar vvar_page __attribute__((visibility("hidden")));++struct timespec { long tv_sec; long tv_nsec; };+struct timeval { long tv_sec; long tv_usec; };+struct timezone { int tz_minuteswest; int tz_dsttime; };++#define SYS_gettimeofday 96+#define SYS_time 201+#define SYS_clock_gettime 228+#define SYS_getcpu 309++#define CLOCK_REALTIME 0+#define CLOCK_MONOTONIC 1+#define CLOCK_MONOTONIC_RAW 4+#define CLOCK_REALTIME_COARSE 5+#define CLOCK_MONOTONIC_COARSE 6++static inline long syscall3(long nr, long a, long b, long c)+{+	long ret;+	__asm__ volatile("syscall" : "=a"(ret) : "a"(nr), "D"(a), "S"(b), "d"(c) : "rcx", "r11", "memory");+	return ret;+}++static inline u64 rdtsc(void)+{+	u32 lo, hi;+	__asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi));+	return ((u64)hi << 32) | lo;+}++/* Keep the compiler from moving memory reads across this line. */+#define barrier() __asm__ volatile("" ::: "memory")++/*+ * Nanoseconds since boot (monotonic) and the boot time, read consistently.+ * Returns 0 if the fast path can't be used (then the caller makes the+ * syscall). Same formula as `time::now_ns` in src/time.rs.+ */+static int read_clock(u64 *mono_ns, u64 *boot_ns)+{+	const struct vvar *v = &vvar_page;+	u32 seq;+	do {+		seq = v->seq;+		barrier();+		if (seq & 1) {+			__asm__ volatile("pause");+			continue;+		}+		if (!(v->flags & VVAR_TSC_OK))+			return 0;+		u64 now = rdtsc();+		u64 start = v->tsc_start;+		u64 delta = now > start ? now - start : 0;+		*mono_ns = (u64)(((unsigned __int128)delta * v->mult) >> 32);+		*boot_ns = v->boot_realtime_ns;+		barrier();+	} while ((seq & 1) || v->seq != seq);+	return 1;+}++int __vdso_clock_gettime(int clock, struct timespec *ts)+{+	u64 mono, boot, ns;+	switch (clock) {+	case CLOCK_REALTIME:+	case CLOCK_REALTIME_COARSE:+	case CLOCK_MONOTONIC:+	case CLOCK_MONOTONIC_RAW:+	case CLOCK_MONOTONIC_COARSE:+		if (!read_clock(&mono, &boot))+			break;+		ns = mono;+		if (clock == CLOCK_REALTIME || clock == CLOCK_REALTIME_COARSE)+			ns += boot;+		ts->tv_sec = ns / 1000000000;+		ts->tv_nsec = ns % 1000000000;+		return 0;+	}+	/* Every other clock (CPU time, BOOTTIME, ...) and the slow path. */+	return syscall3(SYS_clock_gettime, clock, (long)ts, 0);+}++int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)+{+	u64 mono, boot;+	if (!read_clock(&mono, &boot))+		return syscall3(SYS_gettimeofday, (long)tv, (long)tz, 0);+	if (tv) {+		u64 us = (mono + boot) / 1000;+		tv->tv_sec = us / 1000000;+		tv->tv_usec = us % 1000000;+	}+	if (tz) {+		tz->tz_minuteswest = 0; /* UTC, like the kernel's own answer */+		tz->tz_dsttime = 0;+	}+	return 0;+}++long __vdso_time(long *t)+{+	u64 mono, boot;+	if (!read_clock(&mono, &boot))+		return syscall3(SYS_time, (long)t, 0, 0);+	long secs = (mono + boot) / 1000000000;+	if (t)+		*t = secs;+	return secs;+}++int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused)+{+	(void)unused;+	if (!(vvar_page.flags & VVAR_RDTSCP_CPU))+		return syscall3(SYS_getcpu, (long)cpu, (long)node, 0);+	/* The kernel put each CPU's number into its TSC_AUX register, which+	 * rdtscp hands back in ecx (Linux does the same). */+	u32 lo, hi, aux;+	__asm__ volatile("rdtscp" : "=a"(lo), "=d"(hi), "=c"(aux));+	if (cpu)+		*cpu = aux & 0xfff;+	if (node)+		*node = aux >> 12;+	return 0;+}++/* The plain names, like Linux's vDSO also offers. */+int clock_gettime(int, struct timespec *) __attribute__((weak, alias("__vdso_clock_gettime")));+int gettimeofday(struct timeval *, struct timezone *) __attribute__((weak, alias("__vdso_gettimeofday")));+long time(long *) __attribute__((weak, alias("__vdso_time")));+int getcpu(unsigned *, unsigned *, void *) __attribute__((weak, alias("__vdso_getcpu")));

@@ -0,0 +1,47 @@+/*+ * Linker script for the vDSO (see vdso.c). One read+execute segment that+ * starts with the ELF header, so the kernel maps the file's bytes as they+ * are. Modeled on Linux's arch/x86/entry/vdso/vdso-layout.lds.S.+ */+SECTIONS+{+	/* The vvar page is mapped right in front of us. */+	vvar_page = . - 4096;++	. = SIZEOF_HEADERS;++	.hash           : { *(.hash) }            :text+	.gnu.hash       : { *(.gnu.hash) }+	.dynsym         : { *(.dynsym) }+	.dynstr         : { *(.dynstr) }+	.gnu.version    : { *(.gnu.version) }+	.gnu.version_d  : { *(.gnu.version_d) }+	.gnu.version_r  : { *(.gnu.version_r) }+	.dynamic        : { *(.dynamic) }         :text :dynamic+	.rodata         : { *(.rodata*) }         :text+	.note           : { *(.note.*) }          :text+	.eh_frame_hdr   : { *(.eh_frame_hdr) }+	.eh_frame       : { KEEP (*(.eh_frame)) }+	.text           : { *(.text*) }++	/DISCARD/ : { *(.data*) *(.bss*) *(.got*) *(.plt*) *(.comment) }+}++PHDRS+{+	text    PT_LOAD    FLAGS(5) FILEHDR PHDRS; /* R + X */+	dynamic PT_DYNAMIC FLAGS(4);              /* R */+}++/* Every exported function carries the version LINUX_2.6, like Linux's. */+VERSION+{+	LINUX_2.6 {+	global:+		clock_gettime; __vdso_clock_gettime;+		gettimeofday;  __vdso_gettimeofday;+		time;          __vdso_time;+		getcpu;        __vdso_getcpu;+	local: *;+	};+}