rust-kernel

@@ -261,7 +261,7 @@ dependencies = [  [[package]] name = "rustbelt"-version = "0.21.0"+version = "0.22.0" dependencies = [  "bootloader",  "conquer-once",

@@ -1,20 +1,11 @@ [package] name = "rustbelt"-version = "0.21.0"+version = "0.22.0" edition = "2024"  [dependencies]-# Bootloader 0.9.x: the version used by the 2nd edition of "Writing an OS in Rust".-# `cargo bootimage` links it with our kernel into a bootable BIOS disk image.-# `map_physical_memory`: the bootloader maps *all* physical RAM somewhere in-# virtual memory and tells us where (BootInfo.physical_memory_offset), so we-# can read/write page tables, which live at physical addresses.-bootloader = { version = "0.9", features = ["map_physical_memory"] } volatile = "0.2.6"      # stops the compiler optimising away VGA buffer writes spin = "0.5.2"          # spinlock Mutex (no OS = no std::sync::Mutex)-x86_64 = "0.15"         # safe wrappers for x86 instructions/ports (hlt, outb, ...)-uart_16550 = "0.3"      # driver for the serial port (COM1)-pic8259 = "0.11"        # driver for the two chained 8259 interrupt controllers pc-keyboard = "0.7"     # turns PS/2 scancodes into key events / characters linked_list_allocator = "0.9" # ready-made heap allocator (fallback for ours) @@ -39,6 +30,19 @@ smoltcp = { version = "0.14", default-features = false, features = [     "socket-tcp", "socket-udp", "socket-icmp", "socket-dhcpv4", "socket-dns", ] } +# Stage 22: the PC-only crates. They are only built for x86_64; the arm64+# kernel (src/arch/aarch64/) uses none of them (and no new crates at all).+[target.'cfg(target_arch = "x86_64")'.dependencies]+# Bootloader 0.9.x: the version used by the 2nd edition of "Writing an OS in Rust".+# `cargo bootimage` links it with our kernel into a bootable BIOS disk image.+# `map_physical_memory`: the bootloader maps *all* physical RAM somewhere in+# virtual memory and tells us where (BootInfo.physical_memory_offset), so we+# can read/write page tables, which live at physical addresses.+bootloader = { version = "0.9", features = ["map_physical_memory"] }+x86_64 = "0.15"         # safe wrappers for x86 instructions/ports (hlt, outb, ...)+uart_16550 = "0.3"      # driver for the serial port (COM1)+pic8259 = "0.11"        # driver for the two chained 8259 interrupt controllers+ [dependencies.lazy_static] version = "1.0" features = ["spin_no_std"]@@ -59,6 +63,12 @@ stack-overflow = [] # address after boot and watch the page-fault handler report it. page-fault = [] +# Stage 22: `--features aarch64` builds the arm64 kernel binary+# (rustbelt-aarch64, see scripts/run-aarch64.sh). `aarch64-fault` makes it+# read an unmapped address on purpose, to check the exception handler.+aarch64 = []+aarch64-fault = ["aarch64"]+ # The library (src/lib.rs) holds all kernel code so integration tests in # tests/ can use it; src/main.rs is the small binary that boots it. [lib]@@ -69,6 +79,16 @@ name = "rustbelt" test = false   # all tests live in the library and tests/ bench = false +# Stage 22: the arm64 kernel. Only built with `--features aarch64` (and+# `--target aarch64-unknown-none-softfloat`), so `cargo build`, `cargo test`+# and `cargo bootimage` for the PC never see it.+[[bin]]+name = "rustbelt-aarch64"+path = "src/arch/aarch64/main.rs"+required-features = ["aarch64"]+test = false+bench = false+ # These two tests replace the test framework with their own `_start`. [[test]] name = "should_panic"

@@ -1152,6 +1152,10 @@ src/boot.rs               stage 21: boot contract v2: /sbin/init -> /bin/init -> user/inittest.c           stage 21: the test init on target/rootfs-gpt.img (tests/boot_v2.rs, scripts/boot-v2-check.sh) scripts/qemu.sh           stage 21: QEMU wrapper for cargo run/test: honours CARGO_TARGET_DIR, adds rootfs-gpt.img for boot_v2 scripts/boot-v2-check.sh  stage 21: boots the headless kernel with the GPT root image, wants QEMU exit 47, fscks both partitions+src/arch/                 stage 22: the arch interface (src/arch/mod.rs), x86_64/ wrappers and the aarch64/ bring-up+                          kernel (its own file list is in "A second CPU family: arm64")+scripts/run-aarch64.sh    stage 22: build the arm64 kernel and boot it on QEMU virt (cortex-a53, -semihosting)+scripts/aarch64-check.sh  stage 22: headless arm64 check: EL1, EL2 and fault boots, banner/MMU/DTB/ticks/exit status src/fs/lock.rs            stage 16: fcntl record locks (F_GETLK/F_SETLK/F_SETLKW), deadlock detection src/fpu.rs                stage 15.1: FPU/SSE/AVX on per CPU (CR0/CR4/XCR0), XSAVE or FXSAVE save/restore scripts/hvc0-session.txt  the commands run-headless.sh feeds into hvc0 (for `getty -1 /dev/hvc0`)@@ -5325,6 +5329,287 @@ Doesn't (yet):   mappings to write. - exFAT still has no rename, file times or dirty flag. +## A second CPU family: arm64 (stage 22)++Everything up to stage 21 runs on one kind of CPU: **x86_64**, the PC+family. Stage 22 starts a second one, **arm64** (also called AArch64 or+ARMv8-A), because the long-term hardware target is a **Raspberry Pi 3**,+whose BCM2837 chip has four ARM **Cortex-A53** cores.++This is a *bring-up*: the smallest kernel that proves the machine works.+It lives on its own git branch, `arm64`, so `master` stays the x86_64+kernel until Tyler decides to merge. It doesn't run on a Pi yet. It runs+on QEMU's **`virt`** board (a made-up but well-documented ARM machine)+with a **Cortex-A53** CPU, so it's the same CPU the Pi 3 has, with+simpler devices around it. When you run it, it:++1. starts at `_start`, and if the CPU started at EL2 it drops to EL1;+2. prints `Rustbelt 0.22.0 (stage 22: arm64 bring-up)` on the serial port;+3. installs the exception vectors;+4. turns on the MMU, with an identity map and a higher-half map;+5. reads the device tree QEMU hands it, and sets up a 1 MiB heap;+6. lists the virtio devices;+7. starts the interrupt controller and the timer and prints a few ticks;+8. ends the QEMU run with exit status 0.++Nothing else: no processes, no system calls, no scheduler, no shell.+Those are all still x86_64-only.++### What's different from x86_64++The ideas are the same; the names and the details change:++| Idea | x86_64 (the PC) | arm64 (this stage) |+|------|-----------------|--------------------|+| Who loads the kernel | the `bootloader` crate, from a disk image (`bootimage`) | QEMU itself: `-kernel rustbelt-aarch64` (a plain ELF file, our own linker script) |+| Privilege levels | rings 0 (kernel) and 3 (user) | **exception levels**: EL0 (user), EL1 (kernel), EL2 (hypervisor), EL3 (firmware) |+| Where faults and interrupts go | the IDT | the **vector table**, address in `VBAR_EL1` |+| "What went wrong?" | the exception number + error code, CR2 | **`ESR_EL1`** (the syndrome: its EC field says what kind), `ELR_EL1` (where), `FAR_EL1` (which address) |+| Page tables | CR3, 4 levels | `TTBR0_EL1` (the low half) and `TTBR1_EL1` (the high half), set up by `TCR_EL1` |+| Interrupt controller | 8259 PIC, later the APIC | the **GIC** (Generic Interrupt Controller), version 2 here |+| Timer | PIT, later the APIC timer | the **generic timer**, built into every core |+| Serial port | 16550 UART at I/O port 0x3F8 | **PL011** UART, memory-mapped at 0x0900_0000 |+| Talking to devices | I/O ports *and* memory-mapped registers | memory-mapped registers only |+| Finding the hardware | ACPI tables, PCI scan | the **device tree** (DTB) |+| Leaving QEMU | write to the `isa-debug-exit` port | **semihosting** `SYS_EXIT` |++A few words in that table, explained:++- **Exception level (EL).** Like rings, but numbered the other way+  round: higher is more privileged. A kernel runs at EL1. Some boards and+  QEMU modes start the CPU at EL2, which belongs to a hypervisor (a+  program that runs virtual machines). We don't want to be a hypervisor,+  so `_start` sets a few EL2 registers ("EL1 runs 64-bit code", "EL1 may+  use the timer") and uses `eret` to "return" into EL1. That's how you go+  *down* a level on ARM: pretend you're returning from an exception that+  came from there.+- **Device tree (DTB).** A small binary file that describes the board:+  how much RAM, where the UART is, which interrupt the timer uses. QEMU+  builds one and puts it at the start of RAM (0x4000_0000) for us; on a+  Pi, the firmware does the same. `src/arch/aarch64/dtb.rs` is a+  hand-written reader for the format (a header, then a stream of "begin+  node", "property", "end node" tokens).+- **Semihosting.** A way for a program to ask the debugger, or here+  QEMU, to do something for it. The instruction `hlt #0xf000` with 0x18+  (`SYS_EXIT`) in `w0` means "end the run", and QEMU exits with the status+  we pass. It only works when QEMU is started with **`-semihosting`**; on+  a real board the same instruction would just fault. (The other option+  would be PSCI `SYSTEM_OFF`, a firmware call that powers the machine+  off, but it can't carry a status code; semihosting can.)++### The boot, step by step++`src/arch/aarch64/boot.rs` is about 40 lines of assembly:++- Only core 0 continues; the others (none on `virt` with one CPU, but+  three on a Pi) go to sleep with `wfe`.+- It remembers the device tree address (in `x0`) and the level the CPU+  started at (`CurrentEL`), drops from EL2 to EL1 if needed, sets the+  stack pointer to the 64 KiB stack from the linker script, clears+  `.bss` (Rust expects zeroed statics), allows FP/SIMD instructions, and+  calls `kmain` in `src/arch/aarch64/main.rs`.++The **linker script** (`src/arch/aarch64/link.ld`) puts the kernel at+0x4020_0000 (2 MiB into RAM, after QEMU's device tree) with `_start`+first. `build.rs` notices it's building for aarch64, passes the script+to the linker, and skips everything else it does (the disk images and+user programs are for the PC kernel).++One trap before the MMU is on: the CPU treats *all* memory as device+memory, and device memory doesn't allow unaligned reads. Rust's+formatting code (`println!` with `{}`) can do unaligned reads, so the+banner is printed one byte at a time (`console::puts_raw`), and+everything with `{}` waits until the MMU is on.++### Exceptions++The vector table (`src/arch/aarch64/exceptions.rs`) has 16 slots of 128+bytes: four kinds (synchronous = caused by an instruction, IRQ, FIQ,+SError) for each of four situations (from EL1 using SP_EL0, from EL1,+from EL0 in 64-bit, from EL0 in 32-bit). Every slot saves all the+registers plus `ELR_EL1` and `SPSR_EL1` (a 272-byte frame) and calls one+Rust function with the slot number.++- An **IRQ** asks the GIC which interrupt it is, runs the timer handler,+  tells the GIC it's done, and returns with `eret`.+- Anything **synchronous** prints the syndrome and stops:++```+[exception] synchronous exception from EL1: ESR_EL1=0x96000005 (EC 0x25: data abort (bad memory access))+[exception] ELR_EL1=0x40209e58 FAR_EL1=0x1000000000 SPSR_EL1=0x400003c5+[exception] halting (QEMU exit status 3)+```++  Read it as: EC 0x25 = a data abort (a load or store went wrong); the+  instruction was at `ELR_EL1`; the address it tried was `FAR_EL1`+  (64 GiB, which isn't mapped). The low bits of ESR (0x05) say "a+  translation fault at level 1": there was no page table entry. To see+  this yourself, build with `--features aarch64-fault`+  (`scripts/run-aarch64.sh --fault`).++A **panic** prints its message and exits with status 2.++### The MMU++`src/arch/aarch64/paging.rs` keeps it as small as it can be:++- the **4 KiB granule** (page size), with **1 GiB blocks** at level 1, so+  one table of 512 entries maps 512 GiB and there are no lower-level+  tables at all;+- **two tables**: `TTBR0_EL1` maps addresses as themselves (the identity+  map: 0x4020_0000 is 0x4020_0000), `TTBR1_EL1` maps the same memory again+  at `0xffff_ff80_0000_0000 + physical` (the **higher half**);+- two memory types in `MAIR_EL1`: **Normal** (cached; the RAM) and+  **Device** (uncached, strictly ordered; the first 1 GiB, where `virt`+  puts its devices);+- device memory can't run code, and nothing is user-accessible.++The kernel still runs at its low address (moving it into the higher half+needs it linked there and a jump across; that's for later), but after+the MMU is on, it reads a word back through its high alias to prove the+map works, and from then on talks to the UART, the GIC and the virtio+devices through their higher-half addresses (`mmio::use_high_half`).++### Interrupts and the timer++- **GICv2** (`src/arch/aarch64/gic.rs`) has two parts: the+  *distributor* (0x0800_0000), where each interrupt is switched on and+  given a priority, and the *CPU interface* (0x0801_0000), where a core+  asks "which one is it?" (read `IAR`) and says "done" (write `EOIR`).+- The **generic timer** (`src/arch/aarch64/timer.rs`) counts at a fixed+  rate (`CNTFRQ_EL0`, 62.5 MHz on QEMU). We use the *virtual* timer: put+  "fire in N counts" into `CNTV_TVAL_EL0`, and it raises interrupt 27+  (PPI 11, an interrupt private to the core). The handler counts a tick+  and sets the next one. 100 times a second, like the PC's PIT.+- The addresses and the interrupt number are constants for `virt`, but+  the board also reads them from the device tree and prints them, so you+  can see they agree.++### The layout: `src/arch/`++```+src/arch/mod.rs                 the arch interface: entry, console, interrupts, timer, paging, exit+src/arch/x86_64/mod.rs          x86_64 implementation: thin wrappers around the existing code+src/arch/aarch64/main.rs        the arm64 kernel binary (`rustbelt-aarch64`): kmain+src/arch/aarch64/mod.rs         banner, current_el, exit (semihosting), halt+src/arch/aarch64/boot.rs        _start: park other cores, EL2 -> EL1, stack, .bss, kmain+src/arch/aarch64/link.ld        linker script: load at 0x4020_0000, _start first, 64 KiB stack+src/arch/aarch64/exceptions.rs  vector table, IRQ dispatch, the ESR/ELR/FAR report+src/arch/aarch64/paging.rs      MAIR/TCR/TTBR0/TTBR1, 1 GiB blocks, identity + higher half+src/arch/aarch64/gic.rs         GICv2 distributor + CPU interface+src/arch/aarch64/timer.rs       generic (virtual) timer at 100 Hz+src/arch/aarch64/console.rs     print!/println! on the board's UART; puts_raw for before the MMU+src/arch/aarch64/mmio.rs        register reads/writes, low or higher-half addresses+src/arch/aarch64/dtb.rs         a small device tree (FDT) reader+src/arch/aarch64/board/mod.rs   the Board, Uart and IrqController traits+src/arch/aarch64/board/pl011.rs the PL011 UART+src/arch/aarch64/board/virt.rs  QEMU virt: constants, checked against the device tree+src/arch/aarch64/board/raspi3.rs Raspberry Pi 3: addresses and TODOs only, not used yet+```++The interface is six small pieces, and each CPU family fills them in:+**entry** (how the kernel starts), **console** (print a string),+**interrupts** (install handlers, unmask), **timer** (ticks),+**paging** (turn on the MMU) and **exit** (end the QEMU run with a+status). The **board** layer sits below that on arm64, because the same+CPU can sit on very different boards: the board says where the UART and+the interrupt controller are and which interrupt the timer uses.++**What's x86_64-only for now.** Every module that existed before stage 22+(memory, processes, system calls, file systems, drivers, networking...)+is behind `#[cfg(target_arch = "x86_64")]` in `src/lib.rs`, so an arm64+build simply doesn't see them, and the x86_64 build is exactly what it+was. Only `src/arch/` is built for both. The x86-only crates+(`bootloader`, `x86_64`, `uart_16550`, `pic8259`) moved to a+`[target.'cfg(target_arch = "x86_64")'.dependencies]` table in+`Cargo.toml`; nothing new was added (the arm64 heap uses the+`linked_list_allocator` crate we already had, and all the arm64 register+code is hand-written inline assembly).++### Running it++You need two Debian packages: `qemu-system-arm` (which has+`qemu-system-aarch64`) and `gcc-aarch64-linux-gnu` (for `objdump` /+`readelf` / `nm` on the kernel; Rust doesn't need it to build). Then:++```+scripts/run-aarch64.sh                # build and boot (Ctrl-A X quits QEMU if it hangs)+scripts/run-aarch64.sh --el2          # start at EL2 to see the drop to EL1+scripts/run-aarch64.sh --fault        # the deliberate data abort, exit status 3+scripts/run-aarch64.sh -- -device virtio-rng-device   # extra QEMU options after --+scripts/aarch64-check.sh              # the headless check (below)+```++Under the hood that's:++```+cargo build --bin rustbelt-aarch64 --features aarch64 --target aarch64-unknown-none-softfloat+qemu-system-aarch64 -M virt -cpu cortex-a53 -m 512M -nographic -semihosting \+    -kernel target/aarch64-unknown-none-softfloat/debug/rustbelt-aarch64+```++The `aarch64` feature matters: the `rustbelt-aarch64` binary needs it,+so a plain `cargo build` or `cargo test` (the PC kernel) skips it. Don't+use `cargo run` for arm64; it would hand the ELF to the PC's `bootimage`+runner.++A whole run looks like this:++```+Rustbelt 0.22.0 (stage 22: arm64 bring-up)+[boot] QEMU virt board, Cortex-A53; entered at EL1, running at EL1+[boot] exception vectors installed (VBAR_EL1)+[mmu] on (SCTLR_EL1.M = 1): 4 KiB granule, identity map + higher half at 0xffffff8000000000; a word at 0x40337030 reads 0x5ca1ab1e through 0xffffff8040337030+[mmu] devices are now reached through the higher half (this line went to the UART at 0xffffff8009000000)+[heap] 1024 KiB at 0x40237000; a Vec of 1..=1000 sums to 500500+[dtb] device tree at 0x40000000, 1048576 bytes+[dtb] memory@40000000: RAM at 0x40000000, 512 MiB+[dtb] pl011@9000000: PL011 UART at 0x9000000+[dtb] intc@8000000: GICv2, distributor at 0x8000000, CPU interface at 0x8010000+[dtb] timer: generic timer, virtual timer = PPI 11 (INTID 27)+[virtio] 32 virtio-mmio slots, 1 in use+[virtio] 0xa003e00: device ID 4 (entropy source (rng)), virtio-mmio version 1+[timer] generic timer at 62500000 Hz, ticking 100 times a second (INTID 27)+[timer] tick 1+...+[timer] tick 5+[timer] 100 ticks took 1025 ms (should be about 1000)+[exit] bring-up done; leaving QEMU through semihosting SYS_EXIT, status 0+```++### Testing it++The arm64 kernel isn't part of `cargo test` (that's the PC kernel's+test suite, and it still has to pass on this branch: 198 tests).+`scripts/aarch64-check.sh` boots it three times, headless, each under a+30-second `timeout`, and checks the output:++| Boot | What it checks |+|------|----------------|+| normal, at EL1, with a virtio rng and a virtio block device | the banner, EL1, vectors, the MMU line and the higher-half read-back, the UART through the higher half, the heap, the GIC and the timer interrupt from the device tree, both virtio devices listed, tick 5, 100 ticks timed, the exit line, and QEMU exit status **0** |+| at EL2 (`-M virt,virtualization=on`) | "entered at EL2, running at EL1", ticks, exit status 0 |+| built with `--features aarch64-fault` | an `EC 0x25` data abort with `ELR_EL1` in the kernel and `FAR_EL1=0x1000000000`, exit status **3** |++It prints one PASS/FAIL line per check and the PID of each QEMU it+started (it only ever stops those, through `timeout`).++### What stage 22 does and doesn't do++Does: everything above, on QEMU `virt` with a Cortex-A53.++Doesn't (yet):++- **No Raspberry Pi.** `board/raspi3.rs` holds the Pi 3's addresses and+  TODOs; see "Hardware roadmap" below.+- The kernel **runs at its low address**; only devices use the higher+  half. Only **1 GiB blocks**, no 4 KiB pages yet, and no frame+  allocator (the heap is a fixed 1 MiB array).+- **One core.** No user mode (EL0), no system calls, no scheduler, no+  shell: everything after the bring-up is still x86_64-only.+- **Semihosting only works under QEMU** (with `-semihosting`). On real+  hardware "exit" would have to be a PSCI call or a loop.+- virtio devices are only **listed**, not driven.+ ## Roadmap  Done: all 12 chapters of the 2nd edition, plus preemptive kernel threads,@@ -5383,9 +5668,61 @@ timers, user ids), and Unix sockets with `/proc/self` (stage 18). - [x] The vDSO, static PIE and mremap (stage 19): `__vdso_clock_gettime`/`gettimeofday`/`time`/`getcpu` with a seqlocked vvar page and `AT_SYSINFO_EHDR`, a nanosecond kernel clock, ET_DYN programs without PT_INTERP at a random base (`AT_PHDR`/`AT_ENTRY`/`AT_BASE`, `PT_GNU_STACK`/`PT_TLS`/`PT_GNU_RELRO`), musl-gcc C programs in the build, `mremap` with `MREMAP_MAYMOVE`/`MREMAP_FIXED`, the threads flake fixed, Linux device numbers and `ttyname` — [man 7 vdso](https://man7.org/linux/man-pages/man7/vdso.7.html), [man 2 mremap](https://man7.org/linux/man-pages/man2/mremap.2.html) - [x] Plain static programs and timerfd (stage 20): the kernel moved to `0x200_0000_0000` so user space starts at 4 MiB and stock `musl-gcc -static` ET_EXEC binaries run unmodified; `timerfd_create`/`settime`/`gettime` on the existing timer table, readable with `read`/poll/select/epoll - [x] Finding the root and starting init (stage 21): the GPT "Linux root (x86-64)" partition mounted at `/`, `/sbin/init` -> `/bin/init` -> `/bin/sh` as pid 1 with build-time argv/env knobs and its exit status passed to QEMU, an exFAT "Microsoft basic data" partition at `/mnt/share` (read-write), `msync`, `RUSTBELT_DISK0` and `CARGO_TARGET_DIR` everywhere — [Discoverable Partitions Specification](https://uapi-group.org/specifications/specs/discoverable_partitions_specification/), [man 2 msync](https://man7.org/linux/man-pages/man2/msync.2.html)+- [x] A second CPU family, arm64 (stage 22, on the `arm64` branch): `src/arch/{x86_64,aarch64}` with a small interface, a `-kernel` ELF with its own linker script on QEMU `virt` with a Cortex-A53, EL2 -> EL1, exception vectors with an ESR/ELR/FAR report, MMU on (identity + higher half), a device tree reader, GICv2 and the generic timer, semihosting exit, a board layer with QEMU virt and a Raspberry Pi 3 skeleton — [Arm: Learn the architecture (exception model, MMU, GIC, generic timer)](https://developer.arm.com/documentation/102412/latest/), [OSDev: Raspberry Pi Bare Bones](https://wiki.osdev.org/Raspberry_Pi_Bare_Bones), [Devicetree specification](https://www.devicetree.org/specifications/), [QEMU: virt board](https://www.qemu.org/docs/master/system/arm/virt.html)+- [ ] arm64 on the Raspberry Pi 3 (see "Hardware roadmap" below) - [ ] The kernel/OS split and a musl port (next; waits for Tyler's go; see the missing-syscall list in "musl prerequisites") - [ ] A Hyprland-style tiling compositor on the framebuffer (later) +### Hardware roadmap: Raspberry Pi 3, then 4, then 5++The first real board is the **Raspberry Pi 3** (BCM2837, four Cortex-A53+cores, 1 GiB of RAM). Stage 22 got the CPU going on QEMU `virt`; the Pi 3+has the same CPU but different devices around it. In order:++1. **QEMU `raspi3b` first.** QEMU has a Pi 3 model (`-M raspi3b`), so the+   Pi 3 board code can be tested headless like everything else before it+   touches real hardware. It loads the kernel at 0x8_0000 and starts all+   four cores (we park three), at EL2.+2. **Memory map.** RAM from 0, the chip's peripherals at+   0x3F00_0000-0x3FFF_FFFF, the ARM "local" peripherals at 0x4000_0000.+   Peripherals sit *inside* the first 1 GiB, so the MMU needs 2 MiB+   blocks (one more table level) to make RAM Normal and the peripherals+   Device. The device tree from the firmware (or QEMU) has the details.+3. **Serial.** Two UARTs: the **PL011** (UART0, 0x3F20_1000), the same+   kind as on `virt` (the `Pl011` driver is shared), and the **mini-UART**+   (0x3F21_5040), whose speed depends on the GPU's clock. On a real Pi 3+   the PL011 is wired to Bluetooth unless `config.txt` swaps them, and+   both need their GPIO pins (14/15) set up.+4. **Interrupts and the timer.** No GIC on the Pi 3. Instead, the+   **BCM2835 interrupt controller** (0x3F00_B200: "pending" and "enable"+   registers for the chip's devices), plus the **ARM local interrupt+   controller** (0x4000_0000), which routes each core's generic timer+   (`LOCAL_TIMER_INT_CTRL0`, `LOCAL_IRQ_SOURCE0`). The generic timer code+   from stage 22 stays; only who delivers its interrupt changes. That's+   what the `IrqController` trait in `board/` is for.+5. **Screen.** The **mailbox** (0x3F00_B880) is how the ARM cores ask the+   GPU for things; one request is "give me a framebuffer of this size",+   and the answer is an address to draw pixels at. The framebuffer+   console from the virtio-gpu stage can then draw there.+6. **SD card.** The **EMMC** controller (0x3F30_0000, SDHCI-style) or the+   simpler **SDHOST** (0x3F20_2000). Then the existing `BlockDevice`+   trait, partitions and file systems work unchanged.+7. **Real hardware.** `objcopy` the ELF to `kernel8.img`, link it at+   0x8_0000, put it on the SD card's FAT partition with the Pi firmware+   and `arm_64bit=1` in `config.txt`, and watch the serial pins with a+   USB-serial cable.+8. **Much later: USB and networking.** The Pi 3's Ethernet is a+   **LAN9514** chip on its **USB** bus (DWC2 controller at+   0x3F98_0000), so networking needs a USB host stack first: a big job.++**Raspberry Pi 4** (BCM2711, Cortex-A72): a **GIC-400**, which is a GICv2,+so `gic.rs` works there; peripherals move to 0xFE00_0000, and the SD+controller changes. QEMU has a basic `raspi4b` model.++**Raspberry Pi 5** (BCM2712, Cortex-A76): also a GIC; most of its I/O+sits behind a separate I/O chip, and QEMU has no model of it, so it comes+last.+ ### Filesystem roadmap (Tyler's plan)  Everything below plugs into the VFS above: a filesystem is a `FileSystem`

@@ -140,6 +140,15 @@ const PART_SECTORS: u64 = DISK_SECTORS - 2 * PART_START;  fn main() {     let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());+    // Stage 22: the arm64 kernel needs none of the disks or user programs+    // (they're x86_64 programs for the PC kernel), just its linker script.+    if env::var("CARGO_CFG_TARGET_ARCH").as_deref() == Ok("aarch64") {+        let ld = manifest.join("src/arch/aarch64/link.ld");+        println!("cargo:rerun-if-changed={}", ld.display());+        println!("cargo:rerun-if-changed=build.rs");+        println!("cargo:rustc-link-arg-bin=rustbelt-aarch64=-T{}", ld.display());+        return;+    }     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());     let user_dir = manifest.join("user");     println!("cargo:rerun-if-changed=user");

@@ -0,0 +1,75 @@+#!/usr/bin/env bash+# Stage 22: headless check of the arm64 bring-up on QEMU virt (Cortex-A53).+# Not part of `cargo test` (that's the x86_64 kernel's). Three boots:+#   1. normal, starting at EL1, with a virtio rng + block device attached:+#      banner, MMU, DTB, heap, virtio, 5 ticks, exit status 0+#   2. the same kernel starting at EL2 (virtualization=on): drops to EL1+#   3. built with --features aarch64-fault: a data abort is reported+#      (ESR/ELR/FAR) and the run ends with status 3+# Logs go to $LOGDIR (default target/aarch64-check). Exit 0 = all good.+set -uo pipefail+cd "$(dirname "$0")/.."+: "${CARGO_TARGET_DIR:=$PWD/target}"+export CARGO_TARGET_DIR+TRIPLE=aarch64-unknown-none-softfloat+LOGDIR="${LOGDIR:-$CARGO_TARGET_DIR/aarch64-check}"+mkdir -p "$LOGDIR"+FAIL=0+pass() { echo "PASS  $1"; }+fail() { echo "FAIL  $1"; FAIL=1; }+expect() { # expect <log> <description> <grep -E pattern>+    if grep -qE "$3" "$1"; then pass "$2"; else fail "$2 (no /$3/ in $1)"; fi+}++# The fault kernel first (it's a different build), kept as a copy...+cargo build --bin rustbelt-aarch64 --features aarch64-fault --target "$TRIPLE" 2>"$LOGDIR/build-fault.log" \+    || { echo "FAIL  build (fault)"; cat "$LOGDIR/build-fault.log"; exit 1; }+cp "$CARGO_TARGET_DIR/$TRIPLE/debug/rustbelt-aarch64" "$LOGDIR/rustbelt-aarch64-fault.elf"+# ...then the normal one, which stays in place for run-aarch64.sh.+cargo build --bin rustbelt-aarch64 --features aarch64 --target "$TRIPLE" 2>"$LOGDIR/build.log" \+    || { echo "FAIL  build"; cat "$LOGDIR/build.log"; exit 1; }++DISK="$LOGDIR/blank.img"+[ -f "$DISK" ] || truncate -s 1M "$DISK"++# 1. Normal boot at EL1.+RB_TIMEOUT=30 scripts/run-aarch64.sh --no-build -- \+    -device virtio-rng-device \+    -drive if=none,id=d0,format=raw,file="$DISK" -device virtio-blk-device,drive=d0 \+    >"$LOGDIR/el1.log" 2>&1+rc=$?+L="$LOGDIR/el1.log"+[ $rc = 0 ] && pass "EL1 boot: QEMU exit status 0" || fail "EL1 boot: QEMU exit status $rc (want 0)"+expect "$L" "banner" '^Rustbelt 0\.22\.0 \(stage 22: arm64 bring-up\)'+expect "$L" "at EL1" 'entered at EL1, running at EL1'+expect "$L" "exception vectors" 'exception vectors installed'+expect "$L" "MMU on, higher half reads back" '\[mmu\] on \(SCTLR_EL1\.M = 1\).*reads 0x5ca1ab1e'+expect "$L" "UART through the higher half" 'UART at 0xffffff8009000000'+expect "$L" "heap" 'sums to 500500'+expect "$L" "DTB: GICv2 found" '\[dtb\] .*GICv2, distributor at 0x8000000'+expect "$L" "DTB: timer INTID 27" 'virtual timer = PPI 11 \(INTID 27\)'+expect "$L" "virtio: rng listed" 'device ID 4 \(entropy'+expect "$L" "virtio: block listed" 'device ID 2 \(block'+expect "$L" "timer ticks 1..5" '\[timer\] tick 5'+expect "$L" "100 ticks timed" '100 ticks took [0-9]+ ms'+expect "$L" "clean exit line" 'leaving QEMU through semihosting SYS_EXIT, status 0'++# 2. Starting at EL2.+RB_TIMEOUT=30 scripts/run-aarch64.sh --no-build --el2 >"$LOGDIR/el2.log" 2>&1+rc=$?+L="$LOGDIR/el2.log"+[ $rc = 0 ] && pass "EL2 boot: QEMU exit status 0" || fail "EL2 boot: QEMU exit status $rc (want 0)"+expect "$L" "EL2 -> EL1 drop" 'entered at EL2, running at EL1'+expect "$L" "EL2 boot ticks" '\[timer\] tick 5'++# 3. The deliberate data abort.+RB_TIMEOUT=30 RB_KERNEL="$LOGDIR/rustbelt-aarch64-fault.elf" scripts/run-aarch64.sh --no-build >"$LOGDIR/fault.log" 2>&1+rc=$?+L="$LOGDIR/fault.log"+[ $rc = 3 ] && pass "fault: QEMU exit status 3" || fail "fault: QEMU exit status $rc (want 3)"+expect "$L" "fault: ESR data abort" 'ESR_EL1=0x96[0-9a-f]{6} \(EC 0x25: data abort'+expect "$L" "fault: ELR/FAR" 'ELR_EL1=0x402[0-9a-f]+ FAR_EL1=0x1000000000'++grep -h '^\[run-aarch64\] qemu pid' "$LOGDIR"/el1.log "$LOGDIR"/el2.log "$LOGDIR"/fault.log+if [ $FAIL = 0 ]; then echo "aarch64-check: ALL PASS (logs in $LOGDIR)"; else echo "aarch64-check: FAILED (logs in $LOGDIR)"; fi+exit $FAIL

@@ -0,0 +1,59 @@+#!/usr/bin/env bash+# Stage 22: build the arm64 kernel and boot it on QEMU's "virt" board+# with a Cortex-A53 (the Raspberry Pi 3's CPU).+#+#   scripts/run-aarch64.sh                 # build + boot, output on this terminal+#   scripts/run-aarch64.sh --el2           # start at EL2 (virtualization=on)+#   scripts/run-aarch64.sh --fault         # build with --features aarch64-fault+#   scripts/run-aarch64.sh --no-build ...  # boot what's already built+#   RB_KERNEL=path/to/elf scripts/run-aarch64.sh --no-build   # boot another ELF+#   extra QEMU options go after `--`, e.g.  -- -device virtio-rng-device+#+# The kernel ends the run itself with semihosting SYS_EXIT (that's why+# -semihosting is on), and this script exits with the kernel's status:+# 0 = good, 2 = panic, 3 = exception. A hung kernel is stopped by+# `timeout` after RB_TIMEOUT seconds (default 30) -> status 124.+# Only the QEMU this script started is ever stopped.+set -euo pipefail+cd "$(dirname "$0")/.."+: "${CARGO_TARGET_DIR:=$PWD/target}"+export CARGO_TARGET_DIR+TRIPLE=aarch64-unknown-none-softfloat+MACHINE=virt+FEATURES=aarch64+BUILD=1+EXTRA=()+while [ $# -gt 0 ]; do+    case "$1" in+        --el2) MACHINE=virt,virtualization=on ;;+        --fault) FEATURES=aarch64-fault ;;+        --no-build) BUILD=0 ;;+        --) shift; EXTRA=("$@"); break ;;+        *) echo "unknown option $1" >&2; exit 64 ;;+    esac+    shift+done+if [ "$BUILD" = 1 ]; then+    cargo build --bin rustbelt-aarch64 --features "$FEATURES" --target "$TRIPLE" >&2+fi+KERNEL="${RB_KERNEL:-$CARGO_TARGET_DIR/$TRIPLE/debug/rustbelt-aarch64}"+[ -f "$KERNEL" ] || { echo "no kernel at $KERNEL" >&2; exit 1; }++timeout --foreground "${RB_TIMEOUT:-30}" \+    qemu-system-aarch64 -M "$MACHINE" -cpu cortex-a53 -m 512M \+        -nographic -monitor none -serial stdio -semihosting \+        -kernel "$KERNEL" "${EXTRA[@]}" </dev/null &+TPID=$!+QPID=+for _ in $(seq 40); do+    QPID=$(pgrep -P "$TPID" || true)+    [ -n "$QPID" ] && break+    kill -0 "$TPID" 2>/dev/null || break+    sleep 0.05+done+echo "[run-aarch64] qemu pid ${QPID:-?} (under timeout pid $TPID)" >&2+set +e+wait "$TPID"+rc=$?+echo "[run-aarch64] exit status $rc" >&2+exit $rc

@@ -89,7 +89,7 @@ set -e # Lines the shell self-test must have printed (only for a plain boot). if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     expect=(-        "Rustbelt 0.21.0 (stage 21: root partition and init) x86_64" # ver+        "Rustbelt 0.22.0 (stage 22: arm64 bring-up) x86_64" # ver         "[smp] 4 CPUs online"                          # smp.rs started 3 more cores         "[fpu] "                                       # XSAVE (-cpu max) or FXSAVE (qemu64)         "B:  /mnt/disk0   fat32"                       # drives@@ -160,7 +160,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         "[nbtest] all checks passed"         "[fputest] all checks passed"                  # stage 15.1: musl prerequisites         "[pathtest] all checks passed"-        "rawabi: uname: Rustbelt 0.21.0"               # a no-libc gcc binary+        "rawabi: uname: Rustbelt 0.22.0"               # a no-libc gcc binary         "rawabi: all checks passed"         "sh: nosuchprogram: command not found"         "cd: there is no drive Q:"@@ -185,7 +185,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then         echo "[run-headless] MISSING: frames back at the baseline after the shell"; missing=1     fi     # The other direction: the host fetched a page from httpd in the guest.-    if grep -qF "Hello from Rustbelt! Rustbelt 0.21.0" "$CURL_OUT" 2>/dev/null; then+    if grep -qF "Hello from Rustbelt! Rustbelt 0.22.0" "$CURL_OUT" 2>/dev/null; then         echo "[run-headless] host curl got httpd's page: $(head -1 "$CURL_OUT")"     else         echo "[run-headless] MISSING: host curl through hostfwd to httpd"; missing=1@@ -197,7 +197,7 @@ if [ -z "${1:-}" ] && [ "$code" = 33 ]; then     # drop the '\r's before comparing whole lines.     tr -d '\r' < "$HVC_OUT" > "$HVC_OUT.txt" 2>/dev/null && mv "$HVC_OUT.txt" "$HVC_OUT"     # The shell on hvc0 ran the host's commands and answered over hvc0.-    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.21.0 (stage 21" \+    for line in "Rustbelt getty: shell on this terminal" "Rustbelt 0.22.0 (stage 22" \                 "$ echo hi over hvc0" "hi over hvc0" "fb0" "hvc0"; do         grep -qF -- "$line" "$HVC_OUT" 2>/dev/null || { echo "[run-headless] MISSING on hvc0: $line"; missing=1; }     done

@@ -46,7 +46,7 @@ fputest | grep passed pathtest | grep passed rawabi | grep -v "ok   " TXT-    set -- "Rustbelt 0.21.0 (stage 21: root partition and init) x86_64" "tile: 4 windows on 1024x768, console 61x46" \+    set -- "Rustbelt 0.22.0 (stage 22: arm64 bring-up) x86_64" "tile: 4 windows on 1024x768, console 61x46" \         "^C" "(interrupted: SIGINT)" "threads: 4 threads summed 1..=40000000 = 800000020000000" \         "echoserver: epoll: 3 clients, 9 echoes" "[threads] all checks passed" \         "[fputest] all checks passed" "[pathtest] all checks passed" \

@@ -0,0 +1,57 @@+//! What differs from board to board, behind three small traits. The rest of+//! the arm64 code only talks to `board::current()`.+//!+//! - `virt`: QEMU's generic Arm board. Implemented and tested; addresses+//!   come from its device tree, with the usual values as a fallback.+//! - `raspi3`: the Raspberry Pi 3 (BCM2837). A skeleton: the addresses are+//!   written down, the drivers are TODO (see the README's Pi roadmap).++pub mod pl011;+pub mod raspi3;+pub mod virt;++/// A serial port: one byte out.+pub trait Uart: Sync {+    fn putc(&self, byte: u8);+}++/// An interrupt controller.+pub trait IrqController: Sync {+    /// Reset it and switch it on (all interrupts still disabled).+    fn init(&self);+    /// Let interrupt number `irq` through.+    fn enable(&self, irq: u32);+    /// Which interrupt is this? (`None`: a spurious one, nothing to do.)+    fn ack(&self) -> Option<u32>;+    /// Done with `irq`.+    fn eoi(&self, irq: u32);+}++/// What kind of memory a 1 GiB block of physical addresses holds.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub enum Kind {+    /// RAM: cached, can hold code.+    Normal,+    /// Device registers: uncached, in order, never executed.+    Device,+}++/// A board.+pub trait Board: Sync {+    fn name(&self) -> &'static str;+    fn uart(&self) -> &dyn Uart;+    fn irq(&self) -> &dyn IrqController;+    /// The interrupt number of the EL1 virtual timer at this controller.+    fn timer_irq(&self) -> u32;+    /// The physical memory map, in 1 GiB blocks: (first address, kind).+    fn memory_map(&self) -> &'static [(u64, Kind)];+    /// Read the device tree at `dtb` (if any) and use what it says.+    /// Returns lines for the boot log.+    fn configure(&self, dtb: Option<&super::dtb::Fdt>) -> alloc::vec::Vec<alloc::string::String>;+}++/// The board this kernel was built for. (Only `virt` for now; picking the+/// Pi 3 will be a cargo feature once its drivers exist.)+pub fn current() -> &'static dyn Board {+    &virt::VIRT+}

@@ -0,0 +1,47 @@+//! The Arm PL011 UART: QEMU virt's serial port, and the Pi 3's "UART0"+//! (which the Pi wires to Bluetooth by default; the mini-UART is the other+//! one). QEMU needs no setup at all; on real hardware the firmware has set+//! the baud rate, so `init` only makes sure it's on, 8 bits, FIFOs on.++use super::Uart;+use crate::arch::aarch64::mmio;+use core::sync::atomic::{AtomicUsize, Ordering};++const DR: usize = 0x00; // data register+const FR: usize = 0x18; // flags+const FR_TXFF: u32 = 1 << 5; // transmit FIFO full+const LCRH: usize = 0x2c; // line control+const CR: usize = 0x30; // control++pub struct Pl011 {+    base: AtomicUsize, // physical address+}++impl Pl011 {+    pub const fn new(base: usize) -> Self {+        Pl011 { base: AtomicUsize::new(base) }+    }+    pub fn base(&self) -> usize {+        self.base.load(Ordering::Relaxed)+    }+    pub fn set_base(&self, base: usize) {+        self.base.store(base, Ordering::Relaxed);+    }+    /// 8N1 with FIFOs; transmit + receive + UART enable.+    pub fn init(&self) {+        let b = self.base();+        mmio::write32(b + CR, 0);+        mmio::write32(b + LCRH, (3 << 5) | (1 << 4));+        mmio::write32(b + CR, (1 << 9) | (1 << 8) | 1);+    }+}++impl Uart for Pl011 {+    fn putc(&self, byte: u8) {+        let b = self.base();+        while mmio::read32(b + FR) & FR_TXFF != 0 {+            core::hint::spin_loop();+        }+        mmio::write32(b + DR, byte as u32);+    }+}

@@ -0,0 +1,94 @@+//! The Raspberry Pi 3 (BCM2837, four Cortex-A53 cores): a **skeleton**.+//! Nothing here is used or tested yet; it writes down where things are so+//! the next stages have a place to start (see "Raspberry Pi roadmap" in+//! the README). QEMU can emulate it too: `qemu-system-aarch64 -M raspi3b`.+//!+//! How it differs from QEMU's virt:+//!+//! ```text+//! 0x0000_0000  RAM (1 GiB minus what the GPU keeps; the firmware loads+//!              kernel8.img at 0x8_0000 and passes the DTB in x0)+//! 0x3f00_0000  peripherals ("0x7e..." in Broadcom's manual = 0x3f... here)+//!   0x3f00_b200  the BCM2835 interrupt controller (not a GIC!)+//!   0x3f00_b880  the mailbox (ask the GPU for a framebuffer)+//!   0x3f20_1000  PL011 UART0 (same driver as virt)+//!   0x3f21_5040  the mini-UART (UART1, a different, simpler chip)+//!   0x3f30_0000  EMMC SD card controller (0x3f20_2000: SDHOST, the other one)+//!   0x3f98_0000  USB (DWC2); networking is a LAN9514 on that USB bus+//! 0x4000_0000  the "ARM local" block: per-core timer interrupts, mailboxes+//! ```+//!+//! So RAM and devices share the first GiB: the 1 GiB blocks paging.rs+//! uses for virt aren't fine enough, and the Pi needs 2 MiB blocks (TODO).++use super::pl011::Pl011;+use super::{Board, IrqController, Kind, Uart};+use crate::arch::aarch64::dtb::Fdt;+use alloc::string::String;+use alloc::vec::Vec;++pub const PERIPHERAL_BASE: usize = 0x3f00_0000;+pub const UART0_BASE: usize = PERIPHERAL_BASE + 0x20_1000;+pub const MINI_UART_BASE: usize = PERIPHERAL_BASE + 0x21_5040;+pub const IRQ_BASE: usize = PERIPHERAL_BASE + 0xb200;+pub const MAILBOX_BASE: usize = PERIPHERAL_BASE + 0xb880;+pub const EMMC_BASE: usize = PERIPHERAL_BASE + 0x30_0000;+pub const SDHOST_BASE: usize = PERIPHERAL_BASE + 0x20_2000;+pub const USB_BASE: usize = PERIPHERAL_BASE + 0x98_0000;+pub const LOCAL_BASE: usize = 0x4000_0000;+/// ARM local block: core 0's timer interrupt control (CNTVIRQ is bit 3).+pub const LOCAL_TIMER_INT_CTRL0: usize = LOCAL_BASE + 0x40;+/// ARM local block: core 0's interrupt source register.+pub const LOCAL_IRQ_SOURCE0: usize = LOCAL_BASE + 0x60;++/// The BCM2835 interrupt controller plus the ARM local block. TODO: the+/// timer goes through the local block (set bit 3 of LOCAL_TIMER_INT_CTRL0,+/// read LOCAL_IRQ_SOURCE0); devices through the BCM2835 pending registers.+pub struct Bcm2835Irq;++impl IrqController for Bcm2835Irq {+    fn init(&self) {+        // TODO(Pi 3): disable everything (IRQ_BASE + 0x1c/0x20/0x24).+    }+    fn enable(&self, _irq: u32) {+        // TODO(Pi 3): local timer -> LOCAL_TIMER_INT_CTRL0; devices -> enable registers.+    }+    fn ack(&self) -> Option<u32> {+        None // TODO(Pi 3): read LOCAL_IRQ_SOURCE0, then the pending registers+    }+    fn eoi(&self, _irq: u32) {+        // Nothing to do on this controller: the device clears its own line.+    }+}++pub struct Raspi3 {+    uart: Pl011,+    irq: Bcm2835Irq,+}++/// Not selected by `board::current()` yet.+pub static RASPI3: Raspi3 = Raspi3 { uart: Pl011::new(UART0_BASE), irq: Bcm2835Irq };++/// TODO(Pi 3): needs 2 MiB granularity (RAM up to 0x3f00_0000, devices after).+static MAP: [(u64, Kind); 1] = [(0, Kind::Normal)];++impl Board for Raspi3 {+    fn name(&self) -> &'static str {+        "Raspberry Pi 3 (skeleton)"+    }+    fn uart(&self) -> &dyn Uart {+        &self.uart+    }+    fn irq(&self) -> &dyn IrqController {+        &self.irq+    }+    fn timer_irq(&self) -> u32 {+        3 // CNTVIRQ's bit in LOCAL_IRQ_SOURCE0+    }+    fn memory_map(&self) -> &'static [(u64, Kind)] {+        &MAP+    }+    fn configure(&self, _dtb: Option<&Fdt>) -> Vec<String> {+        Vec::from([String::from("raspi3: skeleton only (TODO: read the DTB the firmware passes in x0)")])+    }+}

@@ -0,0 +1,109 @@+//! QEMU's `virt` board (`qemu-system-aarch64 -M virt -cpu cortex-a53`).+//!+//! Its memory map (QEMU's hw/arm/virt.c), all inside the first 2 GiB:+//!+//! ```text+//! 0x0800_0000  GICv2 distributor (GICD)       } the first GiB:+//! 0x0801_0000  GICv2 CPU interface (GICC)     } devices+//! 0x0900_0000  PL011 UART                     }+//! 0x0a00_0000  32 virtio-mmio slots, 0x200 each+//! 0x4000_0000  RAM (128 MiB by default)       } the second GiB: RAM+//!              QEMU puts the device tree here for an ELF kernel;+//!              we load at 0x4020_0000 (link.ld)+//! ```+//!+//! `configure` reads the real addresses from the device tree; the constants+//! below are only the fallback (and what the first line is printed with).++use super::pl011::Pl011;+use super::{Board, IrqController, Kind, Uart};+use crate::arch::aarch64::dtb::Fdt;+use crate::arch::aarch64::gic::GicV2;+use alloc::format;+use alloc::string::String;+use alloc::vec::Vec;+use core::sync::atomic::{AtomicU32, AtomicU64, Ordering};++pub const UART_BASE: usize = 0x0900_0000;+pub const GICD_BASE: usize = 0x0800_0000;+pub const GICC_BASE: usize = 0x0801_0000;+/// The EL1 virtual timer: PPI 11 = INTID 27.+pub const TIMER_IRQ: u32 = 27;+pub const RAM_BASE: u64 = 0x4000_0000;++pub struct Virt {+    uart: Pl011,+    gic: GicV2,+    timer_irq: AtomicU32,+    ram_size: AtomicU64,+}++pub static VIRT: Virt = Virt {+    uart: Pl011::new(UART_BASE),+    gic: GicV2::new(GICD_BASE, GICC_BASE),+    timer_irq: AtomicU32::new(TIMER_IRQ),+    ram_size: AtomicU64::new(128 << 20),+};++/// First GiB devices, second GiB RAM (QEMU allows more RAM above, but the+/// bring-up only maps the first GiB of it).+static MAP: [(u64, Kind); 2] = [(0, Kind::Device), (0x4000_0000, Kind::Normal)];++impl Virt {+    pub fn ram_size(&self) -> u64 {+        self.ram_size.load(Ordering::Relaxed)+    }+}++impl Board for Virt {+    fn name(&self) -> &'static str {+        "QEMU virt"+    }+    fn uart(&self) -> &dyn Uart {+        &self.uart+    }+    fn irq(&self) -> &dyn IrqController {+        &self.gic+    }+    fn timer_irq(&self) -> u32 {+        self.timer_irq.load(Ordering::Relaxed)+    }+    fn memory_map(&self) -> &'static [(u64, Kind)] {+        &MAP+    }+    fn configure(&self, dtb: Option<&Fdt>) -> Vec<String> {+        let mut log = Vec::new();+        let Some(fdt) = dtb else {+            log.push(format!("no device tree; using virt's usual addresses (TODO: other boards need one)"));+            return log;+        };+        fdt.for_each_node(|n| {+            if n.is_compatible("arm,pl011") {+                if let Some((base, _)) = n.reg(0) {+                    if base as usize != self.uart.base() {+                        self.uart.set_base(base as usize);+                    }+                    log.push(format!("{}: PL011 UART at {:#x}", n.name(), base));+                }+            } else if n.is_compatible("arm,cortex-a15-gic") || n.is_compatible("arm,gic-400") {+                if let (Some((d, _)), Some((c, _))) = (n.reg(0), n.reg(1)) {+                    self.gic.set_bases(d as usize, c as usize);+                    log.push(format!("{}: GICv2, distributor at {:#x}, CPU interface at {:#x}", n.name(), d, c));+                }+            } else if n.is_compatible("arm,armv8-timer") {+                // 4 interrupts of 3 cells (type, number, flags): secure+                // physical, physical, *virtual*, hypervisor. PPI n = INTID 16 + n.+                if let Some(ppi) = n.interrupt_cell(2 * 3 + 1) {+                    self.timer_irq.store(16 + ppi, Ordering::Relaxed);+                    log.push(format!("{}: generic timer, virtual timer = PPI {} (INTID {})", n.name(), ppi, 16 + ppi));+                }+            } else if n.device_type == b"memory\0" {+                if let Some((base, size)) = n.reg(0) {+                    self.ram_size.store(size, Ordering::Relaxed);+                    log.push(format!("{}: RAM at {:#x}, {} MiB", n.name(), base, size >> 20));+                }+            }+        });+        log+    }+}

@@ -0,0 +1,67 @@+//! The entry point. QEMU loads our ELF file (see link.ld) and starts CPU 0+//! at `_start`, with the MMU off, at EL1 (or EL2 with `-M virt,virtualization=on`,+//! the way real boards and firmware usually hand over).+//!+//! What `_start` does, in assembly because there's no stack yet:+//!+//! 1. Any CPU but CPU 0 parks (QEMU's virt keeps the others off anyway;+//!    the Pi 3's firmware spins them, so this matters there).+//! 2. At EL2: make EL1 a 64-bit level (`HCR_EL2.RW`), let it use the+//!    timer (`CNTHCTL_EL2`), give it a safe `SCTLR_EL1`, then `eret` down+//!    to EL1 with interrupts masked.+//! 3. Set the stack pointer (`__stack_top` from link.ld), clear `.bss`+//!    (Rust expects zero-initialised statics), allow FP/SIMD+//!    (`CPACR_EL1.FPEN`; our code doesn't use it, but compiler_builtins+//!    could), and call `kmain(dtb, boot_el)`.++core::arch::global_asm!(+    r#"+.section .text.boot, "ax"+.global _start+_start:+    mrs     x1, mpidr_el1+    and     x1, x1, #0xff+    cbnz    x1, 9f                  // not CPU 0: park+    mov     x19, x0                 // a DTB pointer, if the loader passed one+    mrs     x20, CurrentEL+    lsr     x20, x20, #2            // x20 = the EL we started at+    cmp     x20, #2+    b.ne    1f+    // --- EL2 -> EL1 ---+    mov     x1, #(1 << 31)          // HCR_EL2.RW: EL1 is AArch64+    msr     hcr_el2, x1+    mrs     x1, cnthctl_el2+    orr     x1, x1, #3              // EL1PCTEN | EL1PCEN: EL1 may use the timers+    msr     cnthctl_el2, x1+    msr     cntvoff_el2, xzr        // virtual time = physical time+    movz    x1, #0x0800+    movk    x1, #0x30d0, lsl #16    // SCTLR_EL1: RES1 bits only (MMU, caches off)+    msr     sctlr_el1, x1+    mov     x1, #0x3c5              // SPSR: EL1h, D A I F masked+    msr     spsr_el2, x1+    adr     x1, 1f+    msr     elr_el2, x1+    eret+1:+    msr     spsel, #1               // use SP_EL1+    adrp    x1, __stack_top+    add     x1, x1, :lo12:__stack_top+    mov     sp, x1+    adrp    x1, __bss_start+    add     x1, x1, :lo12:__bss_start+    adrp    x2, __bss_end+    add     x2, x2, :lo12:__bss_end+2:  cmp     x1, x2+    b.hs    3f+    str     xzr, [x1], #8+    b       2b+3:  mov     x1, #(3 << 20)          // CPACR_EL1.FPEN = 0b11+    msr     cpacr_el1, x1+    isb+    mov     x0, x19+    mov     x1, x20+    bl      kmain+9:  wfe+    b       9b+"#+);

@@ -0,0 +1,69 @@+//! Text out through the board's UART. `puts_raw` is for the very first+//! line, before the MMU is on (it only does byte loads: with the MMU off+//! all memory is "device" memory, where an unaligned access faults, and+//! `core::fmt` makes no promises about that). `print!`/`println!` are for+//! after.++use core::fmt::{self, Write};+use spin::Mutex;++struct Console;++impl Write for Console {+    fn write_str(&mut self, s: &str) -> fmt::Result {+        let uart = super::board::current().uart();+        for b in s.bytes() {+            if b == b'\n' {+                uart.putc(b'\r');+            }+            uart.putc(b);+        }+        Ok(())+    }+}++static LOCK: Mutex<()> = Mutex::new(());++/// One line, no formatting, safe with the MMU off.+pub fn puts_raw(s: &str) {+    let uart = super::board::current().uart();+    let bytes = s.as_bytes();+    let mut i = 0;+    while i < bytes.len() {+        uart.putc(bytes[i]);+        i += 1;+    }+    uart.putc(b'\r');+    uart.putc(b'\n');+}++/// The `arch` interface's text out.+pub fn write_str(s: &str) {+    _print(format_args!("{}", s));+}++#[doc(hidden)]+pub fn _print(args: fmt::Arguments) {+    let _guard = LOCK.lock();+    let _ = Console.write_fmt(args);+}++/// Like `print!`, to the arm64 kernel's console.+#[macro_export]+macro_rules! print {+    ($($arg:tt)*) => { $crate::arch::aarch64::console::_print(format_args!($($arg)*)) };+}++/// Like `println!`, to the arm64 kernel's console.+#[macro_export]+macro_rules! println {+    () => { $crate::print!("\n") };+    ($($arg:tt)*) => { $crate::arch::aarch64::console::_print(format_args!("{}\n", format_args!($($arg)*))) };+}++/// For the fatal-exception path only: the interrupted code may have been+/// in the middle of a `println!`, holding the lock, and it will never run+/// again.+pub unsafe fn force_unlock() {+    unsafe { LOCK.force_unlock() };+}

@@ -0,0 +1,144 @@+//! A small reader for the **device tree** (a "DTB", flattened device tree).+//!+//! Arm boards have no PCI-style "ask the hardware what's there", so the+//! firmware (here QEMU) hands the kernel a description: a tree of nodes+//! like `pl011@9000000` with properties like `compatible = "arm,pl011"`+//! and `reg = <address size>`. The format is simple (devicetree.org spec,+//! chapter 5): a header, then a list of tokens (begin node, property, end+//! node), then a table of property names. Every number in it is big-endian.+//!+//! This reader only does what the bring-up needs: walk every node and hand+//! over its name, `compatible`, `reg`, `interrupts` and `device_type`. It+//! assumes `#address-cells = 2` and `#size-cells = 2` (64-bit addresses+//! and sizes), which is what QEMU's virt uses for every node we read.++const MAGIC: u32 = 0xd00d_feed;+const BEGIN_NODE: u32 = 1;+const END_NODE: u32 = 2;+const PROP: u32 = 3;+const NOP: u32 = 4;+#[allow(dead_code)]+const END: u32 = 9; // (the walk stops at anything it doesn't know, END included)+const MAX_DEPTH: usize = 16;++pub struct Fdt {+    data: &'static [u8],+    structs: usize,+    strings: usize,+}++/// One node, as `for_each_node` hands it over.+#[derive(Clone, Copy, Default)]+pub struct Node<'a> {+    pub name: &'a [u8],+    pub compatible: &'a [u8],+    pub reg: &'a [u8],+    pub interrupts: &'a [u8],+    pub device_type: &'a [u8],+    pub depth: usize,+}++fn be32(d: &[u8], at: usize) -> Option<u32> {+    let b = d.get(at..at + 4)?;+    Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))+}++fn cstr(d: &[u8], at: usize) -> &[u8] {+    let rest = d.get(at..).unwrap_or(&[]);+    let n = rest.iter().position(|&b| b == 0).unwrap_or(rest.len());+    &rest[..n]+}++impl Fdt {+    /// The device tree at physical address `addr`, if there is one there.+    ///+    /// # Safety+    /// `addr` must be readable memory (with the MMU on: mapped).+    pub unsafe fn at(addr: usize) -> Option<Fdt> {+        let head = unsafe { core::slice::from_raw_parts(addr as *const u8, 40) };+        if be32(head, 0)? != MAGIC {+            return None;+        }+        let size = be32(head, 4)? as usize;+        if !(40..=(2 << 20)).contains(&size) {+            return None;+        }+        let data = unsafe { core::slice::from_raw_parts(addr as *const u8, size) };+        Some(Fdt { data, structs: be32(data, 8)? as usize, strings: be32(data, 12)? as usize })+    }++    pub fn size(&self) -> usize {+        self.data.len()+    }++    /// Call `f` for every node (children before their parents finish).+    pub fn for_each_node(&self, mut f: impl FnMut(&Node)) {+        let d = self.data;+        let mut stack = [Node::default(); MAX_DEPTH];+        let mut depth = 0usize;+        let mut at = self.structs;+        while let Some(token) = be32(d, at) {+            at += 4;+            match token {+                BEGIN_NODE => {+                    let name = cstr(d, at);+                    at = (at + name.len() + 1 + 3) & !3;+                    if depth < MAX_DEPTH {+                        stack[depth] = Node { name, depth, ..Node::default() };+                    }+                    depth += 1;+                }+                END_NODE => {+                    if depth == 0 {+                        return;+                    }+                    depth -= 1;+                    if depth < MAX_DEPTH {+                        f(&stack[depth]);+                    }+                }+                PROP => {+                    let (Some(len), Some(nameoff)) = (be32(d, at), be32(d, at + 4)) else { return };+                    let value = d.get(at + 8..at + 8 + len as usize).unwrap_or(&[]);+                    at = (at + 8 + len as usize + 3) & !3;+                    if depth == 0 || depth > MAX_DEPTH {+                        continue;+                    }+                    let node = &mut stack[depth - 1];+                    match cstr(d, self.strings + nameoff as usize) {+                        b"compatible" => node.compatible = value,+                        b"reg" => node.reg = value,+                        b"interrupts" => node.interrupts = value,+                        b"device_type" => node.device_type = value,+                        _ => {}+                    }+                }+                NOP => {}+                _ => return, // END (9), or something we don't understand+            }+        }+    }+}++impl Node<'_> {+    /// Is one of the strings in `compatible` exactly `what`?+    pub fn is_compatible(&self, what: &str) -> bool {+        self.compatible.split(|&b| b == 0).any(|s| s == what.as_bytes())+    }++    /// The `i`th (address, size) pair in `reg` (2 + 2 cells).+    pub fn reg(&self, i: usize) -> Option<(u64, u64)> {+        let at = i * 16;+        let cell = |k: usize| be32(self.reg, at + 4 * k).map(|v| v as u64);+        Some(((cell(0)? << 32) | cell(1)?, (cell(2)? << 32) | cell(3)?))+    }++    /// The `i`th 32-bit cell of `interrupts`.+    pub fn interrupt_cell(&self, i: usize) -> Option<u32> {+        be32(self.interrupts, 4 * i)+    }++    pub fn name(&self) -> &str {+        core::str::from_utf8(self.name).unwrap_or("?")+    }+}

@@ -0,0 +1,185 @@+//! Exceptions and interrupts.+//!+//! On arm64 everything that interrupts normal flow is an "exception": a+//! fault, a system call, a hardware interrupt (IRQ). The CPU jumps into a+//! **vector table** whose address is in `VBAR_EL1`: 16 slots of 128 bytes,+//! 4 kinds (synchronous, IRQ, FIQ, SError) for each of 4 situations+//! (current level with SP_EL0, current level with SP_ELx, lower level in+//! 64-bit, lower level in 32-bit). We always run at EL1 on SP_EL1, so slots+//! 4-7 are the ones that matter for now.+//!+//! Each slot saves x0/x1 and jumps to common code that saves the rest+//! (plus ELR_EL1, where to return, and SPSR_EL1, the saved flags) and calls+//! `aarch64_exception(kind, frame)`.+//!+//! - **IRQ**: ask the interrupt controller which one, handle it (the+//!   timer), say we're done, return with `eret`.+//! - **anything synchronous** (a bad memory access, an undefined+//!   instruction, `brk`...): print what happened, using `ESR_EL1` (the+//!   "syndrome": what kind of exception, `EC` = bits 31:26) and `ELR_EL1`+//!   (the instruction that caused it), and stop: under QEMU through+//!   semihosting with exit status 3, so a script can tell.++use super::{board, console, timer};+use core::arch::asm;++core::arch::global_asm!(+    r#"+.macro VENTRY kind+.balign 0x80+    sub     sp, sp, #272+    stp     x0, x1, [sp, #0]+    mov     x0, #\kind+    b       __exc_common+.endm++.section .text.vectors, "ax"+.balign 0x800+.global __vectors+__vectors:+    VENTRY 0+    VENTRY 1+    VENTRY 2+    VENTRY 3+    VENTRY 4+    VENTRY 5+    VENTRY 6+    VENTRY 7+    VENTRY 8+    VENTRY 9+    VENTRY 10+    VENTRY 11+    VENTRY 12+    VENTRY 13+    VENTRY 14+    VENTRY 15++__exc_common:+    stp     x2, x3, [sp, #16]+    stp     x4, x5, [sp, #32]+    stp     x6, x7, [sp, #48]+    stp     x8, x9, [sp, #64]+    stp     x10, x11, [sp, #80]+    stp     x12, x13, [sp, #96]+    stp     x14, x15, [sp, #112]+    stp     x16, x17, [sp, #128]+    stp     x18, x19, [sp, #144]+    stp     x20, x21, [sp, #160]+    stp     x22, x23, [sp, #176]+    stp     x24, x25, [sp, #192]+    stp     x26, x27, [sp, #208]+    stp     x28, x29, [sp, #224]+    mrs     x1, elr_el1+    stp     x30, x1, [sp, #240]+    mrs     x1, spsr_el1+    str     x1, [sp, #256]+    mov     x1, sp+    bl      aarch64_exception+    ldr     x1, [sp, #256]+    msr     spsr_el1, x1+    ldp     x30, x1, [sp, #240]+    msr     elr_el1, x1+    ldp     x2, x3, [sp, #16]+    ldp     x4, x5, [sp, #32]+    ldp     x6, x7, [sp, #48]+    ldp     x8, x9, [sp, #64]+    ldp     x10, x11, [sp, #80]+    ldp     x12, x13, [sp, #96]+    ldp     x14, x15, [sp, #112]+    ldp     x16, x17, [sp, #128]+    ldp     x18, x19, [sp, #144]+    ldp     x20, x21, [sp, #160]+    ldp     x22, x23, [sp, #176]+    ldp     x24, x25, [sp, #192]+    ldp     x26, x27, [sp, #208]+    ldp     x28, x29, [sp, #224]+    ldp     x0, x1, [sp, #0]+    add     sp, sp, #272+    eret+"#+);++/// What the common code saved, in this order.+#[repr(C)]+pub struct Frame {+    pub x: [u64; 31],+    pub elr: u64,+    pub spsr: u64,+}++unsafe extern "C" {+    static __vectors: u8;+}++/// Point VBAR_EL1 at the table.+pub fn init() {+    unsafe {+        asm!("msr vbar_el1, {}", "isb", in(reg) &raw const __vectors as u64);+    }+}++/// Let IRQs in (clear the I bit in DAIF).+pub fn enable() {+    unsafe { asm!("msr daifclr, #2", options(nomem, nostack)) };+}++const KINDS: [&str; 4] = ["synchronous", "IRQ", "FIQ", "SError"];+const FROM: [&str; 4] = ["EL1 with SP_EL0", "EL1", "EL0 (64-bit)", "EL0 (32-bit)"];++/// What `ESR_EL1.EC` means, for the ones we're likely to see.+pub fn describe_ec(ec: u64) -> &'static str {+    match ec {+        0x00 => "unknown reason (an undefined instruction?)",+        0x01 => "WFI/WFE trapped",+        0x07 => "FP/SIMD access while disabled",+        0x0e => "illegal execution state",+        0x15 => "SVC (system call) from 64-bit code",+        0x18 => "MSR/MRS trapped",+        0x20 => "instruction abort from a lower level",+        0x21 => "instruction abort (bad address to run code from)",+        0x22 => "PC not aligned",+        0x24 => "data abort from a lower level",+        0x25 => "data abort (bad memory access)",+        0x26 => "SP not aligned",+        0x2f => "SError",+        0x3c => "BRK instruction",+        _ => "(see the Arm ARM, ESR_EL1)",+    }+}++#[unsafe(no_mangle)]+extern "C" fn aarch64_exception(kind: u64, frame: &mut Frame) {+    if kind == 5 {+        handle_irq();+        return;+    }+    let (esr, far): (u64, u64);+    unsafe {+        asm!("mrs {}, esr_el1", "mrs {}, far_el1", out(reg) esr, out(reg) far);+        console::force_unlock();+    }+    let ec = esr >> 26;+    crate::println!(+        "[exception] {} exception from {}: ESR_EL1={:#010x} (EC {:#04x}: {})",+        KINDS[(kind & 3) as usize],+        FROM[(kind >> 2) as usize & 3],+        esr,+        ec,+        describe_ec(ec)+    );+    crate::println!("[exception] ELR_EL1={:#x} FAR_EL1={:#x} SPSR_EL1={:#x}", frame.elr, far, frame.spsr);+    crate::println!("[exception] halting (QEMU exit status 3)");+    super::exit(3);+}++fn handle_irq() {+    let b = board::current();+    while let Some(irq) = b.irq().ack() {+        if irq == b.timer_irq() {+            timer::on_interrupt();+        } else {+            crate::println!("[irq] unexpected interrupt {}", irq);+        }+        b.irq().eoi(irq);+    }+}

@@ -0,0 +1,66 @@+//! The Arm GICv2 interrupt controller (QEMU virt's default). Two parts:+//!+//! - the **distributor** (GICD) decides which interrupts are on and where+//!   they go;+//! - the **CPU interface** (GICC) is per CPU: `IAR` says which interrupt+//!   fired (and marks it "being handled"), `EOIR` says we're done.+//!+//! Interrupt numbers ("INTIDs"): 0-15 are software interrupts between+//! CPUs, 16-31 are per-CPU ("PPIs", like each CPU's timer), 32 and up are+//! shared devices ("SPIs"). 1020-1023 mean "nothing" (spurious).++use super::board::IrqController;+use super::mmio;+use core::sync::atomic::{AtomicUsize, Ordering};++const GICD_CTLR: usize = 0x000;+const GICD_ISENABLER: usize = 0x100;+const GICD_ICENABLER: usize = 0x180;+const GICD_IPRIORITYR: usize = 0x400;+const GICC_CTLR: usize = 0x000;+const GICC_PMR: usize = 0x004;+const GICC_IAR: usize = 0x00c;+const GICC_EOIR: usize = 0x010;++pub struct GicV2 {+    dist: AtomicUsize,+    cpu: AtomicUsize,+}++impl GicV2 {+    pub const fn new(dist: usize, cpu: usize) -> Self {+        GicV2 { dist: AtomicUsize::new(dist), cpu: AtomicUsize::new(cpu) }+    }+    pub fn set_bases(&self, dist: usize, cpu: usize) {+        self.dist.store(dist, Ordering::Relaxed);+        self.cpu.store(cpu, Ordering::Relaxed);+    }+    pub fn bases(&self) -> (usize, usize) {+        (self.dist.load(Ordering::Relaxed), self.cpu.load(Ordering::Relaxed))+    }+}++impl IrqController for GicV2 {+    fn init(&self) {+        let (d, c) = self.bases();+        mmio::write32(d + GICD_CTLR, 0);+        for i in 0..32 {+            mmio::write32(d + GICD_ICENABLER + 4 * i, 0xffff_ffff); // everything off+        }+        mmio::write32(d + GICD_CTLR, 1); // distributor on+        mmio::write32(c + GICC_PMR, 0xf0); // let through every priority below 0xf0+        mmio::write32(c + GICC_CTLR, 1); // CPU interface on+    }+    fn enable(&self, irq: u32) {+        let d = self.dist.load(Ordering::Relaxed);+        mmio::write8(d + GICD_IPRIORITYR + irq as usize, 0xa0);+        mmio::write32(d + GICD_ISENABLER + 4 * (irq as usize / 32), 1 << (irq % 32));+    }+    fn ack(&self) -> Option<u32> {+        let irq = mmio::read32(self.cpu.load(Ordering::Relaxed) + GICC_IAR) & 0x3ff;+        if irq >= 1020 { None } else { Some(irq) }+    }+    fn eoi(&self, irq: u32) {+        mmio::write32(self.cpu.load(Ordering::Relaxed) + GICC_EOIR, irq);+    }+}

@@ -0,0 +1,39 @@+/* Stage 22: where the arm64 kernel lives in memory (QEMU virt).+ *+ * QEMU's virt has RAM from 0x4000_0000 and, for an ELF kernel, puts the+ * device tree at the very start of it. We start 2 MiB in, at 0x4020_0000,+ * and QEMU loads each part of the ELF file at these addresses (the MMU is+ * off, so they are physical addresses) and jumps to _start.+ */+ENTRY(_start)++SECTIONS+{+    . = 0x40200000;+    __kernel_start = .;++    .text : {+        KEEP(*(.text.boot))     /* _start first */+        *(.text .text.*)+    }+    .rodata : ALIGN(16) { *(.rodata .rodata.*) }+    .got : ALIGN(8) { *(.got .got.*) }+    .data : ALIGN(16) { *(.data .data.*) }++    .bss (NOLOAD) : ALIGN(16) {+        __bss_start = .;+        *(.bss .bss.*)+        *(COMMON)+        . = ALIGN(16);+        __bss_end = .;+    }++    /* 64 KiB of stack, growing down from __stack_top. */+    .stack (NOLOAD) : ALIGN(4096) {+        . += 0x10000;+        __stack_top = .;+    }+    __kernel_end = .;++    /DISCARD/ : { *(.comment) *(.eh_frame*) *(.note*) }+}

@@ -0,0 +1,152 @@+//! Stage 22: the arm64 kernel binary (`rustbelt-aarch64`).+//!+//! Built and run by scripts/run-aarch64.sh:+//!     cargo build --bin rustbelt-aarch64 --features aarch64 \+//!         --target aarch64-unknown-none-softfloat+//!     qemu-system-aarch64 -M virt -cpu cortex-a53 -semihosting \+//!         -nographic -kernel <that ELF file>+//!+//! `_start` (src/arch/aarch64/boot.rs) calls `kmain`, which brings the+//! machine up one piece at a time, printing as it goes, and then ends the+//! QEMU run with status 0. No processes, no scheduler, no shell: those+//! still only exist on the PC.+#![no_std]+#![no_main]++extern crate alloc;++use alloc::vec::Vec;+use core::panic::PanicInfo;+use core::sync::atomic::{AtomicU64, Ordering};+use linked_list_allocator::LockedHeap;+use rustbelt::arch::aarch64::{self as arch, board, console, dtb, exceptions, mmio, paging, timer};+use rustbelt::println;++/// The bring-up's heap: 1 MiB inside the kernel's .bss. (No frame+/// allocator on arm64 yet; this is enough for Vec and String.)+const HEAP_SIZE: usize = 1 << 20;+#[repr(C, align(4096))]+struct HeapSpace([u8; HEAP_SIZE]);+static mut HEAP_SPACE: HeapSpace = HeapSpace([0; HEAP_SIZE]);+#[global_allocator]+static HEAP: LockedHeap = LockedHeap::empty();++/// Where QEMU puts the device tree for an ELF kernel on virt.+const VIRT_DTB: usize = 0x4000_0000;+const TICK_HZ: u64 = 100;++/// A word to read back through the higher half.+static MARKER: AtomicU64 = AtomicU64::new(0);++#[unsafe(no_mangle)]+pub extern "C" fn kmain(dtb_arg: usize, boot_el: u64) -> ! {+    let b = board::current();+    console::puts_raw(arch::BANNER);+    exceptions::init();+    unsafe { paging::init(b) };+    // From here on: caches on, aligned or not, core::fmt is safe.+    println!("[boot] {} board, Cortex-A53; entered at EL{}, running at EL{}", b.name(), boot_el, arch::current_el());+    println!("[boot] exception vectors installed (VBAR_EL1)");++    // The higher half: the same RAM, seen at HIGH_BASE + physical.+    MARKER.store(0x5ca1_ab1e, Ordering::SeqCst);+    let through_high = unsafe { core::ptr::read_volatile(paging::high(&MARKER as *const _ as usize) as *const u64) };+    mmio::use_high_half(paging::HIGH_BASE);+    println!(+        "[mmu] on ({}): 4 KiB granule, identity map + higher half at {:#x}; a word at {:#x} reads {:#x} through {:#x}",+        if paging::enabled() { "SCTLR_EL1.M = 1" } else { "NOT ON?" },+        paging::HIGH_BASE,+        &MARKER as *const _ as usize,+        through_high,+        paging::high(&MARKER as *const _ as usize)+    );+    println!("[mmu] devices are now reached through the higher half (this line went to the UART at {:#x})", mmio::virt(0x0900_0000));++    // The heap.+    unsafe { HEAP.lock().init(&raw mut HEAP_SPACE as usize, HEAP_SIZE) };+    let numbers: Vec<u64> = (1..=1000).collect();+    println!("[heap] {} KiB at {:#x}; a Vec of 1..=1000 sums to {}", HEAP_SIZE >> 10, &raw const HEAP_SPACE as usize, numbers.iter().sum::<u64>());++    // The device tree.+    let dtb_addr = if dtb_arg != 0 { dtb_arg } else { VIRT_DTB };+    let fdt = unsafe { dtb::Fdt::at(dtb_addr) };+    match &fdt {+        Some(f) => println!("[dtb] device tree at {:#x}, {} bytes", dtb_addr, f.size()),+        None => println!("[dtb] no device tree at {:#x}", dtb_addr),+    }+    for line in b.configure(fdt.as_ref()) {+        println!("[dtb] {}", line);+    }+    if let Some(f) = &fdt {+        list_virtio(f);+    }++    #[cfg(feature = "aarch64-fault")]+    {+        // Nothing is mapped at 64 GiB: a translation fault, on purpose.+        println!("[fault] reading 0x10_0000_0000 on purpose (--features aarch64-fault)");+        let v = unsafe { core::ptr::read_volatile(0x10_0000_0000 as *const u64) };+        println!("[fault] read {:#x}?! the exception handler should have stopped us", v);+    }++    // Interrupts: the controller, the timer, then unmask.+    b.irq().init();+    b.irq().enable(b.timer_irq());+    timer::start(TICK_HZ);+    exceptions::enable();+    println!("[timer] generic timer at {} Hz, ticking {} times a second (INTID {})", timer::frequency(), TICK_HZ, b.timer_irq());+    let start = timer::counter();+    let mut shown = 0;+    while timer::ticks() < 100 {+        unsafe { core::arch::asm!("wfi") };+        let t = timer::ticks();+        if t > shown && shown < 5 {+            shown = t;+            println!("[timer] tick {}", t);+        }+    }+    let ms = (timer::counter() - start) * 1000 / timer::frequency();+    println!("[timer] 100 ticks took {} ms (should be about 1000)", ms);++    println!("[exit] bring-up done; leaving QEMU through semihosting SYS_EXIT, status 0");+    arch::exit(0)+}++/// QEMU virt has 32 virtio-mmio slots; most are empty (device ID 0).+fn list_virtio(f: &dtb::Fdt) {+    let mut slots = 0;+    let mut found = Vec::new();+    f.for_each_node(|n| {+        if n.is_compatible("virtio,mmio") {+            if let Some((base, _)) = n.reg(0) {+                slots += 1;+                let base = base as usize;+                let (magic, version, id) = (mmio::read32(base), mmio::read32(base + 4), mmio::read32(base + 8));+                if magic == 0x7472_6976 && id != 0 {+                    found.push((base, version, id));+                }+            }+        }+    });+    found.sort();+    println!("[virtio] {} virtio-mmio slots, {} in use", slots, found.len());+    for (base, version, id) in found {+        let name = match id {+            1 => "network card",+            2 => "block device",+            3 => "console",+            4 => "entropy source (rng)",+            16 => "GPU",+            18 => "input",+            _ => "other",+        };+        println!("[virtio] {:#x}: device ID {} ({}), virtio-mmio version {}", base, id, name, version);+    }+}++#[panic_handler]+fn panic(info: &PanicInfo) -> ! {+    unsafe { console::force_unlock() };+    println!("[panic] {}", info);+    arch::exit(2)+}

@@ -0,0 +1,30 @@+//! Device registers. Devices are reached at their physical address while+//! the MMU is off; once paging.rs has mapped them into the higher half too,+//! `use_high_half` makes every driver go through that mapping instead (the+//! identity mapping stays, so nothing breaks in between).++use core::sync::atomic::{AtomicU64, Ordering};++static OFFSET: AtomicU64 = AtomicU64::new(0);++/// From now on reach devices at `phys + offset` (paging::HIGH_BASE).+pub fn use_high_half(offset: u64) {+    OFFSET.store(offset, Ordering::SeqCst);+}++/// Where the CPU reaches physical address `phys` right now.+pub fn virt(phys: usize) -> usize {+    phys.wrapping_add(OFFSET.load(Ordering::Relaxed) as usize)+}++pub fn read32(phys: usize) -> u32 {+    unsafe { core::ptr::read_volatile(virt(phys) as *const u32) }+}++pub fn write32(phys: usize, value: u32) {+    unsafe { core::ptr::write_volatile(virt(phys) as *mut u32, value) }+}++pub fn write8(phys: usize, value: u8) {+    unsafe { core::ptr::write_volatile(virt(phys) as *mut u8, value) }+}

@@ -0,0 +1,67 @@+//! Stage 22: the arm64 (aarch64) side of the `arch` interface (see+//! src/arch/mod.rs). This is a *bring-up*: enough to boot on QEMU's `virt`+//! board with a Cortex-A53, print, take exceptions, turn on the MMU, and+//! get a timer interrupt through the GIC. No processes, no scheduler yet.+//!+//! | File | Part |+//! |------|------|+//! | `boot.rs` | entry: `_start`, EL2 -> EL1, stack, .bss, then `kmain` |+//! | `console.rs` | text out through the board's UART, `print!`/`println!` |+//! | `exceptions.rs` | the vector table, ESR decoding, IRQ dispatch |+//! | `gic.rs` | the GICv2 interrupt controller (virt) |+//! | `timer.rs` | the generic timer (CNTV), the tick count |+//! | `paging.rs` | MMU: identity map + the same memory again in the higher half |+//! | `dtb.rs` | a small device tree (DTB) reader |+//! | `mmio.rs` | device register access (through the higher half once the MMU is on) |+//! | `board/` | what differs between boards: `virt` (works), `raspi3` (a skeleton) |+//! | `main.rs` | the kernel binary `rustbelt-aarch64` (`kmain`) |+//! | `link.ld` | where the kernel goes in memory (0x4020_0000 on virt) |++pub mod board;+pub mod boot;+pub mod console;+pub mod dtb;+pub mod exceptions;+pub mod gic;+pub mod mmio;+pub mod paging;+pub mod timer;++/// The `arch` interface's `interrupts` (vector table + IRQ mask); the+/// controller itself is the board's (`board::current().irq()`).+pub use exceptions as interrupts;++use core::arch::asm;++/// The banner, the first line the arm64 kernel prints.+pub const BANNER: &str = concat!("Rustbelt ", env!("CARGO_PKG_VERSION"), " (stage 22: arm64 bring-up)");++/// The exception level we're running at now (1 = kernel, 2 = hypervisor).+pub fn current_el() -> u64 {+    let el: u64;+    unsafe { asm!("mrs {}, CurrentEL", out(reg) el) };+    (el >> 2) & 3+}++/// End the QEMU run with `code` as QEMU's exit status, through Arm+/// **semihosting** (QEMU must be started with `-semihosting`): `HLT #0xF000`+/// with operation 0x18 (`SYS_EXIT`) in w0 and, in x1, the address of two+/// words: the reason `ADP_Stopped_ApplicationExit` (0x20026) and the code.+///+/// (PSCI `SYSTEM_OFF` would also stop QEMU, but it can't carry a status.)+/// Without `-semihosting` the HLT is an undefined instruction; on a real+/// board there's no semihosting, so this is a QEMU-only way out.+pub fn exit(code: u32) -> ! {+    let block: [u64; 2] = [0x20026, code as u64];+    unsafe {+        asm!("hlt #0xf000", in("w0") 0x18u32, in("x1") block.as_ptr(), options(nostack));+    }+    halt()+}++/// Stop this CPU for good: mask interrupts and wait forever.+pub fn halt() -> ! {+    loop {+        unsafe { asm!("msr daifset, #0xf", "wfe", options(nomem, nostack)) };+    }+}

@@ -0,0 +1,108 @@+//! Turning on the MMU.+//!+//! Until now every address was a physical address. With the MMU on, the+//! CPU looks each address up in **page tables**. An arm64 CPU has two+//! table pointers: `TTBR0_EL1` for the low half of the address space+//! (addresses starting with 0x0000...) and `TTBR1_EL1` for the high half+//! (0xffff...). Linux keeps the kernel in the high half and programs in the+//! low half; we set up both:+//!+//! - **identity map** (TTBR0): virtual = physical, so the code keeps+//!   running at the address it was linked at when the MMU switches on;+//! - **higher half** (TTBR1): the same memory again at+//!   `HIGH_BASE + physical`. The drivers switch to it right away+//!   (mmio.rs); the kernel's code is still linked low (moving it up is a+//!   later stage).+//!+//! Settings (TCR_EL1): 4 KiB pages ("granule"), 39-bit addresses on both+//! sides (T0SZ = T1SZ = 25), so a walk starts at a level-1 table whose+//! 512 entries cover 1 GiB each. The bring-up only needs 1 GiB **block**+//! entries, no smaller tables: one per (first address, kind) in the board's+//! memory map. Memory types come from MAIR_EL1: index 0 = Device-nGnRnE+//! (registers: no caching, no reordering), index 1 = Normal write-back RAM.++use super::board::{Board, Kind};+use core::arch::asm;+use core::ptr::addr_of_mut;++/// Where the higher half starts with T1SZ = 25: the top 2^39 bytes.+pub const HIGH_BASE: u64 = 0xffff_ff80_0000_0000;++#[repr(C, align(4096))]+struct Table([u64; 512]);++static mut IDENTITY: Table = Table([0; 512]);+static mut HIGH: Table = Table([0; 512]);++const VALID_BLOCK: u64 = 0b01; // level-1 block descriptor+const ATTR_DEVICE: u64 = 0 << 2; // MAIR index 0+const ATTR_NORMAL: u64 = 1 << 2; // MAIR index 1+const INNER_SHAREABLE: u64 = 3 << 8;+const ACCESS_FLAG: u64 = 1 << 10; // "accessed" (else the first access faults)+const PXN: u64 = 1 << 53; // the kernel may not execute it+const UXN: u64 = 1 << 54; // user space may not execute it++fn block(phys: u64, kind: Kind) -> u64 {+    let attrs = match kind {+        Kind::Normal => ATTR_NORMAL | INNER_SHAREABLE | UXN,+        Kind::Device => ATTR_DEVICE | PXN | UXN,+    };+    (phys & !((1 << 30) - 1)) | attrs | ACCESS_FLAG | VALID_BLOCK+}++/// The higher-half address of physical address `phys`.+pub fn high(phys: usize) -> usize {+    HIGH_BASE as usize + phys+}++/// Fill both tables from the board's memory map and switch the MMU on+/// (with the data and instruction caches).+///+/// # Safety+/// Once, early, on CPU 0, with the MMU off.+pub unsafe fn init(board: &dyn Board) {+    let (id, hi) = unsafe { (&mut *addr_of_mut!(IDENTITY), &mut *addr_of_mut!(HIGH)) };+    for &(phys, kind) in board.memory_map() {+        let index = (phys >> 30) as usize;+        id.0[index] = block(phys, kind);+        hi.0[index] = block(phys, kind);+    }+    let mair: u64 = 0xff << 8; // index 1 = 0xff (Normal WB), index 0 = 0x00 (Device-nGnRnE)+    let tcr: u64 = 25            // T0SZ: 39-bit low half+        | (1 << 8) | (1 << 10)   // table walks: inner/outer write-back cached+        | (3 << 12)              // ... inner shareable; TG0 = 0b00 = 4 KiB+        | (25 << 16)             // T1SZ: 39-bit high half+        | (1 << 24) | (1 << 26) | (3 << 28)+        | (2 << 30)              // TG1 = 0b10 = 4 KiB+        | (2 << 32); // IPS: 40-bit physical addresses (what a Cortex-A53 has)+    unsafe {+        asm!(+            "msr mair_el1, {mair}",+            "msr tcr_el1, {tcr}",+            "msr ttbr0_el1, {t0}",+            "msr ttbr1_el1, {t1}",+            "dsb ish",+            "tlbi vmalle1",+            "dsb ish",+            "isb",+            "mrs {s}, sctlr_el1",+            "orr {s}, {s}, #1",          // M: MMU on+            "orr {s}, {s}, #(1 << 2)",   // C: data cache+            "orr {s}, {s}, #(1 << 12)",  // I: instruction cache+            "msr sctlr_el1, {s}",+            "isb",+            mair = in(reg) mair,+            tcr = in(reg) tcr,+            t0 = in(reg) id.0.as_ptr() as u64,+            t1 = in(reg) hi.0.as_ptr() as u64,+            s = out(reg) _,+        );+    }+}++/// Is the MMU on?+pub fn enabled() -> bool {+    let s: u64;+    unsafe { asm!("mrs {}, sctlr_el1", out(reg) s) };+    s & 1 != 0+}

@@ -0,0 +1,50 @@+//! The Arm **generic timer**: every arm64 CPU has a counter that ticks at a+//! fixed rate (`CNTFRQ_EL0` says how fast: 62.5 MHz on QEMU) and a few+//! timers that compare against it. We use the EL1 **virtual** timer+//! (`CNTV_*`): write "fire in N counts" to `CNTV_TVAL_EL0`, switch it on in+//! `CNTV_CTL_EL0`, and it raises its per-CPU interrupt (PPI 11 = INTID 27+//! on the GIC) when the time comes. Each tick re-arms it.++use core::arch::asm;+use core::sync::atomic::{AtomicU64, Ordering};++static TICKS: AtomicU64 = AtomicU64::new(0);+static INTERVAL: AtomicU64 = AtomicU64::new(0);++/// Counts per second.+pub fn frequency() -> u64 {+    let f: u64;+    unsafe { asm!("mrs {}, cntfrq_el0", out(reg) f) };+    f+}++/// The counter right now.+pub fn counter() -> u64 {+    let c: u64;+    unsafe { asm!("isb", "mrs {}, cntvct_el0", out(reg) c) };+    c+}++fn arm(counts: u64) {+    unsafe {+        asm!("msr cntv_tval_el0, {}", "msr cntv_ctl_el0, {}", in(reg) counts, in(reg) 1u64);+    }+}++/// Tick `hz` times a second (the interrupt must be enabled at the GIC).+pub fn start(hz: u64) {+    let interval = frequency() / hz;+    INTERVAL.store(interval, Ordering::Relaxed);+    arm(interval);+}++/// The timer's interrupt: count, and set the next one.+pub fn on_interrupt() {+    TICKS.fetch_add(1, Ordering::Relaxed);+    arm(INTERVAL.load(Ordering::Relaxed));+}++/// Ticks so far (the `arch` interface's `timer::ticks`).+pub fn ticks() -> u64 {+    TICKS.load(Ordering::Relaxed)+}

@@ -0,0 +1,30 @@+//! Stage 22: the code that depends on the CPU family ("architecture").+//!+//! Rustbelt grew up on the PC (x86_64), so almost everything in src/ still+//! assumes it. Stage 22 starts a second family, 64-bit ARM (aarch64, or+//! "arm64"), and puts the CPU-specific parts behind one small interface.+//! Every architecture module provides the same six things:+//!+//! | Part | What it is | x86_64 | aarch64 |+//! |------|------------|--------|---------|+//! | entry | where the CPU first jumps into our code | bootloader 0.9's `entry_point!` (src/main.rs) | `_start` in `aarch64/boot.rs` (EL2 -> EL1, stack, .bss) |+//! | `console` | text out, for the very first messages | COM1 (src/serial.rs) | the board's PL011 UART |+//! | `interrupts` | exception table + interrupt controller | IDT + 8259 PIC / APIC | vector table (VBAR_EL1) + GICv2 |+//! | `timer` | a regular tick | PIT / local APIC timer | the generic timer (virtual, CNTV) |+//! | `paging` | turning on virtual memory | src/memory.rs (the bootloader already did it) | `aarch64/paging.rs`: identity + higher half |+//! | `exit` | ending a QEMU run with a status | `isa-debug-exit` port 0xf4 | semihosting `SYS_EXIT` |+//!+//! `arch::current` is the module for the family being built. The PC kernel+//! doesn't go through this interface yet (src/main.rs is unchanged); the+//! x86_64 module just names the existing pieces so the two can be compared+//! and, later, shared code can call `arch::current::...` instead.++#[cfg(target_arch = "x86_64")]+pub mod x86_64;+#[cfg(target_arch = "x86_64")]+pub use self::x86_64 as current;++#[cfg(target_arch = "aarch64")]+pub mod aarch64;+#[cfg(target_arch = "aarch64")]+pub use self::aarch64 as current;

@@ -0,0 +1,43 @@+//! The PC's side of the `arch` interface (see src/arch/mod.rs). Stage 22+//! adds no new code for x86_64: these are thin names for the pieces that+//! already exist elsewhere in src/, which stay exactly where they were.++/// Text out: COM1, the serial port (src/serial.rs).+pub mod console {+    pub fn write_str(s: &str) {+        crate::serial::_print(format_args!("{}", s));+    }+}++/// GDT, IDT, the PICs, then `sti` (lib.rs's `init`).+pub mod interrupts {+    pub fn init() {+        crate::init();+    }+    pub fn enable() {+        ::x86_64::instructions::interrupts::enable();+    }+}++/// The PIT tick count (src/interrupts.rs).+pub mod timer {+    pub fn ticks() -> u64 {+        crate::interrupts::ticks()+    }+}++/// Paging: the bootloader turned it on; src/memory.rs takes over.+pub mod paging {+    pub use crate::memory::init;+}++/// End the QEMU run: 0 -> exit code 33, anything else -> 35 (isa-debug-exit).+pub fn exit(code: u32) -> ! {+    crate::exit_qemu(if code == 0 { crate::QemuExitCode::Success } else { crate::QemuExitCode::Failed });+    crate::hlt_loop()+}++/// Stop this CPU for good.+pub fn halt() -> ! {+    crate::hlt_loop()+}

@@ -11,7 +11,7 @@ #![no_std] #![cfg_attr(test, no_main)] // when testing the lib itself, it is a kernel too #![feature(custom_test_frameworks)] // nightly: our own test runner-#![feature(abi_x86_interrupt)] // nightly: `extern "x86-interrupt" fn`+#![cfg_attr(target_arch = "x86_64", feature(abi_x86_interrupt))] // nightly: `extern "x86-interrupt" fn` #![test_runner(crate::test_runner)] #![reexport_test_harness_main = "test_main"] // name the generated test entry fn @@ -19,48 +19,91 @@ // It works once we register a global allocator (see allocator.rs). extern crate alloc; +#[cfg(target_arch = "x86_64")] use core::panic::PanicInfo; +// Stage 22: the per-CPU-family code (src/arch/). Everything else below is+// still PC-only for now, so it is behind `cfg(target_arch = "x86_64")`;+// the arm64 kernel (src/arch/aarch64/main.rs) only uses `arch`.+pub mod arch;++#[cfg(target_arch = "x86_64")] pub mod acpi; // ACPI tables: just enough to list the CPUs (MADT)+#[cfg(target_arch = "x86_64")] pub mod allocator; // heap: maps heap pages + the #[global_allocator]+#[cfg(target_arch = "x86_64")] pub mod apic; // local APIC: per-CPU timer, inter-processor interrupts+#[cfg(target_arch = "x86_64")] pub mod boot; // stage 21: boot contract v2 -- pid 1 from the root partition+#[cfg(target_arch = "x86_64")] pub mod block; // block devices: BlockDevice trait, virtio-blk, partitions+#[cfg(target_arch = "x86_64")] pub mod dev; // /dev: null, random, hvc0, fb0, tty, ptmx, pts/N+#[cfg(target_arch = "x86_64")] pub mod elf; // ELF64 loader for user programs+#[cfg(target_arch = "x86_64")] pub mod errno; // Linux-compatible error numbers+#[cfg(target_arch = "x86_64")] pub mod fbcon; // text console drawn on the framebuffer+#[cfg(target_arch = "x86_64")] pub mod fpu; // FPU/SSE/AVX registers: per-thread save/restore (XSAVE or FXSAVE)+#[cfg(target_arch = "x86_64")] pub mod font; // 8x16 bitmap font (Spleen) for fbcon+#[cfg(target_arch = "x86_64")] pub mod futex; // futexes: user-space locks' sleep/wake (threads)+#[cfg(target_arch = "x86_64")] pub mod fs; // VFS: Inode/FileSystem/File traits, mount table, ramfs+#[cfg(target_arch = "x86_64")] pub mod gdt; // GDT (kernel + user segments) + TSS (RSP0, double-fault stack)+#[cfg(target_arch = "x86_64")] pub mod interrupts; // IDT, exception handlers, PIC, timer + keyboard+#[cfg(target_arch = "x86_64")] pub mod net; // networking: virtio-net driver + smoltcp TCP/IP, sockets+#[cfg(target_arch = "x86_64")] pub mod memory; // paging, frame allocator, page tables, shared-frame counts+#[cfg(target_arch = "x86_64")] pub mod mm; // a process's memory map: regions, demand paging, copy-on-write+#[cfg(target_arch = "x86_64")] pub mod pci; // PCI bus enumeration (config space via ports 0xCF8/0xCFC)+#[cfg(target_arch = "x86_64")] pub mod poll; // poll, select, epoll: waiting for many files at once+#[cfg(target_arch = "x86_64")] pub mod process; // processes: fork, execve, exit, wait4, process groups+#[cfg(target_arch = "x86_64")] pub mod serial; // COM1 serial port+#[cfg(target_arch = "x86_64")] pub mod signal; // signals: kill, handlers, signal frames, default actions+#[cfg(target_arch = "x86_64")] pub mod smp; // multi-core: start the other CPUs, per-CPU data+#[cfg(target_arch = "x86_64")] pub mod stack; // kernel stacks with guard pages+#[cfg(target_arch = "x86_64")] pub mod syscall; // syscall/sysret entry + system call implementations+#[cfg(target_arch = "x86_64")] pub mod syscall_ext; // stage 17's syscalls for coreutils/busybox: links, owners, statfs, ids, timers...+#[cfg(target_arch = "x86_64")] pub mod task; // cooperative multitasking: async tasks + executor+#[cfg(target_arch = "x86_64")] pub mod trap; // TrapFrame: saved user registers, the common return to ring 3+#[cfg(target_arch = "x86_64")] pub mod time; // nanosecond clock (TSC calibrated against the PIT)+#[cfg(target_arch = "x86_64")] pub mod timer; // alarm, setitimer, POSIX timers: signals at a deadline (stage 17)+#[cfg(target_arch = "x86_64")] pub mod thread; // threads + the multi-core scheduler (sleep/wake, wait queues)+#[cfg(target_arch = "x86_64")] pub mod tty; // terminals: line discipline, termios, console/hvc0/ptys+#[cfg(target_arch = "x86_64")] pub mod vdso; // stage 19: the vDSO + vvar page, time without a syscall+#[cfg(target_arch = "x86_64")] pub mod vga_buffer; // the console: VGA text mode, or fbcon on a GPU+#[cfg(target_arch = "x86_64")] pub mod virtio; // modern virtio transport + virtio-rng, -console, -gpu  /// Set up everything interrupt-related. Order matters: /// GDT/TSS first (the IDT refers to the IST stack), then the IDT, then the /// PICs, and only then enable interrupts (`sti`).+#[cfg(target_arch = "x86_64")] pub fn init() {     gdt::init();     fpu::init_cpu(); // before any CPU starts: smp.rs copies our CR0/CR4@@ -72,6 +115,7 @@ pub fn init() {  /// Paging + heap + the global frame allocator, in one call (what the test /// kernels need; src/main.rs does the same steps one by one with demos).+#[cfg(target_arch = "x86_64")] pub fn init_memory(boot_info: &'static bootloader::BootInfo) {     let phys_mem_offset = x86_64::VirtAddr::new(boot_info.physical_memory_offset);     let mut mapper = unsafe { memory::init(phys_mem_offset) };@@ -83,6 +127,7 @@ pub fn init_memory(boot_info: &'static bootloader::BootInfo) { /// Multi-core bring-up: the local APIC and its timer (preemption), CPU 0's /// idle thread, and every other CPU the ACPI tables list (see smp.rs). /// Needs `init` and `init_memory`. Returns the number of running CPUs.+#[cfg(target_arch = "x86_64")] pub fn init_smp() -> usize {     smp::init() }@@ -90,6 +135,7 @@ pub fn init_smp() -> usize { /// Storage bring-up: scan the PCI bus, start the disk drivers, read the /// partition tables and mount FAT volumes under /mnt. Needs the heap, the /// frame allocator and `fs::init()`. Returns (PCI devices, log lines).+#[cfg(target_arch = "x86_64")] pub fn init_storage() -> (alloc::vec::Vec<pci::PciDevice>, alloc::vec::Vec<alloc::string::String>) {     let devices = pci::scan();     let mut log = block::init(&devices);@@ -101,6 +147,7 @@ pub fn init_storage() -> (alloc::vec::Vec<pci::PciDevice>, alloc::vec::Vec<alloc /// virtio-gpu. With a GPU the text console moves onto its framebuffer. /// Call before `init_net` (smoltcp's random seed comes from the rng). /// Returns boot log lines.+#[cfg(target_arch = "x86_64")] pub fn init_drivers() -> alloc::vec::Vec<alloc::string::String> {     let mut log = virtio::init(&pci::scan());     if let Some(fb) = virtio::gpu::framebuffer() {@@ -113,11 +160,13 @@ pub fn init_drivers() -> alloc::vec::Vec<alloc::string::String> { /// Networking: loopback, plus the virtio-net card if there is one (smoltcp, /// DHCP and the netd thread). Needs the scheduler (call after `init_smp`). /// Returns the boot log lines.+#[cfg(target_arch = "x86_64")] pub fn init_net() -> alloc::vec::Vec<alloc::string::String> {     net::init(&pci::scan()) }  /// Print a line to *both* the VGA screen and the serial port.+#[cfg(target_arch = "x86_64")] #[macro_export] macro_rules! kprintln {     ($($arg:tt)*) => {{@@ -128,6 +177,7 @@ macro_rules! kprintln {  /// Sleep the CPU until the next interrupt, forever (cheaper than `loop {}`, /// which would keep a host CPU core at 100%).+#[cfg(target_arch = "x86_64")] pub fn hlt_loop() -> ! {     loop {         x86_64::instructions::hlt();@@ -136,6 +186,7 @@ pub fn hlt_loop() -> ! {  /// Exit codes for QEMU's `isa-debug-exit` device. QEMU exits with /// status `(value << 1) | 1`, so Success = 0x10 -> shell exit code 33.+#[cfg(target_arch = "x86_64")] #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u32)] pub enum QemuExitCode {@@ -144,6 +195,7 @@ pub enum QemuExitCode { }  /// Write to I/O port 0xf4 (where we put `isa-debug-exit`) to quit QEMU.+#[cfg(target_arch = "x86_64")] pub fn exit_qemu(exit_code: QemuExitCode) {     use x86_64::instructions::port::Port;     unsafe {@@ -158,10 +210,12 @@ pub fn exit_qemu(exit_code: QemuExitCode) {  /// Anything callable as a test: prints its name, runs it, prints [ok]. /// (If it panics, the panic handler prints [failed] instead.)+#[cfg(target_arch = "x86_64")] pub trait Testable {     fn run(&self); } +#[cfg(target_arch = "x86_64")] impl<T: Fn()> Testable for T {     fn run(&self) {         serial_print!("{}...\t", core::any::type_name::<T>());@@ -171,6 +225,7 @@ impl<T: Fn()> Testable for T { }  /// Called with every `#[test_case]` function in the crate being tested.+#[cfg(target_arch = "x86_64")] pub fn test_runner(tests: &[&dyn Testable]) {     serial_println!("Running {} tests", tests.len());     for test in tests {@@ -180,6 +235,7 @@ pub fn test_runner(tests: &[&dyn Testable]) { }  /// Panic handler for test kernels: report the failure and exit QEMU.+#[cfg(target_arch = "x86_64")] pub fn test_panic_handler(info: &PanicInfo) -> ! {     serial_println!("[failed]\n");     serial_println!("Error: {}\n", info);@@ -206,6 +262,7 @@ fn panic(info: &PanicInfo) -> ! {     test_panic_handler(info) } +#[cfg(target_arch = "x86_64")] #[test_case] fn trivial_assertion() {     assert_eq!(1 + 1, 2);

@@ -1439,7 +1439,7 @@ fn sys_uname(buf: u64) -> Result<u64, Errno> {         "Rustbelt",                     // sysname (`uname -s`)         host.as_str(),                  // nodename (hostname; sethostname changes it)         env!("CARGO_PKG_VERSION"),      // release (`uname -r`)-        "stage 21: root partition and init", // version+        "stage 22: arm64 bring-up",     // version (`uname -v`)         "x86_64",                       // machine         domain.as_str(),                // domainname     ];

@@ -176,7 +176,7 @@ fn scripted_shell_session() {         "piped text\n",         "wc -l\n2\n",         "saved\nmore\n",-        "Rustbelt 0.21.0",+        "Rustbelt 0.22.0",         "B:/ $ exit 3\n",     ] {         assert!(out.contains(expected), "missing {:?} in:\n{}", expected, out);

@@ -137,7 +137,7 @@ fn main() -> i64 {     c.check(n > 0 && cwd[0] == b'/', "getcwd gives an absolute path");     let mut ub = [0u8; 6 * 65];     let u = uname(&mut ub);-    c.check(u[0] == "Rustbelt" && u[3].starts_with("stage 21"), "uname: Rustbelt, stage 21");+    c.check(u[0] == "Rustbelt" && u[3].starts_with("stage 22"), "uname: Rustbelt, stage 22");      let (secs, _) = clock_gettime(0);     c.check(secs > 1_600_000_000, "CLOCK_REALTIME is a real date (after 2020)");