foundry

@@ -0,0 +1,457 @@+//! fputest -- every thread keeps its own FPU/SSE/AVX registers (stage 15.1).+//!+//! The kernel never uses the FPU registers itself, but user code does (any+//! C program's `double` maths, and libc's memcpy/strlen use SSE). So on a+//! context switch the kernel saves one thread's registers and loads the+//! next one's (XSAVE, or FXSAVE on older CPUs), a signal handler gets+//! clean registers and sigreturn puts the old ones back, fork copies them+//! and exec clears them. This program checks all of that:+//!+//! - threads + processes (spread over all CPUs) each load *distinct*+//!   values into xmm0-15 (all 256 bits of ymm0-15 if the CPU has AVX) and+//!   MXCSR, spin and yield for a while, then check nothing changed;+//! - a signal handler that scribbles over every register, synchronously+//!   (kill to ourselves) and asynchronously (another thread's tgkill while+//!   we spin): the interrupted code must see its own values afterwards;+//! - the handler starts with clean registers, finds the saved ones via+//!   uc_mcontext.fpstate (Linux's layout), and a value it changes there is+//!   what sigreturn loads;+//! - an unmasked SSE divide-by-zero raises SIGFPE (FPE_FLTDIV);+//! - fork copies the registers; exec starts with zeros and MXCSR 0x1f80.+//!+//! Our programs are built soft-float, so Rust code here never touches the+//! vector registers: only the `asm!` blocks below do. (That's also why the+//! asm can't name xmm registers as operands: it uses memory instead.)+#![no_std]+#![no_main]++#[macro_use]+#[path = "ulib.rs"]+mod ulib;++extern crate alloc;++use alloc::sync::Arc;+use alloc::vec::Vec;+use core::arch::asm;+use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};+use ulib::*;++/// The 16 vector registers, 32 bytes each (only the low 16 used without AVX).+#[repr(C, align(32))]+#[derive(Clone, Copy, PartialEq)]+struct Regs([u8; 512]);++static AVX: AtomicBool = AtomicBool::new(false);++/// Can we use AVX? The CPU must have it (CPUID.1:ECX bit 28) *and* the+/// kernel must have turned on XSAVE (OSXSAVE, bit 27) with the AVX state+/// in XCR0 (bits 1 and 2): otherwise AVX instructions fault (#UD).+fn detect_avx() -> bool {+    let ecx = core::arch::x86_64::__cpuid(1).ecx;+    if ecx & (1 << 27) == 0 || ecx & (1 << 28) == 0 {+        return false;+    }+    let (lo, _hi): (u32, u32);+    unsafe { asm!("xgetbv", in("ecx") 0, out("eax") lo, out("edx") _hi, options(nomem, nostack)) };+    lo & 6 == 6+}+fn avx() -> bool {+    AVX.load(Ordering::Relaxed)+}++/// Load all vector registers from `r`.+fn load(r: &Regs) {+    unsafe {+        if avx() {+            asm!(+            "vmovdqu ymm0, [{p} + 0]",+            "vmovdqu ymm1, [{p} + 32]",+            "vmovdqu ymm2, [{p} + 64]",+            "vmovdqu ymm3, [{p} + 96]",+            "vmovdqu ymm4, [{p} + 128]",+            "vmovdqu ymm5, [{p} + 160]",+            "vmovdqu ymm6, [{p} + 192]",+            "vmovdqu ymm7, [{p} + 224]",+            "vmovdqu ymm8, [{p} + 256]",+            "vmovdqu ymm9, [{p} + 288]",+            "vmovdqu ymm10, [{p} + 320]",+            "vmovdqu ymm11, [{p} + 352]",+            "vmovdqu ymm12, [{p} + 384]",+            "vmovdqu ymm13, [{p} + 416]",+            "vmovdqu ymm14, [{p} + 448]",+            "vmovdqu ymm15, [{p} + 480]",+                p = in(reg) r.0.as_ptr(), options(nostack, readonly, preserves_flags));+        } else {+            asm!(+            "movdqu xmm0, [{p} + 0]",+            "movdqu xmm1, [{p} + 32]",+            "movdqu xmm2, [{p} + 64]",+            "movdqu xmm3, [{p} + 96]",+            "movdqu xmm4, [{p} + 128]",+            "movdqu xmm5, [{p} + 160]",+            "movdqu xmm6, [{p} + 192]",+            "movdqu xmm7, [{p} + 224]",+            "movdqu xmm8, [{p} + 256]",+            "movdqu xmm9, [{p} + 288]",+            "movdqu xmm10, [{p} + 320]",+            "movdqu xmm11, [{p} + 352]",+            "movdqu xmm12, [{p} + 384]",+            "movdqu xmm13, [{p} + 416]",+            "movdqu xmm14, [{p} + 448]",+            "movdqu xmm15, [{p} + 480]",+                p = in(reg) r.0.as_ptr(), options(nostack, readonly, preserves_flags));+        }+    }+}++/// Store all vector registers (the bytes AVX doesn't have stay 0).+fn store() -> Regs {+    let mut r = Regs([0; 512]);+    unsafe {+        if avx() {+            asm!(+            "vmovdqu [{p} + 0], ymm0",+            "vmovdqu [{p} + 32], ymm1",+            "vmovdqu [{p} + 64], ymm2",+            "vmovdqu [{p} + 96], ymm3",+            "vmovdqu [{p} + 128], ymm4",+            "vmovdqu [{p} + 160], ymm5",+            "vmovdqu [{p} + 192], ymm6",+            "vmovdqu [{p} + 224], ymm7",+            "vmovdqu [{p} + 256], ymm8",+            "vmovdqu [{p} + 288], ymm9",+            "vmovdqu [{p} + 320], ymm10",+            "vmovdqu [{p} + 352], ymm11",+            "vmovdqu [{p} + 384], ymm12",+            "vmovdqu [{p} + 416], ymm13",+            "vmovdqu [{p} + 448], ymm14",+            "vmovdqu [{p} + 480], ymm15",+                p = in(reg) r.0.as_mut_ptr(), options(nostack, preserves_flags));+        } else {+            asm!(+            "movdqu [{p} + 0], xmm0",+            "movdqu [{p} + 32], xmm1",+            "movdqu [{p} + 64], xmm2",+            "movdqu [{p} + 96], xmm3",+            "movdqu [{p} + 128], xmm4",+            "movdqu [{p} + 160], xmm5",+            "movdqu [{p} + 192], xmm6",+            "movdqu [{p} + 224], xmm7",+            "movdqu [{p} + 256], xmm8",+            "movdqu [{p} + 288], xmm9",+            "movdqu [{p} + 320], xmm10",+            "movdqu [{p} + 352], xmm11",+            "movdqu [{p} + 384], xmm12",+            "movdqu [{p} + 416], xmm13",+            "movdqu [{p} + 448], xmm14",+            "movdqu [{p} + 480], xmm15",+                p = in(reg) r.0.as_mut_ptr(), options(nostack, preserves_flags));+        }+    }+    r+}++fn get_mxcsr() -> u32 {+    let mut m = 0u32;+    unsafe { asm!("stmxcsr [{}]", in(reg) &mut m, options(nostack, preserves_flags)) };+    m+}+fn set_mxcsr(m: u32) {+    unsafe { asm!("ldmxcsr [{}]", in(reg) &m, options(nostack, readonly, preserves_flags)) };+}++/// A register image no other `seed` produces (only the parts this CPU has).+fn pattern(seed: u64) -> Regs {+    let mut r = Regs([0; 512]);+    let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1;+    for (i, chunk) in r.0.chunks_mut(8).enumerate() {+        x ^= x << 13;+        x ^= x >> 7;+        x ^= x << 17;+        if avx() || i % 4 < 2 {+            chunk.copy_from_slice(&x.to_le_bytes());+        }+    }+    r+}+/// A distinct MXCSR per seed: all exceptions masked, rounding mode (bits+/// 13-14) and flush-to-zero / denormals-are-zero (15, 6) from the seed.+fn mxcsr_for(seed: u64) -> u32 {+    0x1f80 | ((seed as u32 & 3) << 13) | ((seed as u32 >> 2 & 1) << 15) | ((seed as u32 >> 3 & 1) << 6)+}++/// Load a pattern, spin and yield across many timer ticks (so the thread+/// is switched out and probably migrated), checking it now and then.+/// Returns how many checks found a wrong register.+fn hammer(seed: u64, rounds: u32) -> u32 {+    let want = pattern(seed);+    let mx = mxcsr_for(seed);+    load(&want);+    set_mxcsr(mx);+    let mut bad = 0;+    for round in 0..rounds {+        let mut x = round as u64;+        for _ in 0..20_000 {+            x = core::hint::black_box(x.wrapping_mul(6364136223846793005).wrapping_add(1));+        }+        if round % 3 == 0 {+            sched_yield();+        }+        if store() != want || get_mxcsr() != mx {+            bad += 1;+            load(&want); // keep going with good values+            set_mxcsr(mx);+        }+    }+    set_mxcsr(0x1f80);+    bad+}++// --- signal handlers -----------------------------------------------------------++/// Offsets in the ucontext the kernel passes (same as Linux's): the+/// `fpstate` pointer in uc_mcontext, and inside the FXSAVE area: MXCSR,+/// xmm0, the software-reserved bytes (sw_bytes) and, in the XSAVE area,+/// the upper halves of ymm0-15.+const UC_FLAGS: usize = 0;+const UC_FPSTATE: usize = 40 + 23 * 8;+const FX_MXCSR: usize = 24;+const FX_XMM: usize = 160;+const FX_SW: usize = 464;+const XS_YMM_HI: usize = 576;+const MAGIC1: u32 = 0x4650_5853;+const MAGIC2: u32 = 0x4650_5845;++static SYNC_SEED: AtomicU64 = AtomicU64::new(0);+/// Bit flags the handler sets for main to check.+static H_CLEAN: AtomicU32 = AtomicU32::new(0); // registers zero + MXCSR default on entry+static H_SAVED: AtomicU32 = AtomicU32::new(0); // the frame held the interrupted values+static H_LAYOUT: AtomicU32 = AtomicU32::new(0); // magic numbers / flags right+static ASYNC_HITS: AtomicU32 = AtomicU32::new(0);+const NEW_XMM15: u64 = 0x1515_1515_cafe_f00d;++fn rd32(p: *const u8) -> u32 {+    unsafe { core::ptr::read_unaligned(p as *const u32) }+}+fn rd64(p: *const u8) -> u64 {+    unsafe { core::ptr::read_unaligned(p as *const u64) }+}++/// Scribble over every vector register and MXCSR (a handler that does+/// float maths, or calls memcpy, would).+fn clobber() {+    load(&pattern(0xdead));+    set_mxcsr(0x1f80 | (3 << 13));+}++extern "C" fn on_usr1(_sig: i32, _info: *const SigInfo, uc: *mut u8) {+    let clean = store() == Regs([0; 512]) && get_mxcsr() == 0x1f80;+    H_CLEAN.store(clean as u32, Ordering::SeqCst);+    let want = pattern(SYNC_SEED.load(Ordering::SeqCst));+    let fp = rd64(unsafe { uc.add(UC_FPSTATE) }) as *mut u8;+    if fp.is_null() || fp as u64 % 64 != 0 {+        H_SAVED.store(0, Ordering::SeqCst);+    } else {+        // xmm i is at FX_XMM + 16*i; our image has register i at 32*i.+        let mut ok = rd32(unsafe { fp.add(FX_MXCSR) }) == mxcsr_for(SYNC_SEED.load(Ordering::SeqCst));+        for i in 0..16 {+            for b in 0..16 {+                ok &= unsafe { *fp.add(FX_XMM + 16 * i + b) } == want.0[32 * i + b];+            }+        }+        let flags = rd64(unsafe { uc.add(UC_FLAGS) });+        let magic1 = rd32(unsafe { fp.add(FX_SW) });+        let layout = if magic1 == MAGIC1 {+            // XSAVE frame: sw_bytes say how big it is; magic2 at the end.+            let size = rd32(unsafe { fp.add(FX_SW + 16) }) as usize;+            let xfeatures = rd64(unsafe { fp.add(FX_SW + 8) });+            if avx() {+                for i in 0..16 {+                    for b in 0..16 {+                        ok &= unsafe { *fp.add(XS_YMM_HI + 16 * i + b) } == want.0[32 * i + 16 + b];+                    }+                }+            }+            flags & 1 != 0 && rd32(unsafe { fp.add(size) }) == MAGIC2 && xfeatures & 3 == 3 && (!avx() || xfeatures & 4 != 0)+        } else {+            // FXSAVE-only frame (no XSAVE): fine as long as we're not using AVX.+            magic1 == 0 && !avx()+        };+        H_SAVED.store(ok as u32, Ordering::SeqCst);+        H_LAYOUT.store(layout as u32, Ordering::SeqCst);+        // Change the saved xmm15: sigreturn must load what's in the frame.+        unsafe { core::ptr::write_unaligned(fp.add(FX_XMM + 16 * 15) as *mut u64, NEW_XMM15) };+    }+    clobber();+}++extern "C" fn on_usr2(_sig: i32) {+    clobber();+    ASYNC_HITS.fetch_add(1, Ordering::SeqCst);+}++static FPE_CODE: AtomicU32 = AtomicU32::new(0);+/// SIGFPE from an unmasked SSE exception. Returning would just run the+/// faulting instruction again, so mask all exceptions (and clear the+/// flags) in the *saved* MXCSR first: the retry then gives inf quietly.+extern "C" fn on_fpe(_sig: i32, info: *const SigInfo, uc: *mut u8) {+    FPE_CODE.store(unsafe { (*info).code } as u32, Ordering::SeqCst);+    let fp = rd64(unsafe { uc.add(UC_FPSTATE) }) as *mut u8;+    if !fp.is_null() {+        unsafe { core::ptr::write_unaligned(fp.add(FX_MXCSR) as *mut u32, 0x1f80) };+    }+}++// --- the tests ---------------------------------------------------------------------++/// N threads in this process, each with its own seed; returns bad checks.+fn thread_round(base: u64, n: u64, rounds: u32) -> u32 {+    let bad = Arc::new(AtomicU32::new(0));+    let mut ts = Vec::new();+    for i in 0..n {+        let bad = bad.clone();+        ts.push(spawn_thread(0, move || {+            bad.fetch_add(hammer(base + i, rounds), Ordering::SeqCst);+        }));+    }+    let main_bad = hammer(base + 99, rounds);+    for t in ts {+        match t {+            Ok(t) => t.join(),+            Err(_) => return 1000,+        }+    }+    bad.load(Ordering::SeqCst) + main_bad+}++fn exec_check() -> i64 {+    // Run straight after exec: nothing has touched the vector registers.+    let clean = store() == Regs([0; 512]) && get_mxcsr() == 0x1f80;+    if clean { 0 } else { 1 }+}++fn main() -> i64 {+    AVX.store(detect_avx(), Ordering::Relaxed);+    if arg(1) == Some("exec-check") {+        return exec_check();+    }+    let mut c = Checks::new("fputest");+    println!(+        "fputest: {} registers, {} CPU(s)",+        if avx() { "ymm0-15 (AVX)" } else { "xmm0-15 (SSE)" },+        sysinfo().cpus+    );++    // 1. The registers start clean (main ran just after exec).+    c.check(exec_check() == 0, "a new program starts with zeroed registers and MXCSR 0x1f80");++    // 2. Threads and processes, all with distinct values, on all CPUs.+    let mut children = Vec::new();+    for p in 1..=2u64 {+        let pid = fork();+        if pid == 0 {+            exit(thread_round(p * 1000, 3, 60).min(100) as i64);+        }+        children.push(pid);+    }+    let bad = thread_round(0, 3, 60);+    let mut child_bad = 0;+    for pid in children {+        child_bad += wait(pid);+    }+    c.check(bad == 0, "4 threads kept their own vector registers + MXCSR");+    c.check(child_bad == 0, "so did 4 threads in each of 2 forked processes (12 at once)");++    // 3. fork copies the registers; exec clears them.+    let want = pattern(7);+    load(&want);+    set_mxcsr(mxcsr_for(7));+    let pid = fork();+    if pid == 0 {+        let same = store() == want && get_mxcsr() == mxcsr_for(7);+        if !same {+            exit(2);+        }+        execve("/mnt/disk0/bin/fputest", &["fputest", "exec-check"], &[]);+        exit(3);+    }+    let parent_same = store() == want;+    set_mxcsr(0x1f80);+    let code = wait(pid);+    c.check(parent_same && code != 2, "fork: the child starts with the parent's registers");+    c.check(code == 0, "exec: the new program starts with clean registers");++    // 4. A synchronous signal whose handler clobbers everything.+    sigaction_info(SIGUSR1, on_usr1, 0);+    SYNC_SEED.store(42, Ordering::SeqCst);+    let mut want = pattern(42);+    load(&want);+    set_mxcsr(mxcsr_for(42));+    kill(getpid(), SIGUSR1); // the handler runs on the way back from this+    let got = store();+    let mx = get_mxcsr();+    set_mxcsr(0x1f80);+    c.check(H_CLEAN.load(Ordering::SeqCst) == 1, "the handler starts with clean registers");+    c.check(H_SAVED.load(Ordering::SeqCst) == 1, "uc_mcontext.fpstate holds the interrupted registers");+    c.check(H_LAYOUT.load(Ordering::SeqCst) == 1, "the fpstate has Linux's layout (sw_bytes, magics, uc_flags)");+    want.0[32 * 15..32 * 15 + 8].copy_from_slice(&NEW_XMM15.to_le_bytes());+    c.check(got == want && mx == mxcsr_for(42), "after sigreturn: our registers back, with the handler's xmm15 edit");++    // 5. Asynchronous signals: another thread interrupts us while we spin.+    signal(SIGUSR2, on_usr2);+    let me = gettid();+    let sender = spawn_thread(0, move || {+        for _ in 0..40 {+            tgkill(getpid(), me, SIGUSR2);+            sleep_ms(2);+        }+    });+    let want = pattern(4242);+    load(&want);+    set_mxcsr(mxcsr_for(4242));+    let t0 = now_us();+    let mut bad = 0;+    while ASYNC_HITS.load(Ordering::SeqCst) < 40 && now_us() - t0 < 5_000_000 {+        if store() != want || get_mxcsr() != mxcsr_for(4242) {+            bad += 1;+            load(&want);+            set_mxcsr(mxcsr_for(4242));+        }+    }+    set_mxcsr(0x1f80);+    if let Ok(t) = sender {+        t.join();+    }+    let hits = ASYNC_HITS.load(Ordering::SeqCst);+    println!("fputest: {} async signals, each handler clobbered every register", hits);+    c.check(hits == 40 && bad == 0, "40 async signals with clobbering handlers changed nothing we saw");++    // 6. An unmasked SSE exception (divide by zero) -> SIGFPE, FPE_FLTDIV.+    sigaction_info(SIGFPE, on_fpe, 0);+    let one = 1.0f64.to_bits();+    let zero = 0u64;+    let mut out = 0u64;+    set_mxcsr(0x1f80 & !(1 << 9)); // unmask ZE (bit 9)+    unsafe {+        asm!(+            "movq xmm0, [{a}]",+            "divsd xmm0, [{b}]", // 1.0 / 0.0+            "movq [{o}], xmm0",+            a = in(reg) &one, b = in(reg) &zero, o = in(reg) &mut out,+            options(nostack)+        );+    }+    let mx = get_mxcsr();+    set_mxcsr(0x1f80);+    let code = FPE_CODE.load(Ordering::SeqCst);+    if code == 0 && out == f64::INFINITY.to_bits() && mx & 4 != 0 {+        // QEMU's software CPU (TCG) sets the flag but never raises #XM.+        println!("fputest: skipped SIGFPE check: this CPU doesn't raise SIMD exceptions (QEMU TCG)");+    } else {+        c.check(code == 3, "unmasked 1.0/0.0 -> SIGFPE with si_code FPE_FLTDIV");+        c.check(out == f64::INFINITY.to_bits() && mx & (1 << 9) != 0, "the handler's MXCSR edit took effect (retry gave inf)");+    }++    c.done()+}