rust-kernel

//! The vDSO and its vvar page (stage 19).
//!
//! Programs ask for the time *a lot* (every log line, every timeout, every
//! frame of a game). A system call for each is wasteful: entering the
//! kernel and coming back costs far more than reading a clock. Linux's
//! answer, copied here, is the **vDSO**: a tiny shared library (source in
//! vdso/vdso.c, built by build.rs, embedded in the kernel below) that is
//! mapped into every process. Its `__vdso_clock_gettime` reads the CPU's
//! time stamp counter and turns it into nanoseconds itself.
//!
//! To do that it needs the same numbers the kernel uses (time.rs): the TSC
//! value at boot, the nanoseconds-per-tick multiplier and the wall-clock
//! time at boot. They live in the **vvar page**: one physical page, mapped
//! *read-only* into every process right in front of the vDSO's code. Only
//! the kernel writes it (here, in `publish`), under a seqlock: `seq` is odd
//! while a write is in progress, and readers retry if they saw an odd or
//! changed `seq`.
//!
//! How a program finds it: the auxiliary vector on its initial stack gets
//! `AT_SYSINFO_EHDR` = the address of the vDSO's ELF header (process.rs).
//! The C library walks that ELF image's dynamic symbol table to find
//! `__vdso_clock_gettime` (version "LINUX_2.6") and calls it.
//!
//! Layout in every address space (fixed addresses, just below the stack's
//! guard zone, reserved so mmap never hands them out):
//!
//!   VVAR_ADDR              the vvar page (read-only)
//!   VDSO_ADDR = +4 KiB     the vDSO image (read + execute)
//!
//! The pages are the same physical frames everywhere: allocated once at
//! boot (`init`), and mapped on first touch by the page fault handler
//! (mm.rs), which counts each mapping like any shared frame.
//!
//! When the fast path can't be used -- the TSC isn't calibrated yet, or it
//! isn't *invariant* (on some old CPUs it changes speed with the CPU's
//! clock; see `tsc_trusted` for virtual machines) -- `flags` says so and
//! the vDSO just makes the system call.

use crate::memory;
use crate::mm::{Kind, Mm, Vma, PAGE_SIZE, PROT_EXEC, PROT_READ, VVAR_ADDR};
use alloc::vec::Vec;
use core::sync::atomic::{compiler_fence, Ordering};
use x86_64::structures::paging::PhysFrame;

/// Where the vDSO's ELF header is mapped (AT_SYSINFO_EHDR).
pub const VDSO_ADDR: u64 = VVAR_ADDR + PAGE_SIZE;

/// The vDSO itself, compiled by build.rs from vdso/vdso.c.
pub static IMAGE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/vdso.so"));

/// Pages the image needs.
pub fn image_pages() -> u64 {
    (IMAGE.len() as u64).div_ceil(PAGE_SIZE)
}

/// The vvar page's contents. Must match `struct vvar` in vdso/vdso.c.
#[repr(C)]
struct VVar {
    seq: u32,
    flags: u32,
    tsc_start: u64,
    mult: u64,
    boot_realtime_ns: u64,
}

/// The vDSO may compute the time itself.
pub const VVAR_TSC_OK: u32 = 1;
/// `rdtscp` returns the CPU number (we put it in TSC_AUX, see `init_cpu`).
pub const VVAR_RDTSCP_CPU: u32 = 2;

struct Pages {
    vvar: PhysFrame,
    code: Vec<PhysFrame>,
}
static PAGES: spin::Once<Pages> = spin::Once::new();
/// Only one writer of the vvar page at a time.
static WRITER: spin::Mutex<()> = spin::Mutex::new(());

/// Allocate the vvar page and copy the vDSO into frames (once, at boot:
/// memory::init_frame_allocator calls this), then fill in the vvar page.
/// The frames are never freed: we keep our own reference to each, so
/// processes mapping and unmapping them never drop the count to zero.
pub fn init() {
    PAGES.call_once(|| {
        let vvar = memory::alloc_zeroed_frame().expect("no memory for the vvar page");
        let code = IMAGE
            .chunks(PAGE_SIZE as usize)
            .map(|chunk| {
                let frame = memory::alloc_zeroed_frame().expect("no memory for the vDSO");
                let dst = memory::phys_to_virt(frame.start_address()).as_mut_ptr::<u8>();
                unsafe { core::ptr::copy_nonoverlapping(chunk.as_ptr(), dst, chunk.len()) };
                frame
            })
            .collect();
        Pages { vvar, code }
    });
    publish();
}

/// The frame behind user address `page` in the vvar/vDSO area.
pub fn frame_at(page: u64) -> Option<PhysFrame> {
    let pages = PAGES.r#try()?;
    if page == VVAR_ADDR {
        return Some(pages.vvar);
    }
    let index = page.checked_sub(VDSO_ADDR)? / PAGE_SIZE;
    pages.code.get(index as usize).copied()
}

/// Add the two regions to a fresh address space (execve). Their pages
/// appear on first touch. Returns the AT_SYSINFO_EHDR value (0 = none).
pub fn map_into(mm: &mut Mm) -> u64 {
    if PAGES.r#try().is_none() {
        return 0;
    }
    mm.insert(Vma::new(VVAR_ADDR, VVAR_ADDR + PAGE_SIZE, PROT_READ, Kind::Vvar));
    mm.insert(Vma::new(VDSO_ADDR, VDSO_ADDR + image_pages() * PAGE_SIZE, PROT_READ | PROT_EXEC, Kind::Vdso));
    VDSO_ADDR
}

fn cpuid(leaf: u32) -> core::arch::x86_64::CpuidResult {
    core::arch::x86_64::__cpuid(leaf)
}

/// Does the TSC tick at a constant rate, whatever the CPU's speed and
/// power state? (CPUID 0x8000_0007, EDX bit 8: "invariant TSC".)
pub fn tsc_invariant() -> bool {
    cpuid(0x8000_0000).eax >= 0x8000_0007 && cpuid(0x8000_0007).edx & (1 << 8) != 0
}

/// Are we a virtual machine? (CPUID 1, ECX bit 31, "hypervisor present".)
pub fn under_hypervisor() -> bool {
    cpuid(1).ecx & (1 << 31) != 0
}

/// May user space trust the TSC? If it's invariant, yes. A virtual CPU's
/// TSC also ticks at one fixed rate (QEMU without KVM computes it from its
/// own clock), even though QEMU can't *advertise* "invariant TSC" there --
/// so under a hypervisor we trust it too. (The kernel itself uses the TSC
/// either way; this only decides whether the vDSO may.)
pub fn tsc_trusted() -> bool {
    tsc_invariant() || under_hypervisor()
}

/// Tests: turn the fast path off (true) to exercise the syscall fallback.
static FORCE_SYSCALL: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false);

pub fn force_syscall(on: bool) {
    FORCE_SYSCALL.store(on, Ordering::Relaxed);
    publish();
}

/// Does the CPU have `rdtscp`? (CPUID 0x8000_0001, EDX bit 27.)
pub fn has_rdtscp() -> bool {
    cpuid(0x8000_0000).eax >= 0x8000_0001 && cpuid(0x8000_0001).edx & (1 << 27) != 0
}

/// Per-CPU setup (every CPU, at start): put the CPU number into the
/// TSC_AUX register, which `rdtscp` returns -- the vDSO's getcpu().
pub fn init_cpu(cpu: usize) {
    if has_rdtscp() {
        unsafe { x86_64::registers::model_specific::Msr::new(0xC000_0103).write(cpu as u64) };
    }
}

/// The flags we'd publish right now (for tests and the boot log).
pub fn flags() -> u32 {
    if FORCE_SYSCALL.load(Ordering::Relaxed) {
        return 0;
    }
    let (_, mult, _) = crate::time::clock_params();
    let mut flags = 0;
    if mult != 0 && tsc_trusted() {
        flags |= VVAR_TSC_OK;
    }
    if has_rdtscp() {
        flags |= VVAR_RDTSCP_CPU;
    }
    flags
}

/// Copy the kernel's current clock numbers into the vvar page (after TSC
/// calibration, and at boot). The seqlock write: make `seq` odd, write the
/// fields, make `seq` even again. x86 never reorders stores with other
/// stores, so the compiler fences (which stop the *compiler* reordering)
/// are all we need for readers on other CPUs to see them in this order.
pub fn publish() {
    let Some(pages) = PAGES.r#try() else { return };
    let (tsc_start, mult, boot_realtime_ns) = crate::time::clock_params();
    let flags = flags();
    let vvar = memory::phys_to_virt(pages.vvar.start_address()).as_mut_ptr::<VVar>();
    x86_64::instructions::interrupts::without_interrupts(|| {
        let _writer = WRITER.lock();
        unsafe {
            let seq = core::ptr::addr_of_mut!((*vvar).seq);
            let start = seq.read_volatile();
            seq.write_volatile(start.wrapping_add(1) | 1); // odd: writing
            compiler_fence(Ordering::SeqCst);
            core::ptr::addr_of_mut!((*vvar).flags).write_volatile(flags);
            core::ptr::addr_of_mut!((*vvar).tsc_start).write_volatile(tsc_start);
            core::ptr::addr_of_mut!((*vvar).mult).write_volatile(mult);
            core::ptr::addr_of_mut!((*vvar).boot_realtime_ns).write_volatile(boot_realtime_ns);
            compiler_fence(Ordering::SeqCst);
            seq.write_volatile((start | 1).wrapping_add(1)); // even: done
        }
    });
}

/// The vvar page's sequence number (tests: it must be even when idle).
pub fn seq() -> u32 {
    let Some(pages) = PAGES.r#try() else { return 0 };
    let vvar = memory::phys_to_virt(pages.vvar.start_address()).as_ptr::<VVar>();
    unsafe { core::ptr::addr_of!((*vvar).seq).read_volatile() }
}