//! A microsecond clock for the network stack. //! //! The PIT ticks only ~18.2 times a second (55 ms), far too coarse to time //! a ping (QEMU answers in well under a millisecond) or to run TCP's //! retransmit timers. Every x86-64 CPU has a *time stamp counter* (TSC): a //! 64-bit counter that goes up at a fixed rate, read with `rdtsc`. We don't //! know that rate, so we measure it once against the PIT (apic.rs does this //! while it calibrates the APIC timer, over 2 PIT ticks) and from then on //! `now_us` is just `rdtsc / rate`. //! //! Before calibration (or if it never happens, e.g. in some tests) we fall //! back to counting PIT ticks. //! //! Stage 19: the clock counts nanoseconds now (`now_ns`), and the numbers //! behind it are shared with user space through the vDSO's vvar page (see //! vdso.rs), so programs can read the time without a system call. use core::sync::atomic::{AtomicU64, Ordering}; /// One PIT tick in microseconds (1 / 18.2065 Hz). pub const PIT_TICK_US: u64 = 54_925; static TSC_PER_MS: AtomicU64 = AtomicU64::new(0); static TSC_START: AtomicU64 = AtomicU64::new(0); /// Stage 19: nanoseconds per TSC tick, as a fixed-point number with 32 /// bits after the point (so 2^32 means "1 ns per tick"). Multiplying and /// shifting is much cheaper than dividing, and the vDSO (vdso/vdso.c) does /// the very same arithmetic in user space -- that's why the formula lives /// in one place, `tsc_to_ns`, and both sides use it. 0 = not calibrated. static TSC_MULT: AtomicU64 = AtomicU64::new(0); fn rdtsc() -> u64 { unsafe { core::arch::x86_64::_rdtsc() } } /// Called with a TSC reading at a PIT tick edge (`start`) and the number of /// ticks it took to reach `end`. pub fn calibrate(start: u64, end: u64, pit_ticks: u64) { let per_ms = ((end - start) * 1000 / (pit_ticks * PIT_TICK_US)).max(1); // Keep the clock monotonic: the TSC clock starts at the PIT uptime. let uptime_us = crate::interrupts::ticks() * PIT_TICK_US; TSC_START.store(end.saturating_sub(uptime_us / 1000 * per_ms), Ordering::Relaxed); TSC_PER_MS.store(per_ms, Ordering::Relaxed); TSC_MULT.store(((1_000_000u128 << 32) / per_ms as u128) as u64, Ordering::Relaxed); crate::vdso::publish(); // tell user space's clock too } /// TSC ticks since `TSC_START` -> nanoseconds. Must match `read_clock` in /// vdso/vdso.c exactly, or the two clocks would disagree. pub fn tsc_to_ns(delta: u64, mult: u64) -> u64 { ((delta as u128 * mult as u128) >> 32) as u64 } /// Nanoseconds since boot (CLOCK_MONOTONIC). pub fn now_ns() -> u64 { let mult = TSC_MULT.load(Ordering::Relaxed); if mult == 0 { return crate::interrupts::ticks() * PIT_TICK_US * 1000; } tsc_to_ns(rdtsc().saturating_sub(TSC_START.load(Ordering::Relaxed)), mult) } /// Microseconds since boot. pub fn now_us() -> u64 { now_ns() / 1000 } /// What the vDSO needs to compute the time by itself: (TSC at time 0, /// ns-per-tick multiplier (0 = not calibrated), Unix time at boot in ns). pub fn clock_params() -> (u64, u64, u64) { let boot = boot_realtime_ns(); (TSC_START.load(Ordering::Relaxed), TSC_MULT.load(Ordering::Relaxed), boot) } /// The PIT tick by which `us` microseconds from now will surely have /// passed. For sleeping/timeouts, which may run late but never early: we /// may be anywhere inside the current ~55 ms tick, so one extra tick. pub fn deadline_after_us(us: u64) -> u64 { crate::interrupts::ticks() + us.div_ceil(PIT_TICK_US) + 1 } // --------------------------------------------------------------------------- // Wall-clock time (CLOCK_REALTIME) // --------------------------------------------------------------------------- /// Unix time (nanoseconds since 1970-01-01 UTC) at boot (now_ns = 0), /// read from the CMOS clock on first use. 0 = not read yet. static BOOT_REALTIME_NS: AtomicU64 = AtomicU64::new(0); /// Read a CMOS register (the PC's battery-backed real-time clock). fn cmos(reg: u8) -> u8 { use x86_64::instructions::port::Port; unsafe { Port::::new(0x70).write(reg); Port::::new(0x71).read() } } /// The RTC's date and time as Unix seconds. QEMU's RTC runs on UTC. fn rtc_unix_seconds() -> u64 { x86_64::instructions::interrupts::without_interrupts(|| { // Don't read in the middle of an update (bit 7 of status A). while cmos(0x0A) & 0x80 != 0 { core::hint::spin_loop(); } let [sec, min, hour, day, month, year] = [0x00, 0x02, 0x04, 0x07, 0x08, 0x09].map(cmos); let status_b = cmos(0x0B); // Usually the values are BCD (0x59 = 59) unless bit 2 says binary. let bin = |v: u8| if status_b & 0x04 != 0 { v as u64 } else { ((v >> 4) * 10 + (v & 0xf)) as u64 }; let mut h = bin(hour & 0x7f); if status_b & 0x02 == 0 && hour & 0x80 != 0 { h = (h + 12) % 24; // 12-hour clock, PM } let (y, m, d) = (2000 + bin(year), bin(month), bin(day)); days_from_civil(y, m, d) * 86400 + h * 3600 + bin(min) * 60 + bin(sec) }) } /// Days since 1970-01-01 for a date (Howard Hinnant's algorithm). fn days_from_civil(y: u64, m: u64, d: u64) -> u64 { let y = if m <= 2 { y - 1 } else { y }; let era = y / 400; let yoe = y - era * 400; let mp = (m + 9) % 12; let doy = (153 * mp + 2) / 5 + d - 1; let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; era * 146097 + doe - 719468 } fn boot_realtime_ns() -> u64 { let mut boot = BOOT_REALTIME_NS.load(Ordering::Relaxed); if boot == 0 { boot = (rtc_unix_seconds() * 1_000_000_000).saturating_sub(now_ns()).max(1); BOOT_REALTIME_NS.store(boot, Ordering::Relaxed); } boot } /// Nanoseconds since 1970-01-01 UTC (clock_gettime(CLOCK_REALTIME)). pub fn realtime_ns() -> u64 { boot_realtime_ns() + now_ns() } /// Microseconds since 1970-01-01 UTC. pub fn realtime_us() -> u64 { realtime_ns() / 1000 }