//! The local APIC: every CPU's own interrupt controller. //! //! Each x86 CPU core has a "local APIC" (Advanced Programmable Interrupt //! Controller). We use three of its features: //! //! * **Inter-processor interrupts (IPIs).** One CPU can interrupt another //! by writing the target's APIC ID and a vector into its ICR register. //! That's how CPU 0 wakes the others up at boot (INIT + "startup" IPIs, //! see smp.rs) and how a CPU pokes an idle one when new work arrives. //! * **The APIC timer.** A per-CPU countdown timer. We program it to fire //! every ~10 ms on *every* CPU, which drives preemption (time slices). //! Its speed isn't known in advance, so we measure it against the PIT. //! * **Its ID**, which tells a CPU which CPU it is (see `smp::cpu_id`). //! //! The registers are memory-mapped at physical 0xFEE0_0000 (each CPU sees //! its *own* APIC at that same address). We reach them through the //! physical-memory map. (On real hardware that range is uncached thanks to //! the firmware's MTRR settings; QEMU doesn't care either way.) //! //! The old 8259 PIC stays in charge of device interrupts (PIT timer, //! keyboard) and delivers them to CPU 0 only, through CPU 0's LINT0 pin //! ("virtual wire mode"). Routing device interrupts to any CPU would need //! the I/O APIC; we don't need that yet. use crate::interrupts::{self, InterruptIndex}; use crate::memory; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use x86_64::PhysAddr; // Register offsets (bytes from the APIC base). const ID: usize = 0x20; const TPR: usize = 0x80; // task priority: 0 = accept everything const EOI: usize = 0xB0; const SVR: usize = 0xF0; // spurious vector + "APIC enabled" bit 8 const ICR_LOW: usize = 0x300; const ICR_HIGH: usize = 0x310; const LVT_TIMER: usize = 0x320; const LVT_LINT0: usize = 0x350; const LVT_LINT1: usize = 0x360; const TIMER_INITIAL: usize = 0x380; const TIMER_CURRENT: usize = 0x390; const TIMER_DIVIDE: usize = 0x3E0; const MASKED: u32 = 1 << 16; const TIMER_PERIODIC: u32 = 1 << 17; const DIVIDE_BY_16: u32 = 0b0011; /// Virtual address of the APIC registers (0 = not set up yet). static BASE: AtomicU64 = AtomicU64::new(0); /// Timer count for one 10 ms slice (measured once, on CPU 0). static TIMER_COUNT: AtomicU32 = AtomicU32::new(0); /// Is CPU 0's APIC timer running? (Until then the PIT drives preemption.) static TIMER_ON: AtomicBool = AtomicBool::new(false); fn read(reg: usize) -> u32 { unsafe { core::ptr::read_volatile((BASE.load(Ordering::Relaxed) as usize + reg) as *const u32) } } fn write(reg: usize, value: u32) { unsafe { core::ptr::write_volatile((BASE.load(Ordering::Relaxed) as usize + reg) as *mut u32, value) } } /// Is the APIC usable yet? pub fn is_ready() -> bool { BASE.load(Ordering::Relaxed) != 0 } /// This CPU's APIC ID (0 before `init_bsp`). pub fn id() -> u32 { if is_ready() { read(ID) >> 24 } else { 0 } } /// "End of interrupt": tell the APIC we handled its interrupt (needed for /// the APIC timer and IPIs; PIC interrupts get their EOI at the PIC). pub fn eoi() { write(EOI, 0); } /// Is the APIC timer driving preemption? pub fn timer_running() -> bool { TIMER_ON.load(Ordering::Relaxed) } /// CPU 0: find and enable the APIC, measure its timer, start it. /// Needs interrupts on (it waits for PIT ticks). pub fn init_bsp() { use x86_64::registers::model_specific::Msr; // IA32_APIC_BASE MSR: physical base address in bits 12 and up. let phys = unsafe { Msr::new(0x1B).read() } & 0xF_FFFF_F000; BASE.store(memory::phys_to_virt(PhysAddr::new(phys)).as_u64(), Ordering::Relaxed); enable(); // Keep the PIC's interrupts flowing into this CPU: LINT0 = ExtINT // ("ask the PIC for the vector"), LINT1 = NMI. write(LVT_LINT0, 0x700); write(LVT_LINT1, 0x400); calibrate(); start_timer(); TIMER_ON.store(true, Ordering::Relaxed); } /// Another CPU: enable its APIC and start its timer. Its LINT0 stays /// masked (the PIC talks to CPU 0 only). pub fn init_ap() { enable(); write(LVT_LINT0, MASKED); write(LVT_LINT1, MASKED); start_timer(); } fn enable() { write(SVR, 0x100 | InterruptIndex::ApicSpurious as u32); write(TPR, 0); } /// Measure how far the APIC timer counts in 2 PIT ticks (~110 ms), and /// work out the count for 10 ms. fn calibrate() { write(TIMER_DIVIDE, DIVIDE_BY_16); write(LVT_TIMER, MASKED); // count down, but don't interrupt let start = interrupts::ticks(); while interrupts::ticks() == start { x86_64::instructions::hlt(); // line up with a tick edge } write(TIMER_INITIAL, u32::MAX); let tsc_start = unsafe { core::arch::x86_64::_rdtsc() }; let start = interrupts::ticks(); while interrupts::ticks() < start + 2 { x86_64::instructions::hlt(); } let per_tick = (u32::MAX - read(TIMER_CURRENT)) / 2; // While we're at it, measure the CPU's time stamp counter too (time.rs). crate::time::calibrate(tsc_start, unsafe { core::arch::x86_64::_rdtsc() }, 2); write(TIMER_INITIAL, 0); // One PIT tick is ~54.9 ms; we want 10 ms. let count = (per_tick as u64 * 10 / 55).max(1000) as u32; TIMER_COUNT.store(count, Ordering::Relaxed); } fn start_timer() { write(TIMER_DIVIDE, DIVIDE_BY_16); write(LVT_TIMER, InterruptIndex::ApicTimer as u32 | TIMER_PERIODIC); write(TIMER_INITIAL, TIMER_COUNT.load(Ordering::Relaxed)); } /// Write the interrupt command register: send `command` to APIC `target`, /// then wait until the APIC has delivered it. fn send(target: u32, command: u32) { write(ICR_HIGH, target << 24); write(ICR_LOW, command); // writing the low half sends it while read(ICR_LOW) & (1 << 12) != 0 { core::hint::spin_loop(); // "delivery pending" } } /// Interrupt CPU `target` with `vector` (a normal, "fixed" interrupt). pub fn send_ipi(target: u32, vector: u8) { send(target, 0x4000 | vector as u32); } /// INIT IPI: reset CPU `target` into its "wait for startup IPI" state. pub fn send_init(target: u32) { send(target, 0x4500); } /// Startup IPI: CPU `target` starts in 16-bit real mode at physical /// address `page * 4096`. pub fn send_startup(target: u32, page: u8) { send(target, 0x4600 | page as u32); }