//! Interrupt handling: the IDT, CPU exception handlers and hardware IRQs. //! //! The IDT (Interrupt Descriptor Table) maps each interrupt vector number to //! a handler function: //! * vectors 0..=31 are CPU exceptions (divide error, page fault, ...) //! * vectors 32..=47 are where we remap the 8259 PIC's 16 hardware IRQs //! * 0x40, 0x41, 0xFF come from the local APIC (see apic.rs): its timer, //! the "reschedule" IPI one CPU sends another, and spurious interrupts //! //! All CPUs share this one IDT. Device IRQs (PIT, keyboard) only ever //! arrive on CPU 0; APIC timer ticks and IPIs arrive on every CPU. //! //! Handlers use the special `extern "x86-interrupt"` calling convention: the //! compiler saves/restores all registers and returns with `iretq`. use crate::{gdt, println, serial_println}; use core::sync::atomic::{AtomicU64, Ordering}; use lazy_static::lazy_static; use pic8259::ChainedPics; use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode}; // --------------------------------------------------------------------------- // 8259 PIC (Programmable Interrupt Controller) // --------------------------------------------------------------------------- /// The PICs deliver IRQ 0..15 on vectors 0..15 by default, which clash with /// CPU exceptions. We remap them to 32..47. pub const PIC_1_OFFSET: u8 = 32; pub const PIC_2_OFFSET: u8 = PIC_1_OFFSET + 8; /// Two PICs: primary handles IRQ 0-7, secondary IRQ 8-15 (chained via IRQ 2). pub static PICS: spin::Mutex = spin::Mutex::new(unsafe { ChainedPics::new(PIC_1_OFFSET, PIC_2_OFFSET) }); /// Vector numbers of the hardware interrupts we handle. #[derive(Debug, Clone, Copy)] #[repr(u8)] pub enum InterruptIndex { Timer = PIC_1_OFFSET, // IRQ 0: Programmable Interval Timer, ~18.2 Hz by default Keyboard, // IRQ 1: PS/2 keyboard ApicTimer = 0x40, // each CPU's local APIC timer: time slices (~100 Hz) Reschedule = 0x41, // IPI: "wake up, there's work for you" ApicSpurious = 0xFF, // the APIC's "never mind" interrupt } impl InterruptIndex { fn as_u8(self) -> u8 { self as u8 } } // --------------------------------------------------------------------------- // IDT setup // --------------------------------------------------------------------------- lazy_static! { /// The IDT must live forever (`'static`): the CPU keeps a pointer to it. static ref IDT: InterruptDescriptorTable = { let mut idt = InterruptDescriptorTable::new(); idt.breakpoint.set_handler_fn(breakpoint_handler); idt.page_fault.set_handler_fn(page_fault_handler); idt.general_protection_fault.set_handler_fn(general_protection_fault_handler); idt.invalid_opcode.set_handler_fn(invalid_opcode_handler); unsafe { // Run the double-fault handler on its own known-good stack (IST). idt.double_fault .set_handler_fn(double_fault_handler) .set_stack_index(gdt::DOUBLE_FAULT_IST_INDEX); } idt[InterruptIndex::Timer.as_u8()].set_handler_fn(timer_interrupt_handler); idt[InterruptIndex::Keyboard.as_u8()].set_handler_fn(keyboard_interrupt_handler); // PCI cards' legacy interrupts arrive on whichever IRQ line the // firmware routed them to (QEMU: 5, 9, 10 or 11). idt[PIC_1_OFFSET + 5].set_handler_fn(pci_irq5_handler); idt[PIC_1_OFFSET + 7].set_handler_fn(spurious_irq7_handler); idt[PIC_1_OFFSET + 15].set_handler_fn(spurious_irq15_handler); idt[PIC_1_OFFSET + 9].set_handler_fn(pci_irq9_handler); idt[PIC_1_OFFSET + 10].set_handler_fn(pci_irq10_handler); idt[PIC_1_OFFSET + 11].set_handler_fn(pci_irq11_handler); idt[InterruptIndex::ApicTimer.as_u8()].set_handler_fn(apic_timer_handler); idt[InterruptIndex::Reschedule.as_u8()].set_handler_fn(reschedule_handler); idt[InterruptIndex::ApicSpurious.as_u8()].set_handler_fn(spurious_handler); idt }; } /// Load the IDT into the CPU (`lidt`). Every CPU does this once. pub fn init_idt() { IDT.load(); } // --------------------------------------------------------------------------- // CPU exception handlers // --------------------------------------------------------------------------- /// `int3` (what debuggers use). Harmless: print and carry on. extern "x86-interrupt" fn breakpoint_handler(stack_frame: InterruptStackFrame) { println!("EXCEPTION: BREAKPOINT"); serial_println!("EXCEPTION: BREAKPOINT\n{:#?}", stack_frame); } /// Did the exception happen in user mode (ring 3)? Then it's the process's /// fault, not the kernel's: kill the process and keep the kernel running. /// The CPU puts the interrupted code's CS in the frame; its low 2 bits are /// the privilege level the code ran at. fn kill_if_user(stack_frame: &InterruptStackFrame, what: &'static str) { use x86_64::PrivilegeLevel; if stack_frame.code_segment.rpl() == PrivilegeLevel::Ring3 { serial_println!( "[kernel] {} in user mode at rip {:#x}", what, stack_frame.instruction_pointer.as_u64() ); crate::process::exit_current(crate::process::ExitStatus::Killed(what)); // never returns } } /// General protection fault: e.g. a privileged instruction (`hlt`, `cli`) /// in ring 3, or a bad segment selector. extern "x86-interrupt" fn general_protection_fault_handler( stack_frame: InterruptStackFrame, error_code: u64, ) { kill_if_user(&stack_frame, "general protection fault"); panic!("EXCEPTION: GENERAL PROTECTION FAULT (error {:#x})\n{:#?}", error_code, stack_frame); } /// Invalid opcode (`ud2`, or garbage executed as code). extern "x86-interrupt" fn invalid_opcode_handler(stack_frame: InterruptStackFrame) { kill_if_user(&stack_frame, "invalid opcode"); panic!("EXCEPTION: INVALID OPCODE\n{:#?}", stack_frame); } /// Page fault: accessing unmapped/protected memory. CR2 holds the bad address, /// the error code says why (not present? write? user mode?). A real OS would /// load the page from disk / swap here; we kill a faulting user process, and /// stop if the kernel itself faulted. extern "x86-interrupt" fn page_fault_handler( stack_frame: InterruptStackFrame, error_code: PageFaultErrorCode, ) { use x86_64::registers::control::Cr2; let addr = Cr2::read_raw(); if error_code.contains(PageFaultErrorCode::USER_MODE) { serial_println!("[kernel] user page fault: address {:#x}, error {:?}", addr, error_code); // Just below the user stack is a guard zone that's never mapped. if crate::process::in_user_stack_guard(addr) { kill_if_user(&stack_frame, "stack overflow"); } kill_if_user(&stack_frame, "page fault"); } // The kernel touched a kernel stack's guard page: that thread overflowed. if crate::stack::is_guard_page(addr) { crate::thread::die_of_stack_overflow(addr); } println!("EXCEPTION: PAGE FAULT at {:?}", Cr2::read()); serial_println!( "EXCEPTION: PAGE FAULT\nAccessed address: {:?}\nError code: {:?}\n{:#?}", Cr2::read(), error_code, stack_frame ); // In the deliberate page-fault demo, reaching here means success. #[cfg(all(feature = "headless", feature = "page-fault"))] { serial_println!("[ok] page fault was caught"); crate::exit_qemu(crate::QemuExitCode::Success); } crate::hlt_loop(); } /// Double fault: an exception happened while handling another one. Must not /// return (hence `-> !`). Runs on the IST stack set up in gdt.rs. extern "x86-interrupt" fn double_fault_handler( stack_frame: InterruptStackFrame, _error_code: u64, // always 0 for double faults ) -> ! { // The usual way a kernel stack overflow shows up: the thread's stack // pointer ran into the guard page, the page fault couldn't push its // frame there, and that second fault became a double fault. CR2 still // holds the guard page address. Kill just that thread. let addr = x86_64::registers::control::Cr2::read_raw(); if crate::stack::is_guard_page(addr) { crate::thread::die_of_stack_overflow(addr); } serial_println!("EXCEPTION: DOUBLE FAULT\n{:#?}", stack_frame); // In the deliberate stack-overflow test, reaching here means success. #[cfg(all(feature = "headless", feature = "stack-overflow"))] { serial_println!("[ok] stack overflow was caught by the double fault handler"); crate::exit_qemu(crate::QemuExitCode::Success); } panic!("EXCEPTION: DOUBLE FAULT"); } // --------------------------------------------------------------------------- // Hardware interrupt handlers // --------------------------------------------------------------------------- /// Timer ticks since boot. Atomic, so it's safe to touch from an interrupt. pub static TICKS: AtomicU64 = AtomicU64::new(0); /// PIT ticks since boot (~18.2 per second, ~55 ms each). pub fn ticks() -> u64 { TICKS.load(Ordering::Relaxed) } /// IRQ 0 (CPU 0 only). Fires ~18.2 times per second (the PIT's power-on /// default). It's the system clock: uptime, and waking sleeping threads. extern "x86-interrupt" fn timer_interrupt_handler(_stack_frame: InterruptStackFrame) { let ticks = TICKS.fetch_add(1, Ordering::Relaxed) + 1; // Show uptime in the top-right corner of the screen, updated each second. if ticks % 18 == 0 { crate::vga_buffer::show_uptime(ticks / 18); } // Headless runs: a safety net in case the shell self-test hangs. #[cfg(feature = "headless")] if ticks == 60 * 18 { serial_println!("[failed] headless run still going after 60 s"); crate::exit_qemu(crate::QemuExitCode::Failed); } // Tell the PIC we're done, or it will never send another timer interrupt. // Must happen *before* switching threads: the thread we switch to might // be brand new and never return through this handler. unsafe { PICS.lock().notify_end_of_interrupt(InterruptIndex::Timer.as_u8()); } // Wake threads whose sleep is over (they're queued for any CPU). crate::thread::wake_sleepers(ticks); // Preemption is the APIC timer's job; before it runs, the PIT does it. if !crate::apic::timer_running() { crate::thread::schedule(); } } /// Every CPU's APIC timer, ~100 times a second: time's up for the running /// thread, let the next one have a go (see thread.rs). extern "x86-interrupt" fn apic_timer_handler(_stack_frame: InterruptStackFrame) { crate::apic::eoi(); // before switching, as above crate::thread::timer_tick(); } /// Another CPU queued work and saw we were idle. extern "x86-interrupt" fn reschedule_handler(_stack_frame: InterruptStackFrame) { crate::apic::eoi(); crate::thread::schedule(); } /// Spurious APIC interrupt: ignore it (and no EOI, by the spec). extern "x86-interrupt" fn spurious_handler(_stack_frame: InterruptStackFrame) {} /// IRQ 1. The keyboard controller has a scancode waiting at I/O port 0x60. /// We must read it (or no more keyboard interrupts arrive), then just queue /// it for the async keyboard task (task/keyboard.rs) to decode and print. extern "x86-interrupt" fn keyboard_interrupt_handler(_stack_frame: InterruptStackFrame) { use x86_64::instructions::port::Port; let mut port = Port::new(0x60); let scancode: u8 = unsafe { port.read() }; crate::task::keyboard::add_scancode(scancode); unsafe { PICS.lock().notify_end_of_interrupt(InterruptIndex::Keyboard.as_u8()); } } /// Unmask PIC IRQ line `irq` (and the cascade line 2 for IRQ 8-15), so a /// PCI card's interrupts get through. Lines are masked until a driver /// asks: an interrupt nobody handles would just fire again forever. pub fn enable_pci_irq(irq: u8) { if !matches!(irq, 5 | 9 | 10 | 11) { return; // no handler for that line: the driver keeps polling } x86_64::instructions::interrupts::without_interrupts(|| unsafe { let mut pics = PICS.lock(); let [mut primary, mut secondary] = pics.read_masks(); if irq < 8 { primary &= !(1 << irq); } else { secondary &= !(1 << (irq - 8)); primary &= !(1 << 2); } pics.write_masks(primary, secondary); }); } /// A PCI device interrupted. Only the network card uses interrupts (the /// disks are polled), so ask it; then tell the PIC we're done. fn pci_irq(irq: u8) { // Lines are shared: ask every driver whether it was its device. crate::net::handle_irq(); crate::virtio::handle_irq(); unsafe { PICS.lock().notify_end_of_interrupt(PIC_1_OFFSET + irq); } } /// *Spurious* PIC interrupts. With a level-triggered line (PCI) the device /// can lower its line after the PIC told the CPU "interrupt!" but before /// the CPU asked "which one?" (here: netd on another CPU read the card's ISR /// register just then). The PIC then answers with its lowest-priority line, /// 7 (primary) or 15 (secondary), without that line really firing. Without /// these handlers the vector would be empty and the CPU would double fault. /// A spurious IRQ 7 gets no end-of-interrupt (nothing is in service); a /// spurious IRQ 15 still went through the primary's cascade line 2, so only /// the primary gets one. extern "x86-interrupt" fn spurious_irq7_handler(_: InterruptStackFrame) {} extern "x86-interrupt" fn spurious_irq15_handler(_: InterruptStackFrame) { use x86_64::instructions::port::Port; unsafe { Port::::new(0x20).write(0x20) }; // EOI to the primary PIC only } extern "x86-interrupt" fn pci_irq5_handler(_: InterruptStackFrame) { pci_irq(5); } extern "x86-interrupt" fn pci_irq9_handler(_: InterruptStackFrame) { pci_irq(9); } extern "x86-interrupt" fn pci_irq10_handler(_: InterruptStackFrame) { pci_irq(10); } extern "x86-interrupt" fn pci_irq11_handler(_: InterruptStackFrame) { pci_irq(11); } // Runs under `cargo test`: if the handler were missing, this would fault. #[test_case] fn test_breakpoint_exception() { x86_64::instructions::interrupts::int3(); }