rust-kernel

//! Interrupt handling: the IDT, CPU exception handlers and hardware IRQs.
//!
//! The IDT (Interrupt Descriptor Table) maps each interrupt vector number to
//! a handler function:
//!   * vectors 0..=31  are CPU exceptions (divide error, page fault, ...)
//!   * vectors 32..=47 are where we remap the 8259 PIC's 16 hardware IRQs
//!   * 0x40, 0x41, 0x42, 0xFF come from the local APIC (see apic.rs): its
//!     timer, the "reschedule" and "TLB shootdown" IPIs one CPU sends
//!     another, and spurious interrupts
//!
//! All CPUs share this one IDT. Device IRQs (PIT, keyboard) only ever
//! arrive on CPU 0; APIC timer ticks and IPIs arrive on every CPU.
//!
//! Most handlers use the special `extern "x86-interrupt"` calling convention:
//! the compiler saves/restores all registers and returns with `iretq`. The
//! ones that can interrupt a user program and must be able to send it a
//! signal (faults, the APIC timer) use trap.rs's stubs instead.

use crate::signal::{self, SigInfo};
use crate::trap::{self, TrapFrame};
use crate::{gdt, println, serial_println};
use core::sync::atomic::{AtomicU64, Ordering};
use lazy_static::lazy_static;
use pic8259::ChainedPics;
use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode};

// ---------------------------------------------------------------------------
// 8259 PIC (Programmable Interrupt Controller)
// ---------------------------------------------------------------------------

/// The PICs deliver IRQ 0..15 on vectors 0..15 by default, which clash with
/// CPU exceptions. We remap them to 32..47.
pub const PIC_1_OFFSET: u8 = 32;
pub const PIC_2_OFFSET: u8 = PIC_1_OFFSET + 8;

/// Two PICs: primary handles IRQ 0-7, secondary IRQ 8-15 (chained via IRQ 2).
pub static PICS: spin::Mutex<ChainedPics> =
    spin::Mutex::new(unsafe { ChainedPics::new(PIC_1_OFFSET, PIC_2_OFFSET) });

/// Vector numbers of the hardware interrupts we handle.
#[derive(Debug, Clone, Copy)]
#[repr(u8)]
pub enum InterruptIndex {
    Timer = PIC_1_OFFSET, // IRQ 0: Programmable Interval Timer, ~18.2 Hz by default
    Keyboard,             // IRQ 1: PS/2 keyboard
    ApicTimer = 0x40,     // each CPU's local APIC timer: time slices (~100 Hz)
    Reschedule = 0x41,    // IPI: "wake up, there's work for you"
    TlbShootdown = 0x42,  // IPI: "flush your TLB" (smp::tlb_shootdown)
    ApicSpurious = 0xFF,  // the APIC's "never mind" interrupt
}

impl InterruptIndex {
    fn as_u8(self) -> u8 {
        self as u8
    }
}

// ---------------------------------------------------------------------------
// IDT setup
// ---------------------------------------------------------------------------

lazy_static! {
    /// The IDT must live forever (`'static`): the CPU keeps a pointer to it.
    static ref IDT: InterruptDescriptorTable = {
        let mut idt = InterruptDescriptorTable::new();
        idt.breakpoint.set_handler_fn(breakpoint_handler);
        unsafe {
            // Exceptions a program can cause, and the APIC timer, go through
            // trap.rs's stubs: they save all registers in a TrapFrame, so a
            // signal can be delivered on the way back to user mode.
            let addr = |f: unsafe extern "C" fn()| x86_64::VirtAddr::new(f as usize as u64);
            idt.divide_error.set_handler_addr(addr(trap::trap_divide_error));
            idt.invalid_opcode.set_handler_addr(addr(trap::trap_invalid_opcode));
            idt.general_protection_fault.set_handler_addr(addr(trap::trap_general_protection));
            idt.page_fault.set_handler_addr(addr(trap::trap_page_fault));
            idt.x87_floating_point.set_handler_addr(addr(trap::trap_x87_error));
            idt.simd_floating_point.set_handler_addr(addr(trap::trap_simd_error));
            idt[InterruptIndex::ApicTimer.as_u8()].set_handler_addr(addr(trap::trap_apic_timer));

            // Run the double-fault handler on its own known-good stack (IST).
            idt.double_fault
                .set_handler_fn(double_fault_handler)
                .set_stack_index(gdt::DOUBLE_FAULT_IST_INDEX);
        }
        idt[InterruptIndex::Timer.as_u8()].set_handler_fn(timer_interrupt_handler);
        idt[InterruptIndex::Keyboard.as_u8()].set_handler_fn(keyboard_interrupt_handler);
        // PCI cards' legacy interrupts arrive on whichever IRQ line the
        // firmware routed them to (QEMU: 5, 9, 10 or 11).
        idt[PIC_1_OFFSET + 5].set_handler_fn(pci_irq5_handler);
        idt[PIC_1_OFFSET + 7].set_handler_fn(spurious_irq7_handler);
        idt[PIC_1_OFFSET + 15].set_handler_fn(spurious_irq15_handler);
        idt[PIC_1_OFFSET + 9].set_handler_fn(pci_irq9_handler);
        idt[PIC_1_OFFSET + 10].set_handler_fn(pci_irq10_handler);
        idt[PIC_1_OFFSET + 11].set_handler_fn(pci_irq11_handler);
        idt[InterruptIndex::Reschedule.as_u8()].set_handler_fn(reschedule_handler);
        idt[InterruptIndex::TlbShootdown.as_u8()].set_handler_fn(tlb_shootdown_handler);
        idt[InterruptIndex::ApicSpurious.as_u8()].set_handler_fn(spurious_handler);
        idt
    };
}

/// Load the IDT into the CPU (`lidt`). Every CPU does this once.
pub fn init_idt() {
    IDT.load();
}

// ---------------------------------------------------------------------------
// CPU exception handlers
// ---------------------------------------------------------------------------

/// `int3` (what debuggers use). Harmless: print and carry on.
extern "x86-interrupt" fn breakpoint_handler(stack_frame: InterruptStackFrame) {
    println!("EXCEPTION: BREAKPOINT");
    serial_println!("EXCEPTION: BREAKPOINT\n{:#?}", stack_frame);
}

/// Rust side of trap.rs's stubs: CPU exceptions a program can cause, and
/// the APIC timer. On the way out, if we're returning to user mode, act on
/// pending signals (that may run a handler, or end the process).
pub extern "C" fn trap_handler(frame: &mut TrapFrame) {
    match frame.vector {
        0x40 => {
            // Every CPU's APIC timer, ~100 times a second: time's up for the
            // running thread, let the next one have a go (see thread.rs).
            crate::apic::eoi(); // before switching: the next thread may be new
            crate::thread::timer_tick();
        }
        14 => page_fault(frame),
        0 => exception(frame, signal::SIGFPE, 1 /* FPE_INTDIV */, frame.rip, "divide error"),
        6 => exception(frame, signal::SIGILL, 1 /* ILL_ILLOPC */, frame.rip, "invalid opcode"),
        13 => exception(frame, signal::SIGSEGV, signal::SI_KERNEL, 0, "general protection fault"),
        16 => exception(frame, signal::SIGFPE, 0, frame.rip, "x87 floating-point error"),
        19 => exception(frame, signal::SIGFPE, simd_fpe_code(), frame.rip, "SIMD floating-point error"),
        v => panic!("unexpected trap {}", v),
    }
    signal::deliver(frame);
}

/// SIGFPE's si_code for an SSE exception (#XM, only raised if the program
/// unmasked it in MXCSR): which one, from MXCSR's status flags.
fn simd_fpe_code() -> i32 {
    let m = crate::fpu::mxcsr();
    let unmasked = m & !(m >> 7) & 0x3f; // flags whose mask bit (7-12) is clear
    match unmasked {
        f if f & 1 != 0 => 7,  // FPE_FLTINV: invalid operation
        f if f & 4 != 0 => 3,  // FPE_FLTDIV: divide by zero
        f if f & 8 != 0 => 4,  // FPE_FLTOVF: overflow
        f if f & 16 != 0 => 5, // FPE_FLTUND: underflow
        f if f & 32 != 0 => 6, // FPE_FLTRES: inexact result
        _ => 0,
    }
}

/// An exception in user mode is the program's fault, not the kernel's: it
/// gets a signal (which kills it, unless it has a handler). In the kernel
/// it's a bug: stop.
fn exception(frame: &TrapFrame, sig: u32, code: i32, addr: u64, what: &'static str) {
    if !frame.from_user() {
        panic!("EXCEPTION: {} (error {:#x})\n{:#x?}", what, frame.error, frame);
    }
    serial_println!("[kernel] {} in user mode at rip {:#x}", what, frame.rip);
    signal::force(sig, SigInfo::fault(sig, code, addr), what);
}

/// Page fault: CR2 holds the address, the error code says why (page not
/// present? write? user mode? instruction fetch?). In user mode this is
/// usually *not* an error: demand paging and copy-on-write (mm.rs) fix the
/// page up and the instruction runs again. Only if the address isn't in
/// any region (or the region forbids the access) does the program get
/// SIGSEGV. The kernel itself faults on user memory only when another
/// thread changed it during a syscall (below); any other kernel-mode page
/// fault is a bug.
fn page_fault(frame: &mut TrapFrame) {
    use crate::mm::Fault;
    use x86_64::registers::control::Cr2;
    let addr = Cr2::read_raw();
    let error = PageFaultErrorCode::from_bits_truncate(frame.error);
    if frame.from_user() {
        // We came from ring 3, so we hold no locks: interrupts may come in
        // (reading a file-backed page waits for the disk).
        x86_64::instructions::interrupts::enable();
        let write = error.contains(PageFaultErrorCode::CAUSED_BY_WRITE);
        let exec = error.contains(PageFaultErrorCode::INSTRUCTION_FETCH);
        let p = crate::thread::current_process().expect("user fault without a process");
        let result = if (crate::memory::USER_START..crate::memory::USER_END).contains(&addr) {
            p.mm.lock().handle_fault(addr, write, exec)
        } else {
            Err(Fault::Unmapped) // a kernel address
        };
        drop(p);
        let Err(fault) = result else { return }; // fixed: retry the instruction
        serial_println!("[kernel] user page fault: address {:#x}, rip {:#x}, {:?}, {:?}", addr, frame.rip, error, fault);
        let (sig, code, why) = match fault {
            // Just below the user stack is a guard zone that's never mapped.
            Fault::Unmapped if crate::process::in_user_stack_guard(addr) => (signal::SIGSEGV, signal::SEGV_MAPERR, "stack overflow"),
            Fault::Unmapped => (signal::SIGSEGV, signal::SEGV_MAPERR, "page fault"),
            Fault::Protection => (signal::SIGSEGV, signal::SEGV_ACCERR, "page fault"),
            Fault::Io => (signal::SIGBUS, 2 /* BUS_ADRERR */, "I/O error"),
            Fault::OutOfMemory => (signal::SIGKILL, signal::SI_KERNEL, "out of memory"),
        };
        signal::force(sig, SigInfo::fault(sig, code, addr), why);
        return;
    }
    // The kernel touched user memory a syscall had checked (user_slice), but
    // another thread of the process changed the mapping since: munmap'd it,
    // or fork made it copy-on-write. Fix it up like a user fault; if the
    // memory is really gone, map a blank page so the kernel can finish, and
    // kill the process (Linux would return EFAULT; we have no way back to
    // the syscall from here).
    if (crate::memory::USER_START..crate::memory::USER_END).contains(&addr) {
        if let Some(p) = crate::thread::current_process() {
            if frame.rflags & (1 << 9) != 0 {
                x86_64::instructions::interrupts::enable(); // as it was
            }
            let write = error.contains(PageFaultErrorCode::CAUSED_BY_WRITE);
            let mut mm = p.mm.lock();
            if mm.handle_fault(addr, write, false).is_ok() {
                return;
            }
            serial_println!("[kernel] syscall touched unmapped user memory {:#x} (another thread unmapped it?)", addr);
            if mm.map_scratch(addr).is_some() {
                drop(mm);
                signal::force(signal::SIGSEGV, SigInfo::fault(signal::SIGSEGV, signal::SEGV_MAPERR, addr), "bad user pointer in a syscall");
                return;
            }
        }
    }
    // The kernel touched a kernel stack's guard page: that thread overflowed.
    if crate::stack::is_guard_page(addr) {
        crate::thread::die_of_stack_overflow(addr);
    }
    println!("EXCEPTION: PAGE FAULT at {:?}", Cr2::read());
    serial_println!(
        "EXCEPTION: PAGE FAULT\nAccessed address: {:?}\nError code: {:?}\n{:#x?}",
        Cr2::read(),
        error,
        frame
    );

    // In the deliberate page-fault demo, reaching here means success.
    #[cfg(all(feature = "headless", feature = "page-fault"))]
    {
        serial_println!("[ok] page fault was caught");
        crate::exit_qemu(crate::QemuExitCode::Success);
    }

    crate::hlt_loop();
}

/// Double fault: an exception happened while handling another one. Must not
/// return (hence `-> !`). Runs on the IST stack set up in gdt.rs.
extern "x86-interrupt" fn double_fault_handler(
    stack_frame: InterruptStackFrame,
    _error_code: u64, // always 0 for double faults
) -> ! {
    // The usual way a kernel stack overflow shows up: the thread's stack
    // pointer ran into the guard page, the page fault couldn't push its
    // frame there, and that second fault became a double fault. CR2 still
    // holds the guard page address. Kill just that thread.
    let addr = x86_64::registers::control::Cr2::read_raw();
    if crate::stack::is_guard_page(addr) {
        crate::thread::die_of_stack_overflow(addr);
    }
    serial_println!("EXCEPTION: DOUBLE FAULT\n{:#?}", stack_frame);

    // In the deliberate stack-overflow test, reaching here means success.
    #[cfg(all(feature = "headless", feature = "stack-overflow"))]
    {
        serial_println!("[ok] stack overflow was caught by the double fault handler");
        crate::exit_qemu(crate::QemuExitCode::Success);
    }

    panic!("EXCEPTION: DOUBLE FAULT");
}

// ---------------------------------------------------------------------------
// Hardware interrupt handlers
// ---------------------------------------------------------------------------

/// Timer ticks since boot. Atomic, so it's safe to touch from an interrupt.
pub static TICKS: AtomicU64 = AtomicU64::new(0);

/// PIT ticks since boot (~18.2 per second, ~55 ms each).
pub fn ticks() -> u64 {
    TICKS.load(Ordering::Relaxed)
}

/// IRQ 0 (CPU 0 only). Fires ~18.2 times per second (the PIT's power-on
/// default). It's the system clock: uptime, and waking sleeping threads.
extern "x86-interrupt" fn timer_interrupt_handler(_stack_frame: InterruptStackFrame) {
    let ticks = TICKS.fetch_add(1, Ordering::Relaxed) + 1;

    // Show uptime in the top-right corner of the screen, updated each second.
    if ticks % 18 == 0 {
        crate::vga_buffer::show_uptime(ticks / 18);
    }

    // Headless runs: a safety net in case the shell self-test hangs.
    #[cfg(feature = "headless")]
    if ticks == 60 * 18 {
        serial_println!("[failed] headless run still going after 60 s");
        crate::exit_qemu(crate::QemuExitCode::Failed);
    }

    // Tell the PIC we're done, or it will never send another timer interrupt.
    // Must happen *before* switching threads: the thread we switch to might
    // be brand new and never return through this handler.
    unsafe {
        PICS.lock().notify_end_of_interrupt(InterruptIndex::Timer.as_u8());
    }

    // Wake threads whose sleep is over (they're queued for any CPU).
    crate::thread::wake_sleepers(ticks);

    // Preemption is the APIC timer's job; before it runs, the PIT does it.
    if !crate::apic::timer_running() {
        crate::thread::schedule();
    }
}

/// Another CPU queued work and saw we were idle.
extern "x86-interrupt" fn reschedule_handler(_stack_frame: InterruptStackFrame) {
    crate::apic::eoi();
    crate::thread::schedule();
}

/// Another CPU changed page tables we may have cached (threads, smp.rs).
extern "x86-interrupt" fn tlb_shootdown_handler(_stack_frame: InterruptStackFrame) {
    crate::smp::handle_tlb_shootdown();
    crate::apic::eoi();
}

/// Spurious APIC interrupt: ignore it (and no EOI, by the spec).
extern "x86-interrupt" fn spurious_handler(_stack_frame: InterruptStackFrame) {}

/// IRQ 1. The keyboard controller has a scancode waiting at I/O port 0x60.
/// We must read it (or no more keyboard interrupts arrive), then just queue
/// it for the async keyboard task (task/keyboard.rs) to decode and print.
extern "x86-interrupt" fn keyboard_interrupt_handler(_stack_frame: InterruptStackFrame) {
    use x86_64::instructions::port::Port;

    let mut port = Port::new(0x60);
    let scancode: u8 = unsafe { port.read() };
    crate::task::keyboard::add_scancode(scancode);

    unsafe {
        PICS.lock().notify_end_of_interrupt(InterruptIndex::Keyboard.as_u8());
    }
}

/// Unmask PIC IRQ line `irq` (and the cascade line 2 for IRQ 8-15), so a
/// PCI card's interrupts get through. Lines are masked until a driver
/// asks: an interrupt nobody handles would just fire again forever.
pub fn enable_pci_irq(irq: u8) {
    if !matches!(irq, 5 | 9 | 10 | 11) {
        return; // no handler for that line: the driver keeps polling
    }
    x86_64::instructions::interrupts::without_interrupts(|| unsafe {
        let mut pics = PICS.lock();
        let [mut primary, mut secondary] = pics.read_masks();
        if irq < 8 {
            primary &= !(1 << irq);
        } else {
            secondary &= !(1 << (irq - 8));
            primary &= !(1 << 2);
        }
        pics.write_masks(primary, secondary);
    });
}

/// A PCI device interrupted. Only the network card uses interrupts (the
/// disks are polled), so ask it; then tell the PIC we're done.
fn pci_irq(irq: u8) {
    // Lines are shared: ask every driver whether it was its device.
    crate::net::handle_irq();
    crate::virtio::handle_irq();
    unsafe {
        PICS.lock().notify_end_of_interrupt(PIC_1_OFFSET + irq);
    }
}

/// *Spurious* PIC interrupts. With a level-triggered line (PCI) the device
/// can lower its line after the PIC told the CPU "interrupt!" but before
/// the CPU asked "which one?" (here: netd on another CPU read the card's ISR
/// register just then). The PIC then answers with its lowest-priority line,
/// 7 (primary) or 15 (secondary), without that line really firing. Without
/// these handlers the vector would be empty and the CPU would double fault.
/// A spurious IRQ 7 gets no end-of-interrupt (nothing is in service); a
/// spurious IRQ 15 still went through the primary's cascade line 2, so only
/// the primary gets one.
extern "x86-interrupt" fn spurious_irq7_handler(_: InterruptStackFrame) {}
extern "x86-interrupt" fn spurious_irq15_handler(_: InterruptStackFrame) {
    use x86_64::instructions::port::Port;
    unsafe { Port::<u8>::new(0x20).write(0x20) }; // EOI to the primary PIC only
}

extern "x86-interrupt" fn pci_irq5_handler(_: InterruptStackFrame) {
    pci_irq(5);
}
extern "x86-interrupt" fn pci_irq9_handler(_: InterruptStackFrame) {
    pci_irq(9);
}
extern "x86-interrupt" fn pci_irq10_handler(_: InterruptStackFrame) {
    pci_irq(10);
}
extern "x86-interrupt" fn pci_irq11_handler(_: InterruptStackFrame) {
    pci_irq(11);
}

// Runs under `cargo test`: if the handler were missing, this would fault.
#[test_case]
fn test_breakpoint_exception() {
    x86_64::instructions::interrupts::int3();
}