rust-kernel

//! Virtio: the "virtual hardware" standard QEMU, KVM, cloud VMs and
//! Firecracker all speak. (Stage 13; virtio-blk and virtio-net from stages
//! 6 and 12 predate this module and talk the older *legacy* interface on
//! their own -- see block/virtio_blk.rs.)
//!
//! A virtio device is a PCI card that doesn't pretend to be real hardware.
//! Instead of emulating, say, a 1990s graphics chip register by register,
//! guest and host agree on a simple shared-memory protocol:
//!
//! * **Feature bits.** The device offers features, the driver accepts the
//!   subset it understands. (We always accept VERSION_1, bit 32: "I speak
//!   the modern, little-endian, spec-1.0 interface".)
//! * **Virtqueues** (queue.rs): ring buffers in guest RAM. The driver puts
//!   buffer addresses into a queue and *notifies* the device (one register
//!   write); the device does the work and puts the buffer back on the
//!   "used" ring. A request is just bytes in a buffer whose format each
//!   device type defines (a GPU command, a block read, random bytes...).
//! * **Config space**: a few device-specific registers (the console's size,
//!   the GPU's number of scanouts...).
//!
//! **Legacy vs modern.** Old ("legacy", virtio 0.9.5) devices put all that in
//! one I/O-port BAR with fixed offsets -- simple, and what virtio_blk.rs and
//! virtio_net.rs use. Modern (virtio 1.0+) devices put several register
//! blocks into memory BARs and describe *where* with PCI capabilities
//! (vendor-specific, id 0x09). QEMU's rng, console and net cards are
//! "transitional" (they speak both); virtio-gpu is modern-only, so this
//! module does the modern way, and the three stage-13 drivers share it:
//!
//!   rng.rs      virtio-rng: entropy for getrandom(), /dev/random, smoltcp
//!   console.rs  virtio-console: a second terminal, /dev/hvc0
//!   gpu.rs      virtio-gpu (2D): a framebuffer, /dev/fb0, the fb console
//!
//! Spec: https://docs.oasis-open.org/virtio/virtio/v1.2/virtio-v1.2.html

pub mod console;
pub mod gpu;
pub mod queue;
pub mod rng;

use crate::pci::PciDevice;
use core::ptr::{read_volatile, write_volatile};
pub use queue::Virtqueue;

/// Virtio's PCI vendor id (Red Hat's). Modern device ids are 0x1040 + type
/// (gpu = type 16 -> 0x1050); transitional ones are 0x1000.. (rng 0x1005,
/// console 0x1003).
pub const VENDOR: u16 = 0x1af4;

// Device status bits: the driver walks through these while starting up.
const STATUS_ACKNOWLEDGE: u8 = 1; // "I noticed you"
const STATUS_DRIVER: u8 = 2; // "and I have a driver for you"
const STATUS_DRIVER_OK: u8 = 4; // "set up, go"
const STATUS_FEATURES_OK: u8 = 8; // "I accept these features"

/// Feature bit 32: the modern interface. Required for modern-only devices.
const F_VERSION_1: u64 = 1 << 32;

// Capability types (the `cfg_type` byte of a virtio PCI capability).
const CAP_COMMON: u8 = 1; // the common config: features, status, queues
const CAP_NOTIFY: u8 = 2; // doorbells: write a queue number to kick it
const CAP_ISR: u8 = 3; // interrupt status (reading it acknowledges)
const CAP_DEVICE: u8 = 4; // device-specific config

// Offsets inside the common config block (virtio spec 4.1.4.3).
const DEVICE_FEATURE_SELECT: u64 = 0x00;
const DEVICE_FEATURE: u64 = 0x04;
const DRIVER_FEATURE_SELECT: u64 = 0x08;
const DRIVER_FEATURE: u64 = 0x0c;
const DEVICE_STATUS: u64 = 0x14;
const QUEUE_SELECT: u64 = 0x16;
const QUEUE_SIZE: u64 = 0x18;
const QUEUE_ENABLE: u64 = 0x1c;
const QUEUE_NOTIFY_OFF: u64 = 0x1e;
const QUEUE_DESC: u64 = 0x20;
const QUEUE_DRIVER: u64 = 0x28; // the "available" ring
const QUEUE_DEVICE: u64 = 0x30; // the "used" ring

/// A modern virtio PCI device: virtual addresses of its register blocks.
pub struct Transport {
    pub pci: PciDevice,
    common: u64,
    notify: u64,
    notify_multiplier: u32,
    isr: u64,
    device: u64,
}

// Plain MMIO helpers. Volatile: every access must really reach the device.
fn r8(a: u64) -> u8 { unsafe { read_volatile(a as *const u8) } }
fn r16(a: u64) -> u16 { unsafe { read_volatile(a as *const u16) } }
fn r32(a: u64) -> u32 { unsafe { read_volatile(a as *const u32) } }
fn w8(a: u64, v: u8) { unsafe { write_volatile(a as *mut u8, v) } }
fn w16(a: u64, v: u16) { unsafe { write_volatile(a as *mut u16, v) } }
fn w32(a: u64, v: u32) { unsafe { write_volatile(a as *mut u32, v) } }
fn w64(a: u64, v: u64) { w32(a, v as u32); w32(a + 4, (v >> 32) as u32) }

impl Transport {
    /// Find the register blocks of a modern virtio device by walking its
    /// vendor-specific capabilities. Each one says: block type, which BAR,
    /// offset inside the BAR, length.
    pub fn new(pci: &PciDevice) -> Option<Transport> {
        if pci.vendor_id != VENDOR {
            return None;
        }
        pci.enable_memory_and_bus_master();
        let (mut common, mut notify, mut isr, mut device, mut multiplier) = (0, 0, 0, 0, 0);
        for cap in pci.capabilities(0x09) {
            let cfg_type = pci.read_u8(cap + 3);
            let bar = pci.read_u8(cap + 4);
            let offset = pci.read_u32(cap + 8) as u64;
            let length = pci.read_u32(cap + 12) as u64;
            if bar > 5 || pci.bar(bar) & 1 != 0 {
                continue; // we only do memory BARs (there's also a PCI-config window type)
            }
            let base = pci.bar_address(bar);
            if base == 0 {
                continue;
            }
            let virt = crate::memory::map_mmio(base + offset, length.max(1)).as_u64();
            match cfg_type {
                CAP_COMMON if common == 0 => common = virt,
                CAP_NOTIFY if notify == 0 => {
                    notify = virt;
                    multiplier = pci.read_u32(cap + 16); // extra field of this cap
                }
                CAP_ISR if isr == 0 => isr = virt,
                CAP_DEVICE if device == 0 => device = virt,
                _ => {}
            }
        }
        if common == 0 || notify == 0 || isr == 0 {
            return None; // a legacy-only device
        }
        Some(Transport { pci: *pci, common, notify, notify_multiplier: multiplier, isr, device })
    }

    /// Reset, then negotiate features: we accept `wanted` (plus VERSION_1)
    /// as far as the device offers them. Returns the agreed set.
    pub fn begin_init(&self, wanted: u64) -> Option<u64> {
        let status = self.common + DEVICE_STATUS;
        w8(status, 0); // reset
        while r8(status) != 0 {
            core::hint::spin_loop();
        }
        w8(status, STATUS_ACKNOWLEDGE);
        w8(status, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
        // Features are 64 bits read/written as two 32-bit halves.
        let mut offered = 0u64;
        for half in 0..2u32 {
            w32(self.common + DEVICE_FEATURE_SELECT, half);
            offered |= (r32(self.common + DEVICE_FEATURE) as u64) << (32 * half);
        }
        let agreed = offered & (wanted | F_VERSION_1);
        if agreed & F_VERSION_1 == 0 {
            return None;
        }
        for half in 0..2u32 {
            w32(self.common + DRIVER_FEATURE_SELECT, half);
            w32(self.common + DRIVER_FEATURE, (agreed >> (32 * half)) as u32);
        }
        w8(status, STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK);
        // The device clears FEATURES_OK again if it can't live with our choice.
        if r8(status) & STATUS_FEATURES_OK == 0 {
            return None;
        }
        Some(agreed)
    }

    /// Set up virtqueue number `index` with at most `max_size` entries.
    /// `interrupts`: false = we'll poll it, tell the device not to bother.
    pub fn setup_queue(&self, index: u16, max_size: u16, interrupts: bool) -> Option<Virtqueue> {
        w16(self.common + QUEUE_SELECT, index);
        let device_max = r16(self.common + QUEUE_SIZE);
        if device_max == 0 {
            return None; // no such queue
        }
        // Split queues need a power of two; QEMU's sizes are, so min() keeps it.
        let size = device_max.min(max_size);
        w16(self.common + QUEUE_SIZE, size);
        let notify_off = r16(self.common + QUEUE_NOTIFY_OFF) as u64;
        let doorbell = self.notify + notify_off * self.notify_multiplier as u64;
        let queue = Virtqueue::new(index, size, doorbell, interrupts)?;
        w64(self.common + QUEUE_DESC, queue.desc_phys());
        w64(self.common + QUEUE_DRIVER, queue.avail_phys());
        w64(self.common + QUEUE_DEVICE, queue.used_phys());
        w16(self.common + QUEUE_ENABLE, 1);
        Some(queue)
    }

    /// Last step: tell the device we're ready.
    pub fn driver_ok(&self) {
        let status = self.common + DEVICE_STATUS;
        w8(status, r8(status) | STATUS_DRIVER_OK);
    }

    /// Read (and so acknowledge) the interrupt status: bit 0 = a queue was
    /// used, bit 1 = the device config changed.
    pub fn read_isr(&self) -> u8 {
        r8(self.isr)
    }

    /// 32-bit word of the device-specific config.
    pub fn config_u32(&self, offset: u64) -> u32 {
        if self.device == 0 { 0 } else { r32(self.device + offset) }
    }
    pub fn set_config_u32(&self, offset: u64, value: u32) {
        if self.device != 0 {
            w32(self.device + offset, value)
        }
    }
}

/// Interrupt handler part, called by interrupts.rs for shared PCI lines.
/// Every device on a shared line must have its ISR read, or (level
/// triggered) the line stays asserted forever. Only the console uses
/// interrupts; the rng and the GPU are polled.
pub fn handle_irq() {
    console::handle_irq();
    gpu::handle_irq();
    rng::handle_irq();
}

/// Bring up all three stage-13 drivers. Returns boot log lines.
pub fn init(devices: &[PciDevice]) -> alloc::vec::Vec<alloc::string::String> {
    let mut log = alloc::vec::Vec::new();
    log.push(rng::init(devices));
    log.extend(console::init(devices));
    log.extend(gpu::init(devices));
    log
}