rust-kernel / src / arch / aarch64 / paging.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
//! Turning on the MMU.
//!
//! Until now every address was a physical address. With the MMU on, the
//! CPU looks each address up in **page tables**. An arm64 CPU has two
//! table pointers: `TTBR0_EL1` for the low half of the address space
//! (addresses starting with 0x0000...) and `TTBR1_EL1` for the high half
//! (0xffff...). Linux keeps the kernel in the high half and programs in the
//! low half; we set up both:
//!
//! - **identity map** (TTBR0): virtual = physical, so the code keeps
//!   running at the address it was linked at when the MMU switches on;
//! - **higher half** (TTBR1): the same memory again at
//!   `HIGH_BASE + physical`. The drivers switch to it right away
//!   (mmio.rs); the kernel's code is still linked low (moving it up is a
//!   later stage).
//!
//! Settings (TCR_EL1): 4 KiB pages ("granule"), 39-bit addresses on both
//! sides (T0SZ = T1SZ = 25), so a walk starts at a level-1 table whose
//! 512 entries cover 1 GiB each. The bring-up only needs 1 GiB **block**
//! entries, no smaller tables: one per (first address, kind) in the board's
//! memory map. Memory types come from MAIR_EL1: index 0 = Device-nGnRnE
//! (registers: no caching, no reordering), index 1 = Normal write-back RAM.

use super::board::{Board, Kind};
use core::arch::asm;
use core::ptr::addr_of_mut;

/// Where the higher half starts with T1SZ = 25: the top 2^39 bytes.
pub const HIGH_BASE: u64 = 0xffff_ff80_0000_0000;

#[repr(C, align(4096))]
struct Table([u64; 512]);

static mut IDENTITY: Table = Table([0; 512]);
static mut HIGH: Table = Table([0; 512]);

const VALID_BLOCK: u64 = 0b01; // level-1 block descriptor
const ATTR_DEVICE: u64 = 0 << 2; // MAIR index 0
const ATTR_NORMAL: u64 = 1 << 2; // MAIR index 1
const INNER_SHAREABLE: u64 = 3 << 8;
const ACCESS_FLAG: u64 = 1 << 10; // "accessed" (else the first access faults)
const PXN: u64 = 1 << 53; // the kernel may not execute it
const UXN: u64 = 1 << 54; // user space may not execute it

fn block(phys: u64, kind: Kind) -> u64 {
    let attrs = match kind {
        Kind::Normal => ATTR_NORMAL | INNER_SHAREABLE | UXN,
        Kind::Device => ATTR_DEVICE | PXN | UXN,
    };
    (phys & !((1 << 30) - 1)) | attrs | ACCESS_FLAG | VALID_BLOCK
}

/// The higher-half address of physical address `phys`.
pub fn high(phys: usize) -> usize {
    HIGH_BASE as usize + phys
}

/// Fill both tables from the board's memory map and switch the MMU on
/// (with the data and instruction caches).
///
/// # Safety
/// Once, early, on CPU 0, with the MMU off.
pub unsafe fn init(board: &dyn Board) {
    let (id, hi) = unsafe { (&mut *addr_of_mut!(IDENTITY), &mut *addr_of_mut!(HIGH)) };
    for &(phys, kind) in board.memory_map() {
        let index = (phys >> 30) as usize;
        id.0[index] = block(phys, kind);
        hi.0[index] = block(phys, kind);
    }
    let mair: u64 = 0xff << 8; // index 1 = 0xff (Normal WB), index 0 = 0x00 (Device-nGnRnE)
    let tcr: u64 = 25            // T0SZ: 39-bit low half
        | (1 << 8) | (1 << 10)   // table walks: inner/outer write-back cached
        | (3 << 12)              // ... inner shareable; TG0 = 0b00 = 4 KiB
        | (25 << 16)             // T1SZ: 39-bit high half
        | (1 << 24) | (1 << 26) | (3 << 28)
        | (2 << 30)              // TG1 = 0b10 = 4 KiB
        | (2 << 32); // IPS: 40-bit physical addresses (what a Cortex-A53 has)
    unsafe {
        asm!(
            "msr mair_el1, {mair}",
            "msr tcr_el1, {tcr}",
            "msr ttbr0_el1, {t0}",
            "msr ttbr1_el1, {t1}",
            "dsb ish",
            "tlbi vmalle1",
            "dsb ish",
            "isb",
            "mrs {s}, sctlr_el1",
            "orr {s}, {s}, #1",          // M: MMU on
            "orr {s}, {s}, #(1 << 2)",   // C: data cache
            "orr {s}, {s}, #(1 << 12)",  // I: instruction cache
            "msr sctlr_el1, {s}",
            "isb",
            mair = in(reg) mair,
            tcr = in(reg) tcr,
            t0 = in(reg) id.0.as_ptr() as u64,
            t1 = in(reg) hi.0.as_ptr() as u64,
            s = out(reg) _,
        );
    }
}

/// Is the MMU on?
pub fn enabled() -> bool {
    let s: u64;
    unsafe { asm!("mrs {}, sctlr_el1", out(reg) s) };
    s & 1 != 0
}