diff --git a/Cargo.lock b/Cargo.lock index 8447da0a2ee..b47ebd0c4ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3014,10 +3014,12 @@ dependencies = [ "toyos-gpt", "toyos-hda", "toyos-pci", + "toyos-phys", "toyos-ps2", "toyos-quiesce", "toyos-random", "toyos-rootimage", + "toyos-smmu", "toyos-symbols", "toyos-tco", "toyos-tsc", diff --git a/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md b/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md new file mode 100644 index 00000000000..f7a60b0fde3 --- /dev/null +++ b/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md @@ -0,0 +1,24 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# A function the SMMUv3 does not route panics at attach + +`DeviceSpace::create` and `OwnSpace::create` take no function, so on the +SMMUv3 (`kernel/src/arch/aarch64/smmu/domain.rs`) a domain is made for any +caller. `attach` then looks the function up in the unit's routes, and +`Live::stream` panics where it finds none: an enumerated function whose IORT +route names another unit, names none, or was refused +(`kernel/src/arch/aarch64/smmu/mod.rs`, `init`'s route loop). A gap in a +firmware table becomes a kernel panic at `attach`, where it is a refusal at +`create`. Nothing on QEMU's `virt` reaches it: every function there is routed +through its one unit. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, whose claim through an +SMMUv3 domain is the first caller to hand `create` a function a process names. + +**Exit.** `create` takes the function and refuses, by name, one the unit does +not route, before an id is spent; `attach` has no lookup that can fail; and a +guest test, or a host test of the refusal, reds with the refusal deleted. diff --git a/issues/a-late-write-from-a-released-function-halts-the-machine.md b/issues/a-late-write-from-a-released-function-halts-the-machine.md new file mode 100644 index 00000000000..80b27463406 --- /dev/null +++ b/issues/a-late-write-from-a-released-function-halts-the-machine.md @@ -0,0 +1,34 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# A late write from a released function halts the machine + +`kernel/src/pcidev/mod.rs`'s `tear_down` clears the function's ownership +(`crate::iommu::note_user_owned(…, None)`) before its reset has quiesced what +it already issued: Bus Master Enable is cleared first, so it starts nothing +new, and its grants stay mapped until it is quiet, so nothing it issued lands +in a page handed on. A transaction already in flight that faults after the +ownership is cleared — at an address a grant never covered, or one an +`unmap` before the release took back — is read by the fault handler as a +function's no process drives: `owner=kernel`, and `crate::iommu::fault::conclude` +halts the machine. One process's bug then takes the machine down, which is the +thing a claim exists to prevent. + +This holds on x86-64 under VT-d today, and on AArch64 it is reachable once a +function can be claimed through an SMMUv3 domain. No test makes a write +outlive its function's release; QEMU's devices finish a DMA inside the access +that starts it, so no guest of this suite can. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, its claim of a function +through an SMMUv3 domain (the stage's exit: netd claims its NIC), which makes +the AArch64 half reachable and lands before it. + +**Exit.** A fault on a function between its release and its reset's quiet is +handed to the released claim's record and not to the kernel, the ownership +cleared only once the reset is quiet; a test that makes such a write — a +claimed function's grant unmapped while it is aimed there, its claim released +before the write lands — sees the record and a machine still running, and +reds with the ownership cleared where it is today. diff --git a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md index 06244c6ad8e..0f17a3bdc83 100644 --- a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md +++ b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md @@ -28,14 +28,20 @@ the two registers' offsets are `kernel/src/arch/aarch64/irqchip.rs`'s: one register's fields have two crates. `toyos_its::TRANSLATION_FRAME` and `toyos_gicv3::FRAME` are both the GIC's 64 KiB frame. -**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which -writes these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns -the descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for -every CPU, owns the redistributor. +**The distributor.** `toyos-its/src/lpi.rs`'s `Layout::new` decodes +`GICD_TYPER`'s `LPIS`, `IDbits` and `num_LPIs`, and +`kernel/src/arch/aarch64/irqchip.rs`'s `route_iommu_events` decodes its +`ITLinesNumber`: one register's fields have two homes. + +**Owner.** `issues/toyos-runs-on-arm64.md`. The descriptor is stage 4's, its +owed break-before-make ordering of a live entry's replacement, which rewrites +how `kernel/src/arch/aarch64/paging.rs` writes an entry. The redistributor and +the distributor are stage 6's ITS work, which edits `irqchip.rs` and reads +`GICR_TYPER` for every CPU and `GICD_TYPER` for its LPIs. **Exit.** Each bit and the level index has one declaration that the kernel's tables, the loader's and the unit's are all written from, and each -redistributor register's fields have one crate: +redistributor or distributor register's fields have one home: ``` rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_SHAREABLE|AF|PXN|UXN): u64' \ @@ -43,5 +49,6 @@ rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_ rg -n 'fn index' kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs ``` -print nothing, and `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one -directory. +print nothing, `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one +directory, and `rg -l 'ITLinesNumber|IDbits|num_LPIs' toyos-its/src kernel/src +kernel/gicv3/src` names files of one crate. diff --git a/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md b/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md new file mode 100644 index 00000000000..462fdf12d04 --- /dev/null +++ b/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md @@ -0,0 +1,32 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# The SMMUv3's global errors are read and never delivered + +`kernel/src/arch/aarch64/smmu/` reads `SMMU_GERROR` against `SMMU_GERRORN` +only where something else brings it there: in every bounded wait on a register +or the command queue, and after each drain of the event queue. Its own wired +interrupt, the IORT's `GERR` GSIV, is routed nowhere and +`SMMU_IRQ_CTRL.GERROR_IRQEN` stays clear, so an error the unit raises between +those reads goes unseen until the next one, and one it raises on a quiet +machine is never seen (IHI 0070 H.a §7.5 lists them): + +- `SFM_ERR`: the unit entered Service Failure Mode, terminating every client + transaction and no longer accessing its queues (§12.3). +- `EVENTQ_ABT_ERR`: a write to the event queue aborted, and events were lost + with no record of them; read only after a drain, which a lost record does + not start. +- `CMDQ_ERR` outside a wait: none can arise there today, since every command + is issued under a wait for its `CMD_SYNC`. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, its ITS work: it edits +`kernel/src/arch/aarch64/irqchip.rs` and `trap.rs`, where a second SPI is +routed and taken. + +**Exit.** `SMMU_IRQ_CTRL.GERROR_IRQEN` is set and the `GERR` SPI reaches a +handler that names each active error and acknowledges it in `SMMU_GERRORN`; +`virt_smmu`, or a test beside it, raises one — a `CMD_SYNC` behind an illegal +command is QEMU's `CMDQ_ERR` — and reds with the routing deleted. diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 334c8319800..b314d302cdf 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -493,6 +493,8 @@ toyos-gicv3 = { path = "gicv3" } toyos-gpt = { path = "../toyos-gpt" } toyos-hda = { path = "../toyos-hda" } toyos-pci = { path = "pci" } +toyos-phys = { path = "../toyos-phys" } +toyos-smmu = { path = "../toyos-smmu" } toyos-tco = { path = "../toyos-tco" } toyos-tsc = { path = "../toyos-tsc" } toyos-ps2 = { path = "ps2" } diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 5dffb8814ab..394348dcc98 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -157,6 +157,14 @@ actuators! { #[allow(dead_code)] psci_withheld = "psci-withheld"; + /// Give the `iommu-testdev` off bus 0 no route, and once the SMMUv3 is + /// armed have it and the one on bus 0 write on no domain, then the one on + /// bus 0 on a domain at an address it maps, and there again once it is + /// taken back. Judged by `virt_smmu`. + // The SMMUv3 is AArch64's alone, so x86-64 builds an accessor it never reads. + #[allow(dead_code)] + smmu_selftest = "smmu-selftest"; + /// Time the same read loop on every CPU, either side of the `mov cr0` that enables caching. control_regs_bench = "control-regs-bench"; diff --git a/kernel/src/arch/aarch64/iommu_unit.rs b/kernel/src/arch/aarch64/iommu_unit.rs deleted file mode 100644 index bb07a406314..00000000000 --- a/kernel/src/arch/aarch64/iommu_unit.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! The IOMMU: an SMMUv3 the IORT names, the port's stage 6. Until then no -//! unit exists, and [`init`] says so; a domain is refused rather than built. - -use crate::drivers::pci::PciDevice; -use crate::log; - -pub fn init( - _rsdp_addr: u64, - _devices: &[PciDevice], - _windows: &[toyos_abi::boot::RootBridgeWindow], -) { - log!("IOMMU: the SMMUv3 is the port's stage 6; no device is translated this boot"); -} - -pub mod domain { - use crate::iommu::{DomainId, IommuError, Iova, StreamId}; - - /// No unit is driven, so there is no domain to give. - pub fn create(_room: u64) -> Result<(DomainId, Iova), IommuError> { - Err(IommuError::NoUnit) - } - - pub fn map(_id: DomainId, _phys: u64, _bytes: u64) -> Result { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn map_at(_id: DomainId, _at: Iova, _phys: u64, _bytes: u64) -> Result<(), IommuError> { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn place(_id: DomainId, _at: Iova, _phys: u64, _bytes: u64) -> Result { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn unmap(_id: DomainId, _at: Iova, _bytes: u64) -> Result<(), IommuError> { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn attach(_stream: StreamId, _id: DomainId) { - unreachable!("no domain exists: `create` refuses every one") - } -} - -pub mod interrupt { - use crate::iommu::{Refused, StreamId}; - - pub struct Msi { - pub address: u32, - pub data: u32, - } - - pub struct Pin { - pub low: u32, - pub high: u32, - } - - /// No unit, so nothing remaps: callers deliver directly. - pub fn is_armed() -> bool { - false - } - - pub fn msi(_source: StreamId, _vector: u8, _dest: u32) -> Result { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn claim(_slot: usize, _source: StreamId, _vector: u8) -> Msi { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn release(_slot: usize, _source: StreamId) { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn pin(_apic_id: u8, _vector: u8, _dest: u32, _level: bool) -> Result { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } -} - -pub mod fault { - use crate::iommu::StreamId; - - pub fn user_owned(_stream: StreamId, _slot: Option) { - owed!("SMMUv3 fault reporting", "stage 6") - } - - pub fn service() { - owed!("SMMUv3 fault reporting", "stage 6") - } -} diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index ebef83ce249..e58f914e583 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -4,8 +4,9 @@ //! timer's EL1 virtual timer (Arm ARM K.a, chapter D12), whose PPI the GTDT //! names. //! -//! **What this kernel takes is SGIs and the timer's PPI, and nothing else.** -//! No SPI is routed and no LPI exists: a device's interrupt is a message the +//! **What this kernel takes is SGIs, the timer's PPI and the IOMMU's event +//! SPI ([`route_iommu_events`]), and nothing else.** No other SPI is routed +//! and no LPI exists: a device's interrupt is a message the //! GICv3 ITS translates, and every device that would take one is either a //! driver the small-kernel track moves out of the kernel or a claimed function, //! which the SMMUv3 of the port's stage 6 must translate first @@ -16,7 +17,7 @@ //! zero by the entry from EL2 the two count alike. It is level-triggered, so a //! handler that neither re-arms nor stops it takes it again at once. -use core::sync::atomic::{AtomicBool, AtomicU32, Ordering::Relaxed}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed}; use alloc::vec::Vec; @@ -62,6 +63,21 @@ const CTLR_GRP1: u32 = 1 << 1; const RWP: u32 = 1 << 31; /// `GICD_PIDR2.ArchRev`, bits 7:4: 3 is GICv3, 4 is GICv4. const GICD_PIDR2: u64 = 0xFFE8; +/// `GICD_TYPER.ITLinesNumber`, bits 4:0: the distributor takes INTIDs below +/// `32 * (ITLinesNumber + 1)`, and none from 1020. +const GICD_TYPER: u64 = 0x0004; +/// An SPI's bit in each of the word-per-32 registers, its byte in +/// `GICD_IPRIORITYR`, its two bits in `GICD_ICFGR` and its doubleword in +/// `GICD_IROUTER`. +const GICD_IGROUPR: u64 = 0x0080; +const GICD_ISENABLER: u64 = 0x0100; +const GICD_ICENABLER: u64 = 0x0180; +const GICD_ISPENDR: u64 = 0x0200; +const GICD_ICPENDR: u64 = 0x0280; +const GICD_IPRIORITYR: u64 = 0x0400; +const GICD_ICFGR: u64 = 0x0C00; +const GICD_IGRPMODR: u64 = 0x0D00; +const GICD_IROUTER: u64 = 0x6000; /// A redistributor's frames: `RD_base`, then `SGI_base` 64 KiB above it. const GICR_CTLR: u64 = 0x0000; @@ -92,6 +108,11 @@ const SPURIOUS: u32 = 1023; static TIMER_INTID: AtomicU32 = AtomicU32::new(0); static TIMER_EDGE: AtomicBool = AtomicBool::new(false); +/// The distributor's physical base, from [`init`]. +static DISTRIBUTOR: AtomicU64 = AtomicU64::new(0); +/// The IOMMU's event SPI; zero until [`route_iommu_events`]. +static IOMMU_EVENTS: AtomicU32 = AtomicU32::new(0); + /// Wait until `done`, for at most `1 / per_second` of a second counted at the /// rate firmware states: the boot has no calibrated clock yet. fn settles(per_second: u64, what: &str, done: impl Fn() -> bool) { @@ -159,6 +180,7 @@ pub fn init(rsdp_addr: u64) -> Gic { } let gicd = gicd.expect("GIC: the MADT names no distributor"); let distributor = crate::mm::paging::map_mmio(gicd, FRAME, MmioPolicy::Uncacheable); + DISTRIBUTOR.store(gicd, Relaxed); let revision = distributor.read_u32(GICD_PIDR2) >> 4 & 0xF; assert!(revision >= 3, "GIC: GICD_PIDR2.ArchRev is {revision}, and this kernel drives a GICv3 or later"); @@ -277,6 +299,49 @@ pub(super) fn timer_intid() -> u32 { TIMER_INTID.load(Relaxed) } +/// Take SPI `intid` on this CPU at [`PRIORITY`], edge-triggered, as the +/// IOMMU's event interrupt: an SMMUv3's wired interrupts are edge-triggered +/// (IHI 0070 H.a §3.18.2, §12.4), and the IORT names no trigger. Refused +/// with the distributor's limit where it takes no such SPI. +pub(super) fn route_iommu_events(intid: u32) -> Result<(), u32> { + let gicd = Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME); + let limit = (32 * ((gicd.read_u32(GICD_TYPER) & 0x1F) + 1)).min(1020); + if !(32..limit).contains(&intid) { + return Err(limit); + } + let (word, bit) = (u64::from(intid / 32) * 4, 1 << (intid % 32)); + gicd.write_u32(GICD_ICENABLER + word, bit); + settles(100, "GICD_CTLR after disabling an SPI", || gicd.read_u32(GICD_CTLR) & RWP == 0); + gicd.write_u32(GICD_IGROUPR + word, gicd.read_u32(GICD_IGROUPR + word) | bit); + gicd.write_u32(GICD_IGRPMODR + word, gicd.read_u32(GICD_IGRPMODR + word) & !bit); + let priority = GICD_IPRIORITYR + u64::from(intid & !3); + let shift = 8 * (intid % 4); + gicd.write_u32(priority, gicd.read_u32(priority) & !(0xFF << shift) | u32::from(PRIORITY) << shift); + let config = GICD_ICFGR + u64::from(intid / 16) * 4; + gicd.write_u32(config, gicd.read_u32(config) | 0b10 << (2 * (intid % 16))); + gicd.write_u64(GICD_IROUTER + 8 * u64::from(intid), toyos_gicv3::unpacked_affinity(cpu::hardware_id())); + gicd.write_u32(GICD_ICPENDR + word, bit); + IOMMU_EVENTS.store(intid, Relaxed); + gicd.write_u32(GICD_ISENABLER + word, bit); + Ok(()) +} + +/// Pend the IOMMU's event SPI again, for records its handler left behind: the +/// unit raises none for them. Taken once the handler's `end` deactivates it: +/// the write makes an active SPI active and pending, which is never signalled, +/// and deactivation leaves it pending (IHI 0069D §8.9.16, §4.1.2, §4.1.1). +pub(super) fn pend_iommu_events() { + let intid = IOMMU_EVENTS.load(Relaxed); + assert!(intid != 0, "GIC: the IOMMU's event SPI is pended before it is routed"); + let gicd = Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME); + gicd.write_u32(GICD_ISPENDR + u64::from(intid / 32) * 4, 1 << (intid % 32)); +} + +/// The IOMMU's event SPI, or zero where [`route_iommu_events`] routed none. +pub(super) fn iommu_events() -> u32 { + IOMMU_EVENTS.load(Relaxed) +} + /// Write `ICC_SGI1R_EL1` whole: the SGI it names is raised, after every store /// before it. Only a `DSB` orders a system register write after stores to /// Normal memory, so without it a target could take the SGI before the store diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index aeb2aef53e2..a3c29dfc62d 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -23,7 +23,6 @@ pub mod entropy; pub mod entry; pub mod fpu; pub mod hw; -pub mod iommu_unit; pub mod irqchip; pub mod keyboard_controller; pub mod paging; @@ -33,12 +32,15 @@ pub mod pmu; pub mod power; pub mod psci; pub mod rtc; +pub mod smmu; pub mod smp; pub mod switch; pub mod tlb; pub mod trap; pub mod watchdog; +pub use smmu as iommu_unit; + /// The machine every program image this kernel loads must be built for. pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::Aarch64; diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs new file mode 100644 index 00000000000..4b61ff63cb7 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -0,0 +1,185 @@ +//! A device's address space: one context descriptor, tagged with an ASID of +//! its own, over stage 1 tables that map 2 MiB blocks and nothing else +//! (`toyos_smmu::table`), and the device addresses it hands out, over the +//! unit's 48-bit input (`crate::iommu::window`). +//! +//! A new leaf needs no invalidation — the unit caches no translation that +//! faulted — and a removed one is invalidated by the domain's ASID, behind a +//! `CMD_SYNC`, before [`unmap`] returns. + +use toyos_phys::Phys; +use toyos_smmu::config::{Cd, Ste}; +use toyos_smmu::queue::Command; +use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS}; +use toyos_smmu::Asid; + +use super::{window, Live, UNIT}; +use crate::iommu::window::Window; +use crate::iommu::{DomainId, IommuError, Iova, StreamId}; +use crate::log; +use crate::mm::{PAGE_2M, PAGE_SIZE}; + +/// Domain ids start at 1: `DomainId` is never 0. +const FIRST: u16 = 1; + +pub(super) struct Domain { + asid: Asid, + root: Phys<12>, + context: Phys<6>, + addresses: Window, +} + +impl Live { + fn domain(&mut self, id: DomainId) -> &mut Domain { + &mut self.domains[usize::from(id.raw() - FIRST)] + } + + /// One 2 MiB block of `phys` at `at`, its tables grown on the way. + fn put(&mut self, root: Phys<12>, at: u64, phys: u64) { + let block = Phys::<21>::new(phys).expect("SMMU: a 2 MiB-aligned page of memory"); + let path = plan(at, Leaf::Memory(block, Access::ReadWrite), &self.unit).expect("SMMU: a handed-out address"); + let mut table = root; + let (leaf, walk) = path.indices().split_last().expect("a path names at least its leaf's table"); + for (level, index) in walk.iter().enumerate() { + let slot = window(table.get(), PAGE_SIZE).subregion(*index as u64 * 8, 8); + table = match entry(slot.read_u64(0), level) { + Entry::Table(below) => below, + Entry::Invalid => { + let below = Phys::new(self.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); + slot.write_u64(0, next(below, &self.unit).expect("SMMU: a table below the unit's output size")); + below + } + Entry::Mapped => panic!("SMMU: {at:#x}'s level {level} entry is a block, where a table goes"), + }; + } + let slot = window(table.get(), PAGE_SIZE).subregion(*leaf as u64 * 8, 8); + // A live leaf here is memory a holder still reaches: the caller takes + // a mapping back before it puts another at its address. + assert!(entry(slot.read_u64(0), walk.len()) == Entry::Invalid, "SMMU: {at:#x} was still mapped when a new leaf was written there"); + slot.write_u64(0, path.descriptor); + } + + /// Clear the 2 MiB block at `at`; `false` where nothing is mapped there. + fn take(&mut self, root: Phys<12>, at: u64) -> bool { + let any = Phys::<21>::new(0).expect("0 is aligned"); + let Some(path) = plan(at, Leaf::Memory(any, Access::Read), &self.unit) else { return false }; + let mut table = root; + let (leaf, walk) = path.indices().split_last().expect("a path names at least its leaf's table"); + for (level, index) in walk.iter().enumerate() { + match entry(window(table.get(), PAGE_SIZE).read_u64(*index as u64 * 8), level) { + Entry::Table(below) => table = below, + Entry::Invalid | Entry::Mapped => return false, + } + } + let slot = window(table.get(), PAGE_SIZE).subregion(*leaf as u64 * 8, 8); + if entry(slot.read_u64(0), walk.len()) != Entry::Mapped { + return false; + } + slot.write_u64(0, 0); + true + } +} + +/// A new domain, with `room` bytes of it handed out first: refused, with no id +/// spent, where it would have less than that. +pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { + let mut held = UNIT.lock(); + let live = held.as_mut().ok_or(IommuError::NoUnit)?; + let domains = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; + let id = u16::try_from(usize::from(FIRST) + live.domains.len()).map_err(|_| IommuError::DomainsExhausted(domains))?; + let asid = live.unit.asid(id).ok_or(IommuError::DomainsExhausted(domains))?; + let (addresses, first) = Window::new(INPUT_BITS as u8, &live.reserved, room)?; + let root = Phys::new(live.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); + let context = Phys::new(live.memory.alloc(64)).expect("SMMU: a descriptor below the unit's output size"); + let words = Cd::new(root, asid, &live.unit).expect("SMMU: a table below the unit's output size").words(); + let descriptor = window(context.get(), 64); + // `V` is in the first doubleword, so it goes last. + for (i, word) in words.iter().enumerate().skip(1) { + descriptor.write_u64(8 * i as u64, *word); + } + descriptor.write_u64(0, words[0]); + let domain = Domain { asid, root, context, addresses }; + log!( + "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {:#x} to {:#x}", + root.get(), + context.get(), + asid.get(), + addresses.floor(), + addresses.ceiling() + ); + live.domains.push(domain); + Ok((DomainId::new(id), first)) +} + +pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { + if !phys.is_multiple_of(PAGE_2M) { + return Err(IommuError::Unaligned(phys)); + } + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let domain = live.domain(id); + let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling()))?; + let root = domain.root; + for offset in (0..bytes).step_by(PAGE_2M as usize) { + live.put(root, at.raw() + offset, phys + offset); + } + log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + Ok(at) +} + +/// Put `bytes` at `phys` at `at` again, room this domain handed out before +/// and whose mapping was taken back. +pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + place(id, at, phys, bytes)?; + log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x} again", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + Ok(()) +} + +/// [`map_at`] without its record line, for a mapping a holder makes and takes +/// back as often as it likes. Room never handed out is a kernel bug: [`map`] +/// may yet hand it out. +pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { + if !phys.is_multiple_of(PAGE_2M) { + return Err(IommuError::Unaligned(phys)); + } + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let domain = live.domain(id); + assert!(domain.addresses.handed_out(at, bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); + let root = domain.root; + for offset in (0..bytes).step_by(PAGE_2M as usize) { + live.put(root, at.raw() + offset, phys + offset); + } + Ok(id.raw()) +} + +/// Take `bytes` at `at` back: no device reaches them once this returns. +pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let (root, asid) = (live.domain(id).root, live.domain(id).asid); + let mut result = Ok(()); + for offset in (0..bytes).step_by(PAGE_2M as usize) { + let here = at.raw() + offset; + if !live.take(root, here) { + result = Err(IommuError::NotMapped(Iova::translated(here))); + break; + } + } + // Whatever was cleared before a refusal is gone from the unit too. + live.issue(&[Command::InvalidateAsid(asid)]); + result +} + +/// Move `function` onto domain `id`: translating through it, and only it, +/// once this returns. +pub fn attach(function: StreamId, id: DomainId) { + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let stream = live.stream(function); + let context = live.domain(id).context; + let ste = Ste::stage1(context, &live.unit).expect("SMMU: a descriptor below the unit's output size"); + live.write_entry(stream, ste); + crate::iommu::fault::attached(function, id.raw()); + log!("iommu: {function} moves to domain{}", id.raw()); +} diff --git a/kernel/src/arch/aarch64/smmu/fault.rs b/kernel/src/arch/aarch64/smmu/fault.rs new file mode 100644 index 00000000000..995602fd0f9 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/fault.rs @@ -0,0 +1,103 @@ +//! The event queue, drained from the unit's wired event interrupt and never +//! polled. +//! +//! The handler allocates nothing and takes no lock: what it reads is +//! published once, whole, before the interrupt is enabled, and what it writes +//! is atomics. What a record ends is `crate::iommu::fault`'s, the policy every +//! backend applies; this reads the queue into it. +//! +//! One interrupt reads at most a queue's worth of records, so a device that +//! never stops writing, which nothing here can stop when no enumerated +//! function is behind its stream, cannot hold the CPU. The unit pulses its +//! interrupt only as the queue goes from empty to non-empty (IHI 0070 H.a +//! §3.18.2), so a record left behind raises nothing: a drain that stops short +//! of empty pends the interrupt again itself. + +use alloc::boxed::Box; +use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; + +use toyos_smmu::queue::{event, Events}; +use toyos_smmu::unit as reg; + +use super::Registers; +use crate::iommu::fault::{self as policy, Access, Fault, Who}; +use crate::iommu::StreamId; +use crate::log; +use crate::mm::Mmio; + +/// What the handler reads, published once by [`arm`]. +struct Armed { + regs: Registers, + queue: Mmio, + events: Events, + /// Each enumerated function the IORT routes through the unit, and its + /// StreamID. + routes: Box<[(StreamId, u32)]>, + /// Events this unit has recorded for the life of the boot. + faults: AtomicU32, +} + +static ARMED: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +fn armed() -> Option<&'static Armed> { + // SAFETY: non-null only as `arm` stored it: a leaked box, never freed, + // published with Release once it was whole. + unsafe { ARMED.load(Ordering::Acquire).as_ref() } +} + +/// Everything the handler reads, before the unit's event interrupt is +/// enabled: the registers, the queue and its indexes, and the routes. +pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, routes: &[(StreamId, u32)]) { + let armed = + Box::new(Armed { regs, queue, events, routes: routes.into(), faults: AtomicU32::new(0) }); + let first = ARMED.compare_exchange(core::ptr::null_mut(), Box::leak(armed), Ordering::Release, Ordering::Relaxed); + assert!(first.is_ok(), "SMMU: the event handler was armed twice"); +} + +/// The events the unit has recorded, as the handler has read them. +#[cfg(feature = "boot-actuators")] +pub(super) fn recorded() -> u32 { + armed().map_or(0, |armed| armed.faults.load(Ordering::Relaxed)) +} + +/// The unit's event interrupt. +pub fn service() { + let armed = armed().expect("SMMU: the event interrupt is enabled only once the handler is armed"); + let (regs, events) = (armed.regs, armed.events); + let mut kernel_owned = 0usize; + let mut cons = regs.read(reg::EVENTQ_CONS); + let mut prod = regs.read(reg::EVENTQ_PROD); + for _ in 0..events.entries() { + if events.is_empty(prod, cons) { + break; + } + if events.overflowed(prod, cons) { + log!("iommu: the SMMUv3's event queue overflowed: earlier events are lost"); + } + let at = events.slot(cons) as u64 * 32; + let record = event([0, 8, 16, 24].map(|offset| armed.queue.read_u64(at + offset))); + let who = match armed.routes.iter().find(|(_, stream)| *stream == record.stream) { + Some(&(function, _)) => Who::Function(function), + None => Who::Stream(record.stream), + }; + let (address, access) = match record.attempt { + Some(attempt) => (attempt.address, if attempt.write { Access::Write } else { Access::Read }), + None => (0, Access::Unrecorded), + }; + let fault = Fault { who, address, access, reason: record.number, name: record.code.name() }; + if policy::report(0, &armed.faults, fault) { + kernel_owned += 1; + } + cons = events.after(cons, prod); + regs.write(reg::EVENTQ_CONS, cons); + prod = regs.read(reg::EVENTQ_PROD); + } + if !events.is_empty(prod, cons) { + super::super::irqchip::pend_iommu_events(); + } + let errors = reg::active_errors(regs.read(reg::GERROR), regs.read(reg::GERRORN)); + if errors & reg::GERROR_EVENTQ_ABORT != 0 { + log!("iommu: the SMMUv3 could not write its event queue: events are lost (GERROR {errors:#x})"); + } + policy::conclude(kernel_owned); +} diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs new file mode 100644 index 00000000000..379fa132a55 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -0,0 +1,452 @@ +//! The IOMMU: the SMMUv3 the IORT names (Arm IHI 0070 H.a), programmed with +//! `toyos_smmu`'s encodings. Every register and Arm name of the unit is at or +//! below this module. +//! +//! [`init`] arms the unit with every stream aborting and no bypass anywhere: +//! `GBPA` aborts before anything else is written, so nothing passes while the +//! unit is programmed; and the unit is enabled. A stream translates only once +//! [`domain::attach`] puts it on a domain of its own, through stage 1 alone; +//! no domain maps memory by identity. An enumerated function's stream on no +//! domain aborts and records nothing (`Ste::ABORT`): firmware may leave one +//! mastering the bus until its driver resets it. **Every stream no +//! enumerated function is routed from has one answer**, inside the stream +//! table or past it: it aborts and is recorded — `C_BAD_STE` on an entry left +//! invalid, `C_BAD_STREAMID` under `CR2.RECINVSID` — and the machine goes on, +//! as it does for an unenumerated requester on VT-d. A unit this +//! kernel cannot program, and every unit of a machine whose IORT names more +//! than one, is left aborting every transaction, and no domain is given. +//! +//! The unit reads what it is given coherently — `toyos_smmu::unit::probe` +//! refuses one that does not — so a store it reads is published by the order +//! [`Mmio`] gives it ahead of the register write or the command that names it. +//! +//! One lock, [`UNIT`], over the unit's registers, queues, tables and domains. +//! The event interrupt takes none ([`fault`]). + +pub mod domain; +pub mod fault; +#[cfg(feature = "boot-actuators")] +mod selftest; + +use alloc::vec::Vec; + +use toyos_acpi::{IortRefused, Node, Route, TableError}; +use toyos_phys::Phys; +use toyos_smmu::config::Ste; +use toyos_smmu::queue::{Command, Commands, Signal}; +use toyos_smmu::unit::{self as reg, Unit}; + +use crate::drivers::acpi::direct_phys; +use crate::drivers::pci::PciDevice; +use crate::iommu::StreamId; +use crate::log; +use crate::mm::pmm::{self, PhysPage}; +use crate::mm::policy::MmioPolicy; +use crate::mm::{DirectMap, Mmio, PAGE_2M}; +use crate::sync::Lock; +use crate::time::{Duration, Tripwire}; + +/// Register pages 0 and 1, 64 KiB each (§6.1). +const REGISTER_PAGE: u64 = 0x1_0000; + +/// The largest queues this kernel makes, as log2 of their entries: 4 KiB each. +const COMMANDS_LOG2: u8 = 8; +const EVENTS_LOG2: u8 = 7; + +/// A stream table no larger than one 2 MiB page. +const STREAMS_LOG2_MAX: u32 = 15; + +/// How long the unit is given to take a register write or consume its +/// commands before the kernel panics: a unit half-way through either is one +/// whose reach nothing can state. +const UNIT_TIMEOUT: Tripwire = Tripwire::absurd( + Duration::from_secs(1), + "a unit that takes no register write or command for a second is one whose reach nothing can state", +); + +static UNIT: Lock> = Lock::new(None); + +/// The unit's two register pages, addressed by `toyos_smmu::unit`'s offsets. +#[derive(Clone, Copy)] +struct Registers(Mmio); + +impl Registers { + fn read(self, at: usize) -> u32 { + self.0.read_u32(at as u64) + } + + fn write(self, at: usize, value: u32) { + self.0.write_u32(at as u64, value); + } + + fn write64(self, at: usize, value: u64) { + self.0.write_u64(at as u64, value); + } + + /// Wait until `done`, panicking past [`UNIT_TIMEOUT`] or on a command the + /// unit stopped its queue at. + fn wait(self, what: &str, done: impl Fn() -> bool) { + let deadline = crate::clock::nanos_since_boot() + UNIT_TIMEOUT.nanos(); + while !done() { + let errors = reg::active_errors(self.read(reg::GERROR), self.read(reg::GERRORN)); + if errors & reg::GERROR_CMDQ != 0 { + let cons = self.read(reg::CMDQ_CONS); + panic!("SMMU: the command queue stopped at CONS {cons:#x}: {:?}", reg::command_error(cons)); + } + assert!( + crate::clock::nanos_since_boot() < deadline, + "SMMU: {what} not seen within {UNIT_TIMEOUT}; GERROR {errors:#x}" + ); + core::hint::spin_loop(); + } + } + + /// Write `value` to the control register at `at` and wait for its + /// acknowledgement register, the word above it, to read it back. + fn control(self, at: usize, value: u32, what: &str) { + self.write(at, value); + self.wait(what, || self.read(at + 4) == value); + } +} + +/// The armed unit and everything it reads. +struct Live { + regs: Registers, + unit: Unit, + commands: Commands, + queue: Mmio, + /// `SMMU_CMDQ_PROD` as last written. + prod: u32, + streams: Mmio, + memory: Memory, + /// Each enumerated function the IORT routes through this unit, and its + /// StreamID. + routes: Vec<(StreamId, u32)>, + domains: Vec, + /// What no domain's addresses may reach, as `(start, end)`. + reserved: Vec<(u64, u64)>, +} + +impl Live { + /// `commands`, then a `CMD_SYNC`, consumed: every one has taken effect + /// when this returns. + fn issue(&mut self, commands: &[Command]) { + let (regs, queue) = (self.regs, self.commands); + for command in commands.iter().copied().chain([Command::Sync(Signal::None)]) { + let prod = self.prod; + regs.wait("room in its command queue", || !queue.is_full(prod, regs.read(reg::CMDQ_CONS))); + let at = queue.slot(prod) as u64 * 16; + let [low, high] = command.words(); + self.queue.write_u64(at, low); + self.queue.write_u64(at + 8, high); + self.prod = queue.after(prod); + } + let prod = self.prod; + regs.write(reg::CMDQ_PROD, prod); + regs.wait("its CMD_SYNC consumed", || queue.is_empty(prod, regs.read(reg::CMDQ_CONS))); + } + + /// The StreamID the IORT gives `function`'s requests through this unit. + fn stream(&self, function: StreamId) -> u32 { + let route = self.routes.iter().find(|(rid, _)| *rid == function); + route.unwrap_or_else(|| panic!("SMMU: {function} is no function the IORT routes through this unit")).1 + } + + /// Point stream `stream`'s entry at `ste` and have the unit forget what + /// it held for the old one. The first doubleword goes last: it holds `V`, + /// `Config` and the context pointer, and the others are read only under + /// the configuration it names. + fn write_entry(&mut self, stream: u32, ste: Ste) { + let words = ste.words(); + let at = u64::from(stream) * 64; + for (i, word) in words.iter().enumerate().skip(1) { + self.streams.write_u64(at + 8 * i as u64, *word); + } + self.streams.write_u64(at, words[0]); + self.issue(&[Command::ForgetStream(stream)]); + } +} + +/// What the unit reads, carved from 2 MiB pages and never given back: a page +/// the unit may still walk is not a page to free. +struct Memory { + pages: Vec, + /// Bytes taken from the newest page; full to begin with. + used: u64, +} + +impl Memory { + const fn new() -> Self { + Self { pages: Vec::new(), used: PAGE_2M } + } + + /// `bytes` of zeroes, aligned to their own size: a power of two from 64 + /// bytes to 2 MiB. + fn alloc(&mut self, bytes: u64) -> u64 { + assert!(bytes.is_power_of_two() && (64..=PAGE_2M).contains(&bytes), "SMMU: {bytes:#x} bytes of tables"); + let mut at = self.used.next_multiple_of(bytes); + if at + bytes > PAGE_2M { + self.pages.push(pmm::alloc_page().expect("SMMU: no memory for the unit's tables")); + at = 0; + } + self.used = at + bytes; + let phys = self.pages.last().expect("a page was just pushed").direct_map().phys() + at; + let zeroed = window(phys, bytes); + for offset in (0..bytes).step_by(8) { + zeroed.write_u64(offset, 0); + } + phys + } +} + +/// `bytes` of [`Memory`] at `phys`. +fn window(phys: u64, bytes: u64) -> Mmio { + // SAFETY: every caller names memory `Memory::alloc` handed out, which is + // never freed and which the direct map covers for the machine's life. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } +} + +/// The SMMUv3, armed; a refusal leaves every unit the IORT names aborting +/// what it is given, or says why one could not be told to. +pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::RootBridgeWindow]) { + let iort = match toyos_acpi::iort(direct_phys(), rsdp_addr) { + Ok(iort) => iort, + Err(IortRefused::Table(TableError::Absent)) => { + log!("IOMMU: no IORT, so no SMMUv3: no device is translated this boot"); + return; + } + Err(why) => { + log!("IOMMU: the IORT is refused: {why:?}; no device is translated this boot"); + return; + } + }; + let named: Vec<_> = iort + .nodes() + .filter_map(|node| match node { + Node::Smmuv3(smmu) => Some(smmu), + _ => None, + }) + .collect(); + // Every unit, before any refusal: `GBPA`'s reset value is IMPLEMENTATION + // DEFINED, and one firmware left bypassing gives its devices all memory. + let mut aborting = Vec::new(); + for smmu in &named { + if !smmu.base.is_multiple_of(REGISTER_PAGE) { + log!("IOMMU: the SMMUv3's registers at {:#x} are not on a 64 KiB page: not touched", smmu.base); + continue; + } + let regs = Registers(crate::mm::paging::map_mmio(smmu.base, 2 * REGISTER_PAGE, MmioPolicy::Uncacheable)); + abort_unprogrammed(regs, smmu.base); + aborting.push((smmu, regs)); + } + let (smmu, regs) = match (named.len(), aborting.pop()) { + (0, _) => { + log!("IOMMU: the IORT names no SMMUv3: no device is translated this boot"); + return; + } + (1, Some(one)) => one, + (1, None) => return, + (count, _) => { + log!("IOMMU: the IORT names {count} SMMUv3s, and this kernel drives one: each is left aborting"); + return; + } + }; + + crate::iommu::fault::describe(devices); + let segment = u32::from(crate::pcidev::segment()); + let mut routes = Vec::new(); + for device in devices { + let function = StreamId::pci(device.bus, device.dev, device.func); + match iort.route(segment, function.requester()) { + #[cfg(feature = "boot-actuators")] + Ok(Route::Translated { stream, .. }) + if crate::actuator::smmu_selftest() && selftest::stands_unrouted(device, devices) => + { + log!("smmu-selftest: {function}, StreamID {stream:#x}, is given no route, as a function never enumerated is not"); + } + Ok(Route::Translated { smmu: by, stream, .. }) if by.base == smmu.base => routes.push((function, stream)), + Ok(route) => log!("IOMMU: {function} is not routed through the SMMUv3: {route:?}"), + Err(why) => log!("IOMMU: {function}'s IORT route is refused: {why:?}"), + } + } + let reserved = windows.iter().map(|w| (w.base, w.end())).collect(); + let Some(live) = program(regs, smmu.base, smmu.coherent_override, smmu.event, routes, reserved) else { + return; + }; + *UNIT.lock() = Some(live); + #[cfg(feature = "boot-actuators")] + if crate::actuator::smmu_selftest() { + selftest::run(devices); + } +} + +/// `GBPA` aborting, the unit disabled and none of its interrupts able to +/// write: from here until `SMMUEN`, and for good if it is never set, every +/// transaction aborts. +fn abort_unprogrammed(regs: Registers, base: u64) { + let gbpa = || regs.read(reg::GBPA); + regs.wait("GBPA free to update", || gbpa() & reg::GBPA_UPDATE == 0); + regs.write(reg::GBPA, gbpa() | reg::GBPA_ABORT | reg::GBPA_UPDATE); + regs.wait("GBPA updated", || gbpa() & reg::GBPA_UPDATE == 0); + assert!( + gbpa() & reg::GBPA_ABORT != 0, + "SMMU: GBPA reads {:#x} after ABORT was written: transactions bypass while SMMUEN is clear", + gbpa() + ); + let cr0 = regs.read(reg::CR0); + if cr0 != 0 { + log!("IOMMU: the SMMUv3 at {base:#x} was handed over with CR0 {cr0:#x}; it goes off first"); + regs.control(reg::CR0, 0, "CR0 cleared"); + } + // An `*_IRQ_CFG0` address that is not zero makes its interrupt a write + // there, which no stream table entry governs; each resets UNKNOWN and is + // written only with its interrupt off (§3.18.2, §6.3.21, §6.3.30, §6.3.34). + regs.control(reg::IRQ_CTRL, 0, "its interrupts disabled"); + regs.write64(reg::GERROR_IRQ_CFG0, 0); + regs.write64(reg::EVENTQ_IRQ_CFG0, 0); + if regs.read(reg::IDR0) & reg::IDR0_PRI != 0 { + regs.write64(reg::PRIQ_IRQ_CFG0, 0); + } + log!("IOMMU: SMMUv3 at {base:#x}: GBPA {:#x}, every transaction aborts while SMMUEN is clear", gbpa()); +} + +/// The unit's tables, queues and interrupt, and `SMMUEN`; `None`, with the +/// reason logged, where it cannot be, the unit left as +/// [`abort_unprogrammed`] left it. +fn program( + regs: Registers, + base: u64, + coherent_override: bool, + event: Option, + routes: Vec<(StreamId, u32)>, + reserved: Vec<(u64, u64)>, +) -> Option { + let refused = |why: core::fmt::Arguments<'_>| log!("IOMMU: the SMMUv3 at {base:#x} is left aborting: {why}"); + let (idr0, idr1, idr5) = (regs.read(reg::IDR0), regs.read(reg::IDR1), regs.read(reg::IDR5)); + let unit = match reg::probe(idr0, idr1, idr5, coherent_override) { + Ok(unit) => unit, + Err(lacks) => { + refused(format_args!("it lacks {lacks:?} (IDR0 {idr0:#x}, IDR1 {idr1:#x}, IDR5 {idr5:#x})")); + return None; + } + }; + // Every table and leaf below is in memory this bounds, so none of + // `toyos_smmu`'s answers past the unit's output size can come back. + let top = pmm::top(); + let last_page = Phys::new((top - 1) & !(crate::mm::PAGE_SIZE - 1)); + if last_page.and_then(|page| toyos_smmu::table::next(page, &unit)).is_none() { + refused(format_args!("memory reaches {top:#x}, past what it outputs (IDR5 {idr5:#x})")); + return None; + } + let Some(event) = event else { + refused(format_args!("the IORT gives its event queue no wired interrupt")); + return None; + }; + let highest = routes.iter().map(|(_, stream)| *stream).max().unwrap_or(0); + let log2 = u32::BITS - highest.leading_zeros(); + if log2 > STREAMS_LOG2_MAX || log2 > u32::from(unit.stream_bits) { + refused(format_args!("StreamID {highest:#x} needs a table of 2^{log2} entries")); + return None; + } + if let Err(limit) = super::irqchip::route_iommu_events(event.get()) { + refused(format_args!("its event interrupt {event} is no SPI the distributor takes below {limit}")); + return None; + } + + let mut memory = Memory::new(); + let table = memory.alloc(64 << log2); + let streams = window(table, 64 << log2); + // Every other entry stays zero, invalid: a stream no function is routed + // from is recorded, as one past the table is. + for (_, stream) in &routes { + for (i, word) in Ste::ABORT.words().iter().enumerate() { + streams.write_u64(u64::from(*stream) * 64 + 8 * i as u64, *word); + } + } + let (commands_log2, events_log2) = + (unit.command_queue_log2.min(COMMANDS_LOG2), unit.event_queue_log2.min(EVENTS_LOG2)); + let commands_at = memory.alloc(16 << commands_log2); + let events_at = memory.alloc(32 << events_log2); + let (Some((table_base, table_cfg)), Some((commands_base, commands)), Some((events_base, events))) = ( + Phys::new(table).and_then(|at| unit.stream_table(at, log2 as u8)), + Phys::new(commands_at).and_then(|at| unit.command_queue(at, commands_log2)), + Phys::new(events_at).and_then(|at| unit.event_queue(at, events_log2)), + ) else { + unreachable!("SMMU: each is aligned to its size, inside the unit's sizes, and below its output size"); + }; + + regs.write(reg::CR1, reg::CR1_WRITE_BACK); + regs.write(reg::CR2, reg::CR2_RECORD_PRIVATE); + regs.write64(reg::STRTAB_BASE, table_base); + regs.write(reg::STRTAB_BASE_CFG, table_cfg); + regs.write64(reg::CMDQ_BASE, commands_base); + regs.write(reg::CMDQ_PROD, 0); + regs.write(reg::CMDQ_CONS, 0); + regs.control(reg::CR0, reg::CR0_CMDQEN, "its command queue enabled"); + let mut live = Live { + regs, + unit, + commands, + queue: window(commands_at, 16 << commands_log2), + prod: 0, + streams, + memory, + routes, + domains: Vec::new(), + reserved, + }; + live.issue(&[Command::ForgetAll, Command::InvalidateAll]); + + regs.write64(reg::EVENTQ_BASE, events_base); + regs.write(reg::EVENTQ_PROD, 0); + regs.write(reg::EVENTQ_CONS, 0); + fault::arm(regs, window(events_at, 32 << events_log2), events, &live.routes); + regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN, "its event queue enabled"); + regs.control(reg::IRQ_CTRL, reg::IRQ_EVENTQ, "its event interrupt enabled"); + regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN | reg::CR0_SMMUEN, "SMMUEN"); + + log!( + "IOMMU: SMMUv3 at {base:#x} armed, CR0ACK {:#x}: {} functions' streams aborting in a table of {}, every \ + other entry invalid; a CMD_SYNC consumed; events on SPI {event}", + regs.read(reg::CR0ACK), + live.routes.len(), + 1u32 << log2, + ); + Some(live) +} + +pub mod interrupt { + use crate::iommu::{Refused, StreamId}; + + pub struct Msi { + pub address: u32, + pub data: u32, + } + + pub struct Pin { + pub low: u32, + pub high: u32, + } + + /// The SMMUv3 remaps no interrupt: a message is an ITS's to translate, + /// and nothing here drives one. + pub fn is_armed() -> bool { + false + } + + pub fn msi(_source: StreamId, _vector: u8, _dest: u32) -> Result { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn claim(_slot: usize, _source: StreamId, _vector: u8) -> Msi { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn release(_slot: usize, _source: StreamId) { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn pin(_apic_id: u8, _vector: u8, _dest: u32, _level: bool) -> Result { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } +} diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs new file mode 100644 index 00000000000..8d34b7774b0 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -0,0 +1,162 @@ +//! `smmu-selftest`: QEMU's `iommu-testdev` (1b36:0005), whose one register +//! read makes a DMA write of a known word at a device address and reads it +//! back at a physical one. Two of them, asked four things of the armed unit: +//! +//! 1. the last one writes on the entry its stream starts with, and is refused +//! with nothing recorded; +//! 2. on a domain of its own, it writes at an address the domain maps, and +//! lands at the memory mapped there and nowhere else; +//! 3. the other, which [`super::init`] gives no route, as it gives none to a +//! function it never enumerated, writes three times under a StreamID +//! inside the stream table, and is refused and counted three times, the +//! third record unwritten, and the machine goes on; +//! 4. once the domain takes its address back, the first writes there again, +//! and is refused and recorded, and the machine halts: the function is +//! one this kernel drives. +//! +//! The writes of 3, and the write of 4, are each made with this CPU's +//! interrupts masked, so one drain reads all of 3's and 4's verdict is on +//! the console before its event is taken, on this CPU, which the event SPI is +//! routed to. Each step says what it saw on its own line; the harness judges +//! them, and the records by the handler's, whose count says the first step +//! recorded nothing. + +use crate::drivers::pci::PciDevice; +use crate::iommu::OwnSpace; +use crate::log; +use crate::mm::policy::MmioPolicy; +use crate::mm::{Mmio, PAGE_2M}; +use crate::time::{Duration, Tripwire}; + +/// `hw/misc/iommu-testdev.h`: Red Hat's vendor and its test device. +const VENDOR: u16 = 0x1b36; +const DEVICE: u16 = 0x0005; + +/// Its BAR0: reading `TRIGGERING` runs an armed DMA; `GVA` is the device +/// address written, `GPA` the physical one read back, `LEN` the bytes, +/// `DBELL` bit 0 arms, and `RESULT` says how it went. +const TRIGGERING: u64 = 0x00; +const GVA_LO: u64 = 0x04; +const GVA_HI: u64 = 0x08; +const LEN: u64 = 0x0c; +const RESULT: u64 = 0x10; +const DBELL: u64 = 0x14; +const GPA_LO: u64 = 0x1c; +const GPA_HI: u64 = 0x20; +const BAR_BYTES: u64 = 0x1000; + +/// `RESULT`: the word landed at `GPA`; the write was refused. +const LANDED: u32 = 0; +const REFUSED: u32 = 0xdead_0002; + +/// How long a refused write's events are given to reach this CPU once it +/// unmasks. +const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( + Duration::from_secs(1), + "the unit records each event and pulses its interrupt while the refused write is still being made", +); + +/// A test device below another: [`super::init`] routes nothing for it, and +/// the stream table, sized to the last one's StreamID, holds its entry. +pub(super) fn stands_unrouted(device: &PciDevice, devices: &[PciDevice]) -> bool { + let at = |d: &PciDevice| (d.bus, d.dev, d.func); + device.is_id(VENDOR, DEVICE) && devices.iter().any(|d| d.is_id(VENDOR, DEVICE) && at(d) > at(device)) +} + +/// `device`'s registers, its memory decode on. No Bus Master Enable: the +/// device writes through the unit's address space whatever its `COMMAND` says. +fn registers(device: &PciDevice) -> Mmio { + let bar = device.memory_bar(0).expect("smmu-selftest: an iommu-testdev's BAR0 is unassigned").address(); + let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); + device.enable_memory_space(); + regs +} + +/// The device's write of four bytes at `at`, read back at `phys`: its `RESULT`. +fn write(regs: Mmio, at: u64, phys: u64) -> u32 { + regs.write_u32(GVA_LO, at as u32); + regs.write_u32(GVA_HI, (at >> 32) as u32); + regs.write_u32(GPA_LO, phys as u32); + regs.write_u32(GPA_HI, (phys >> 32) as u32); + regs.write_u32(LEN, 4); + regs.write_u32(DBELL, 1); + regs.read_u32(TRIGGERING); + regs.read_u32(RESULT) +} + +fn verdict(result: u32, want: u32) -> &'static str { + match (result == want, want) { + (true, REFUSED) => "refused", + (true, _) => "landed there", + (false, _) => "FAIL", + } +} + +pub(super) fn run(devices: &[PciDevice]) { + let mut testdevs = devices.iter().filter(|d| d.is_id(VENDOR, DEVICE)); + let (Some(stray), Some(device), None) = (testdevs.next(), testdevs.next(), testdevs.next()) else { + panic!("smmu-selftest: this machine has not two iommu-testdevs (1b36:0005)"); + }; + let (regs, stray_regs) = (registers(device), registers(stray)); + let page = crate::mm::pmm::alloc_page().expect("smmu-selftest: no page to aim at"); + let phys = page.direct_map().phys(); + let function = crate::iommu::StreamId::pci(device.bus, device.dev, device.func); + let unrouted = crate::iommu::StreamId::pci(stray.bus, stray.dev, stray.func); + + let aborted = write(regs, phys, phys); + log!( + "smmu-selftest: {function}'s write at {phys:#x}, on the entry its stream starts with, answered {aborted:#x}: {}", + verdict(aborted, REFUSED) + ); + + let (space, at) = OwnSpace::create(PAGE_2M).expect("smmu-selftest: a domain on the armed unit"); + space.place(at, phys, PAGE_2M).expect("smmu-selftest: the domain's own first leaf"); + space.attach(device.bus, device.dev, device.func); + let landed = write(regs, at + 0x40, phys + 0x40); + log!( + "smmu-selftest: {function}'s write at {:#x}, mapped to {:#x}, answered {landed:#x}: {}", + at + 0x40, + phys + 0x40, + verdict(landed, LANDED) + ); + space.unmap(at, PAGE_2M).expect("smmu-selftest: the leaf it placed"); + + let before = super::fault::recorded(); + let masked = crate::arch::IrqGuard::close(); + let stray_wrote = [(); 3].map(|()| write(stray_regs, phys, phys)); + for (i, answer) in stray_wrote.into_iter().enumerate() { + log!( + "smmu-selftest: {unrouted}'s write {} at {phys:#x}, under no route, answered {answer:#x}: {}", + i + 1, + verdict(answer, REFUSED) + ); + } + drop(masked); + events_reach_this_cpu(before + 3, "the unrouted function's three refused writes"); + log!("smmu-selftest: the unrouted function's three events were read, and the machine goes on"); + + let masked = crate::arch::IrqGuard::close(); + let taken_back = write(regs, at + 0x40, phys + 0x40); + log!( + "smmu-selftest: {function}'s write at {:#x} again, which its domain no longer maps, answered {taken_back:#x}: {}", + at + 0x40, + verdict(taken_back, REFUSED) + ); + drop(masked); + events_reach_this_cpu(before + 4, "the kernel-driven function's refused write"); + panic!("smmu-selftest: FAIL: the handler read the event of a function this kernel drives and the machine went on"); +} + +/// Spin until the handler has read `want` events in all, which `what` made; +/// `FAIL` past [`EVENT_TIMEOUT`]. +fn events_reach_this_cpu(want: u32, what: &str) { + let deadline = crate::clock::nanos_since_boot() + EVENT_TIMEOUT.nanos(); + while super::fault::recorded() < want { + assert!( + crate::clock::nanos_since_boot() < deadline, + "smmu-selftest: FAIL: the handler had read {} events, not {want}, {EVENT_TIMEOUT} after {what}", + super::fault::recorded() + ); + core::hint::spin_loop(); + } +} diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 2ac9947151c..41881d73319 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -219,6 +219,14 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { storm::sgi(); irqchip::end(intid); } + // Zero until routed, which the kick's arm above takes first. + intid if intid == irqchip::iommu_events() => { + percpu::irq_took(Source::DmaFault); + percpu::preempt_count_up(); + crate::iommu::fault_interrupt(); + percpu::preempt_count_down(); + irqchip::end(intid); + } _ => { percpu::irq_took(Source::Unclaimed); UNCLAIMED.fetch_add(1, Relaxed); diff --git a/kernel/src/arch/x86_64/vtd/domain.rs b/kernel/src/arch/x86_64/vtd/domain.rs index b0027c93edd..35283ea9c3a 100644 --- a/kernel/src/arch/x86_64/vtd/domain.rs +++ b/kernel/src/arch/x86_64/vtd/domain.rs @@ -78,15 +78,14 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { if u32::from(id) >= ceiling { return Err(IommuError::DomainsExhausted(ceiling)); } - let mut domain = Domain::new(&mut TABLES.lock(), id, width, mgaw, &domains.reserved, room)?; - let first = domain.reserve(room).expect("`Domain::new` refuses a domain without the room"); + let (domain, first) = Domain::new(&mut TABLES.lock(), id, width, mgaw, &domains.reserved, room)?; log!( "iommu: domain{id} root={:#x} aw={} mgaw={} addresses from {:#x} to {:#x}", domain.root().phys(), width.bits(), mgaw, - domain.floor(), - domain.ceiling() + domain.window().floor(), + domain.window().ceiling() ); domains.live.push(domain); Ok((DomainId::new(id), first)) @@ -98,7 +97,7 @@ pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { } let mut domains = DOMAINS.lock(); let domain = domains.at(id); - let at = domain.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.ceiling()))?; + let at = domain.window_mut().reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.window().ceiling()))?; let (did, domain) = (domain.id(), *domain); let mut units = UNITS.lock(); table::map(&mut TABLES.lock(), &domain, at, phys, bytes); @@ -138,7 +137,7 @@ pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result = AtomicPtr::new(core::ptr::null_mut()); - -fn functions() -> &'static [Function] { - let published = FUNCTIONS.load(Ordering::Acquire); - if published.is_null() { - return &[]; - } - // SAFETY: non-null only as `describe` stored it: a leaked box, never freed - // or written again, published with Release once it was whole. - unsafe { *published } -} - -/// The first fault this machine took, whole: what a later one says is decided -/// by what the first one already broke. -struct FirstFault { - who: AtomicU32, - address: AtomicU64, - reason: AtomicU32, - unit: AtomicU32, -} - -static FIRST: FirstFault = FirstFault { - who: AtomicU32::new(pci::NO_FUNCTION), - address: AtomicU64::new(0), - reason: AtomicU32::new(0), - unit: AtomicU32::new(0), -}; - -/// Every function this machine enumerated, before any unit is armed: the -/// handler reaches a faulting function's config space through this, with no lock. -pub fn describe(devices: &[PciDevice]) { - let functions: Vec = devices - .iter() - .map(|device| Function { - who: StreamId::pci(device.bus, device.dev, device.func).requester(), - config: DirectMap::phys_of(device.config_window().addr() as *const u8), - domain: AtomicU32::new(0), - faults: AtomicU32::new(0), - user_slot: AtomicU32::new(NO_SLOT), - }) - .collect(); - let functions: &'static [Function] = Box::leak(functions.into_boxed_slice()); - let first = FUNCTIONS.compare_exchange( - core::ptr::null_mut(), - Box::leak(Box::new(functions)), - Ordering::Release, - Ordering::Relaxed, - ); - assert!(first.is_ok(), "iommu: the fault handler's functions were described twice"); -} - -/// Record which domain a function moved to, for the flag the handler sets. -pub fn attached(stream: StreamId, domain: u16) { - if let Some(slot) = find(u32::from(stream.requester())) { - slot.domain.store(u32::from(domain), Ordering::Relaxed); - } -} - -/// Record that a process drives this function on `slot`, or no longer does. -/// -/// The handler reads it to decide what a fault on this stream is *terminal -/// for*: a kernel-driven function has nothing to hand a fault to and the -/// response is the halt below; one a process drives has an owner, so the record -/// goes to that owner's claim and the machine stays up. -pub fn user_owned(stream: StreamId, slot: Option) { - if let Some(function) = find(u32::from(stream.requester())) { - function - .user_slot - .store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); - } -} - -fn find(requester: u32) -> Option<&'static Function> { - functions().iter().find(|f| u32::from(f.who) == requester) -} - /// A unit's fault-record location and count: `CAP.FRO` and `CAP.NFR`. #[derive(Clone, Copy)] pub struct Records { @@ -198,18 +92,12 @@ pub fn arm(index: usize, regs: Mmio, found: Records, vector: u8) { regs.write_u32(FECTL_REG, 0); } -const CONFIG_WINDOW: u64 = crate::mm::PAGE_SIZE; - // The window an armed unit's registers live in, from the address `arm` published. fn window(phys: u64) -> Mmio { - window_of(phys, REGISTER_WINDOW) -} - -fn window_of(phys: u64, size: u64) -> Mmio { // SAFETY: `phys` came from `DirectMap::phys_of` over a window - // `paging::map_mmio` produced at boot and never unmapped, `size` bytes wide; - // this is the same window read back. - unsafe { Mmio::over_phys(DirectMap::from_phys(phys), size) } + // `paging::map_mmio` produced at boot and never unmapped, `REGISTER_WINDOW` + // bytes wide; this is the same window read back. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), REGISTER_WINDOW) } } /// The unit raised its fault event. @@ -227,31 +115,12 @@ pub fn service() { unit.count.load(Ordering::Relaxed), ); } - - if kernel_owned > 0 { - // capture() puts the fault on the panel before the halt takes the - // machine down. - // - // The refusal is the whole response and there is no recovery missing - // from it: a faulting device reached an address this kernel never gave - // it, and nothing here can know what else it already did. **So this - // path halts and never panics** — a report of one carries the fault - // record and `panic_reboot`'s arm line, and no `panicked at` line. - // - // **Only for a stream this kernel drives.** A function a process drives - // has an owner to refuse: its bus mastering is already gone by the time - // this is reached, its claim answers every later call `Io`, and the - // machine — whose other drivers are untouched — goes on. Halting for - // that would be one process's bug taking the whole machine down, which - // is the thing moving a driver out was for. - crate::drivers::panic_console::capture(); - crate::panic::halt_all_cpus(); - } + policy::conclude(kernel_owned); crate::arch::apic::eoi(); } -/// Records drained, of which the answer counts only the ones on a stream this -/// kernel drives — the ones with nobody to hand the fault to. +/// Records drained, of which the answer counts only the ones on an enumerated +/// function this kernel drives — the ones the machine halts on. fn drain(index: usize, regs: Mmio, records: u64, count: u32) -> usize { let status = regs.read_u32(FSTS_REG); if status & FSTS_OVERFLOW != 0 { @@ -275,40 +144,16 @@ fn drain(index: usize, regs: Mmio, records: u64, count: u32) -> usize { (high & 0x7) as u8, ); let reason = ((high >> 32) & 0xFF) as u8; - let address = regs.read_u64(record) & !0xFFF; - // First, before anything that can be slow or say no: a function that - // cannot master the bus raises no second fault, which is the whole of - // the ceiling on a storm. - let stopped = stop(stream); - let seen_here = note(stream); - latch(index, stream, address, reason); - // Before the line, so `owner=` in it is what was actually told. - let owner = owner_of(stream); - if let Some(slot) = owner { - crate::pcidev::note_fault(slot); - } - let count = UNITS[index].faults.fetch_add(1, Ordering::Relaxed) + 1; - log!( - // `owner=` first, because it is the only field that decides whether - // this machine is still running: `tests/common/serial.rs` reads - // `iommu: DMA FAULT owner=kernel` as a death and the other form as - // a record. The reason's name is the last word: every gate takes it - // from there. - "iommu: DMA FAULT owner={} unit{index} stream={stream} addr={address:#018x} \ - access={} reason={reason:#04x} domain={} bme={} unitfaults={count} \ - streamfaults={seen_here} first={} {}", - match owner { - Some(slot) => Owner::Process(slot), - None => Owner::Kernel, - }, - if high & (1u64 << 62) != 0 { "read" } else { "write" }, - blamed(stream), - if stopped { "cleared" } else { "unknown-function" }, - yn(FIRST.who.load(Ordering::Relaxed) == u32::from(stream.requester())), - reason_name(reason), - ); + let fault = Fault { + who: Who::Function(stream), + address: regs.read_u64(record) & !0xFFF, + access: if high & (1u64 << 62) != 0 { Access::Read } else { Access::Write }, + reason, + name: reason_name(reason), + }; + let halts = policy::report(index, &UNITS[index].faults, fault); clear_record(regs, record); - if owner.is_none() { + if halts { seen += 1; } } @@ -320,100 +165,6 @@ fn clear_record(regs: Mmio, record: u64) { regs.write_u32(record + 12, RECORD_FAULT); } -/// The `pcidev` slot a process drives this stream on, or `None` for a stream -/// this kernel drives — including one it never enumerated, which nothing can be -/// handed to either. -fn owner_of(stream: StreamId) -> Option { - match find(u32::from(stream.requester()))?.user_slot.load(Ordering::Acquire) { - NO_SLOT => None, - slot => Some(slot as usize), - } -} - -/// Who a fault was handed to, in the line. A word rather than a number, because -/// which of the two it is decides whether this machine is still running. -enum Owner { - Kernel, - Process(usize), -} - -impl core::fmt::Display for Owner { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Kernel => write!(f, "kernel"), - Self::Process(slot) => write!(f, "slot{slot}"), - } - } -} - -/// Clear Bus Master Enable on whoever faulted; `false` where this machine -/// enumerated no function with that requester id, and the line says so. -fn stop(stream: StreamId) -> bool { - let Some(slot) = find(u32::from(stream.requester())) else { - return false; - }; - pci::stop_bus_mastering(window_of(slot.config, CONFIG_WINDOW)); - true -} - -fn note(stream: StreamId) -> u32 { - find(u32::from(stream.requester())) - .map_or(0, |slot| slot.faults.fetch_add(1, Ordering::Relaxed) + 1) -} - -/// What the handler can say about the faulting function's address space. The -/// middle answer is deliberately weak: a function the kernel never attached -/// carries domain id 0 whether it sits on the identity domain or on one an -/// actuator bound by hand, so the label claims only that nothing recorded one. -enum Blamed { - Own(u32), - Unrecorded, - Unknown, -} - -impl core::fmt::Display for Blamed { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Own(id) => write!(f, "{id}"), - Self::Unrecorded => write!(f, "unrecorded"), - Self::Unknown => write!(f, "unknown"), - } - } -} - -fn blamed(stream: StreamId) -> Blamed { - match find(u32::from(stream.requester())) { - None => Blamed::Unknown, - Some(slot) => match slot.domain.load(Ordering::Relaxed) { - 0 => Blamed::Unrecorded, - id => Blamed::Own(id), - }, - } -} - -/// Take the first fault whole, once: a second finds `who` taken and leaves it. -fn latch(unit: usize, stream: StreamId, address: u64, reason: u8) { - let taken = FIRST.who.compare_exchange( - pci::NO_FUNCTION, - u32::from(stream.requester()), - Ordering::AcqRel, - Ordering::Relaxed, - ); - if taken.is_ok() { - FIRST.address.store(address, Ordering::Relaxed); - FIRST.reason.store(u32::from(reason), Ordering::Relaxed); - FIRST.unit.store(unit as u32, Ordering::Relaxed); - } -} - -fn yn(v: bool) -> char { - if v { - 'y' - } else { - 'n' - } -} - // Only reasons where Linux's dma_remap_fault_reasons and QEMU's VTD_FR_* // agree get a name; 0x0c and 0x0d disagree and are left unnamed. fn reason_name(reason: u8) -> &'static str { diff --git a/kernel/src/arch/x86_64/vtd/mod.rs b/kernel/src/arch/x86_64/vtd/mod.rs index 9dc03c9ca9f..df574044349 100644 --- a/kernel/src/arch/x86_64/vtd/mod.rs +++ b/kernel/src/arch/x86_64/vtd/mod.rs @@ -156,7 +156,7 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) // Before any unit is armed: the handler reaches a faulting function's // config space through this and cannot take a lock to find it. - fault::describe(devices); + crate::iommu::fault::describe(devices); let mut units = 0usize; let mut regions = 0usize; diff --git a/kernel/src/arch/x86_64/vtd/table.rs b/kernel/src/arch/x86_64/vtd/table.rs index 39e2fc2d44a..6b44d87cd7e 100644 --- a/kernel/src/arch/x86_64/vtd/table.rs +++ b/kernel/src/arch/x86_64/vtd/table.rs @@ -7,6 +7,7 @@ use alloc::vec::Vec; +use crate::iommu::window::{self, Window}; use crate::iommu::{AddressWidth, IommuError, Iova, StreamId}; use crate::mm::pmm::{self, PhysPage}; use crate::mm::{DirectMap, Mmio, PAGE_2M}; @@ -218,37 +219,13 @@ pub fn identity_domain(tables: &mut Tables, width: AddressWidth, top: u64) -> (T (root, frames) } -/// One domain's second-level tables, its id, and how far up its addresses have been handed out. +/// One domain's second-level tables, its id, and its addresses. #[derive(Clone, Copy)] pub struct Domain { root: Table, id: u16, width: AddressWidth, - /// Where its addresses start: [`Domain::first_address`] of what its unit - /// translates, which is not [`AddressWidth::bits`] — see - /// [`Domain::translatable_bits`]. - floor: u64, - /// Where they end: [`ceiling`]. - ceiling: u64, - next: u64, -} - -/// Where a domain's addresses end: under what its unit translates, and under -/// the first of `reserved` that reaches above `floor` — a root bridge's window, -/// which a bridge may route peer-to-peer before the unit sees the request -/// (PCIe Base §2.4), or a region firmware reserved (VT-d §3.16). At or below -/// `floor` where one of them covers it. -const fn ceiling(translatable: u8, floor: u64, reserved: &[(u64, u64)]) -> u64 { - let mut ceiling = 1u64 << translatable; - let mut i = 0; - while i < reserved.len() { - let (start, end) = reserved[i]; - if end > floor && start < ceiling { - ceiling = start; - } - i += 1; - } - ceiling + window: Window, } impl Domain { @@ -266,13 +243,8 @@ impl Domain { if mgaw < width.bits() { mgaw } else { width.bits() } } - /// A quarter of the way up what this domain can translate. - const fn first_address(translatable: u8) -> u64 { - 1 << (translatable - 2) - } - - /// A domain with `room` bytes of addresses between its floor and its - /// [`ceiling`], or the reason it has not. + /// A domain with `room` bytes of its [`Window`] handed out at the address + /// answered, or the reason it has not. pub fn new( tables: &mut Tables, id: u16, @@ -280,21 +252,9 @@ impl Domain { mgaw: u8, reserved: &[(u64, u64)], room: u64, - ) -> Result { - let translatable = Self::translatable_bits(width, mgaw); - let floor = Self::first_address(translatable); - // Above memory, so a descriptor still carrying one of these names - // nothing this domain maps and faults rather than landing on a page. - let top = crate::mm::pmm::top(); - if floor <= top { - return Err(IommuError::WindowBelowMemory { translatable, floor, top }); - } - let ceiling = ceiling(translatable, floor, reserved); - // At least one leaf, whatever was asked: a domain with none is no domain. - if ceiling.saturating_sub(floor) < room.next_multiple_of(PAGE_2M).max(PAGE_2M) { - return Err(IommuError::NoRoom { floor, ceiling, room }); - } - Ok(Self { root: tables.alloc(), id, width, floor, ceiling, next: floor }) + ) -> Result<(Self, Iova), IommuError> { + let (window, first) = Window::new(Self::translatable_bits(width, mgaw), reserved, room)?; + Ok((Self { root: tables.alloc(), id, width, window }, first)) } pub fn root(&self) -> Table { @@ -305,101 +265,32 @@ impl Domain { self.id } - pub const fn floor(&self) -> u64 { - self.floor + pub const fn window(&self) -> &Window { + &self.window } - pub const fn ceiling(&self) -> u64 { - self.ceiling - } - - /// Reserve room for `bytes`, rounded up to whole leaves. An address is - /// never handed out twice, unmapped or not: a device holding a stale one - /// would reach whatever took its place. - pub fn reserve(&mut self, bytes: u64) -> Option { - let span = bytes.next_multiple_of(PAGE_2M); - let end = self.next.checked_add(span)?; - if end > self.ceiling() { - return None; - } - let at = Iova::translated(self.next); - self.next = end; - Some(at) - } - - /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: - /// the only room a mapping may be placed in by address. - /// - /// `at` must itself be a leaf `reserve` could have returned, not merely - /// inside a handed-out span: [`map_2m`]'s index floors to the enclosing - /// leaf, so an unaligned `at` this let through would silently place a - /// mapping short of, or overlapping, where the caller named. - pub const fn handed_out(&self, at: Iova, bytes: u64) -> bool { - if !at.raw().is_multiple_of(PAGE_2M) { - return false; - } - let span = bytes.next_multiple_of(PAGE_2M); - if at.raw() < self.floor() { - return false; - } - match at.raw().checked_add(span) { - Some(end) => end <= self.next, - None => false, - } + pub fn window_mut(&mut self) -> &mut Window { + &mut self.window } } -/// [`Domain::handed_out`] is a pure predicate on a `Copy` struct: checked here -/// at compile time rather than under a test harness this no-`std` binary has -/// none of. -const _: () = { - const ROOT: Table = Table { phys: 0 }; - // `translatable = 48` puts `floor()` (a quarter of `1 << 48`) at `1 << 46`, - // itself far past `PAGE_2M`-aligned. - const FLOOR: u64 = Domain::first_address(48); - const ONE_LEAF: Domain = Domain { - root: ROOT, - id: KERNEL_DOMAIN + 1, - width: AddressWidth::Bits48, - floor: FLOOR, - ceiling: 1 << 48, - next: FLOOR + PAGE_2M, - }; - const TWO_LEAVES: Domain = Domain { next: FLOOR + 2 * PAGE_2M, ..ONE_LEAF }; - - // Exactly what one `reserve(PAGE_2M)` handed out. - assert!(ONE_LEAF.handed_out(Iova::translated(FLOOR), PAGE_2M)); - // Short of the floor: nothing this domain has ever reserved. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); - // Past what has been reserved so far. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - // A byte count is rounded up to the leaf it needs, not truncated to fit. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); - // Two leaves handed out; the second is room in its own right. - assert!(TWO_LEAVES.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - // Mid-span but off a leaf boundary: inside the handed-out range without - // being an address `reserve` ever returned. - assert!(!TWO_LEAVES.handed_out(Iova::translated(FLOOR + 1), PAGE_2M)); -}; - -/// [`ceiling`] over the windows the T14's firmware declares, unsorted as it -/// declares them: a 39-bit unit's domain ends where the first window above its -/// floor begins, and a window reaching over the floor leaves it nothing. +/// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the +/// smaller of the two. +/// +/// Asserted here rather than in a guest because no guest holds the shape: QEMU's +/// `intel-iommu` derives both fields from one `aw-bits` property, so its model +/// cannot report a `SAGAW` wider than its `MGAW` at all. const _: () = { - const FLOOR: u64 = Domain::first_address(39); - const T14: [(u64, u64); 7] = [ - (0xA200_0000, 0xBD00_0000), - (0x40_0000_0000, 0x60_3DC0_0000), - (0xA080_0000, 0xA200_0000), - (0xBD00_0000, 0xC000_0000), - (0xFF00_0000, 0xFFB8_0000), - (0xFFD3_A070, 0x1_0000_0000), - (0x60_3DC0_0000, 0x80_0000_0000), - ]; - assert!(FLOOR == 0x20_0000_0000); - assert!(ceiling(39, FLOOR, &T14) == 0x40_0000_0000); - assert!(ceiling(39, FLOOR, &[]) == 1 << 39); - assert!(ceiling(39, FLOOR, &[(0x10_0000_0000, FLOOR + 1)]) < FLOOR); + assert!(Domain::translatable_bits(AddressWidth::Bits48, 39) == 39); + assert!(window::first_address(39) < 1 << 39); + // What a window placed by the table depth alone would be, against the + // ceiling such a unit reports. + assert!(window::first_address(48) >= 1 << 39); + // A unit whose two limits agree is unchanged by any of this. + assert!(Domain::translatable_bits(AddressWidth::Bits48, 48) == 48); + assert!(Domain::translatable_bits(AddressWidth::Bits39, 39) == 39); + // And tables shallower than `MGAW` bind it the other way round. + assert!(Domain::translatable_bits(AddressWidth::Bits39, 48) == 39); }; pub fn map(tables: &mut Tables, domain: &Domain, at: Iova, phys: u64, bytes: u64) { @@ -508,10 +399,11 @@ impl Displaced { } } -/// The entry that moves `requester`'s present entry `old` onto `domain`, and -/// what that leaves the unit holding: the id `old` named, never `domain`'s. -const fn rebind(old: (u64, u64), requester: u16, domain: &Domain) -> ((u64, u64), Displaced) { - let new = context_entry(domain.root, domain.id, domain.width); +/// The entry that moves `requester`'s present entry `old` onto domain `id` +/// at `root`, and what that leaves the unit holding: the id `old` named, never +/// `id`. +const fn rebind(old: (u64, u64), requester: u16, root: Table, id: u16, width: AddressWidth) -> ((u64, u64), Displaced) { + let new = context_entry(root, id, width); (new, Displaced { requester, domain: (old.1 >> 8) as u16 }) } @@ -549,7 +441,7 @@ pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { "iommu: {stream} has no context entry to move — it was not enumerated when its unit \ was programmed" ); - let (new, displaced) = rebind(old, stream.requester(), domain); + let (new, displaced) = rebind(old, stream.requester(), domain.root, domain.id, domain.width); context.replace_pair(index, old, new); displaced } @@ -558,35 +450,8 @@ pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { /// domain's id, the only one its cached entry can match (§6.5.2.1), and its new /// entry names the domain it moved to. const _: () = { - const OWN: Domain = Domain { - root: Table { phys: 0x5000 }, - id: u16::MAX, - width: AddressWidth::Bits39, - floor: 0, - ceiling: 0, - next: 0, - }; let identity = context_entry(Table { phys: 0x3000 }, KERNEL_DOMAIN, AddressWidth::Bits48); - let (new, displaced) = rebind(identity, 0x00F8, &OWN); + let (new, displaced) = rebind(identity, 0x00F8, Table { phys: 0x5000 }, u16::MAX, AddressWidth::Bits39); assert!(displaced.domain == KERNEL_DOMAIN && displaced.requester == 0x00F8); assert!(new.0 == 0x5000 | PRESENT && new.1 == (u16::MAX as u64) << 8 | 1); }; - -/// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the -/// smaller of the two. -/// -/// Asserted here rather than in a guest because no guest holds the shape: QEMU's -/// `intel-iommu` derives both fields from one `aw-bits` property, so its model -/// cannot report a `SAGAW` wider than its `MGAW` at all. -const _: () = { - assert!(Domain::translatable_bits(AddressWidth::Bits48, 39) == 39); - assert!(Domain::first_address(39) < 1 << 39); - // What a window placed by the table depth alone would be, against the - // ceiling such a unit reports. - assert!(Domain::first_address(48) >= 1 << 39); - // A unit whose two limits agree is unchanged by any of this. - assert!(Domain::translatable_bits(AddressWidth::Bits48, 48) == 48); - assert!(Domain::translatable_bits(AddressWidth::Bits39, 39) == 39); - // And tables shallower than `MGAW` bind it the other way round. - assert!(Domain::translatable_bits(AddressWidth::Bits39, 48) == 39); -}; diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index 665088e21bf..adfa2773c63 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -29,9 +29,6 @@ pub const MSIX_ENTRY: u16 = 0; // the message and for the unit to put in an entry. pub(crate) const MSG_DEST: u32 = 0; -/// No requester id: a bus/device/function is sixteen bits, so this is none of them. -pub(crate) const NO_FUNCTION: u32 = u32::MAX; - /// Why a walk of a function's capability list answered no capability. pub enum NoCapability { /// The walk reached the list's terminator and nothing on it carried the id. diff --git a/kernel/src/iommu/fault.rs b/kernel/src/iommu/fault.rs new file mode 100644 index 00000000000..f5504386374 --- /dev/null +++ b/kernel/src/iommu/fault.rs @@ -0,0 +1,320 @@ +//! What a unit's fault handler does with each record it reads, whatever the +//! unit: bounded, allocating nothing and taking no lock. Function state lives +//! in a slice of exactly the enumerated functions, published once before any +//! unit is armed. +//! +//! Whatever the stream, the same things happen first: Bus Master Enable +//! cleared on the function that faulted, the first faulting function latched, +//! and a count kept per unit and per function. Clearing `BME` is also the +//! ceiling on a storm, since a function that cannot master the bus cannot +//! raise a second fault (PCI 3.0 §6.2.2, bit 2 of `COMMAND`). +//! +//! **What differs is who the fault is handed to.** An enumerated function +//! every driver of which is in this kernel has nobody, and its fault is a +//! defect of this kernel, so the terminal action is a halt — the last thing +//! that happens rather than the first. A function a process drives has an +//! owner: `pcidev` is told, that claim refuses every later call, and the +//! machine goes on, because one process's bug taking the machine down is the +//! thing moving a driver out of the kernel was for. A record that names no +//! enumerated function is a device's input this kernel never took on: the +//! unit has already refused it, so it is counted and the machine goes on. +//! Nothing here can clear such a requester's `BME`, so a backend bounds what +//! it reads per interrupt by itself, never by the device, and only a stray's +//! records whose count is a power of two are written, so the device cannot +//! fill the log either. +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; + +use crate::drivers::pci::{self, PciDevice}; +use crate::iommu::StreamId; +use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; + +/// A `pcidev` slot number no claim has: this function is not driven by one. +const NO_SLOT: u32 = u32::MAX; + +/// One enumerated function, published before any unit is armed. +struct Function { + who: StreamId, + /// Physical base of its config window, through which a fault clears `BME`. + config: u64, + domain: AtomicU32, + // Faults a unit has reported against it; non-zero is the per-domain flag. + faults: AtomicU32, + /// The `pcidev` slot a process drives this function on, or [`NO_SLOT`]. + /// What decides whether a fault on it is terminal for the machine. + user_slot: AtomicU32, +} + +/// Exactly the functions this machine enumerated, leaked once by [`describe`]: +/// null until then, and never written again. +static FUNCTIONS: AtomicPtr<&'static [Function]> = AtomicPtr::new(core::ptr::null_mut()); + +fn functions() -> &'static [Function] { + let published = FUNCTIONS.load(Ordering::Acquire); + if published.is_null() { + return &[]; + } + // SAFETY: non-null only as `describe` stored it: a leaked box, never freed + // or written again, published with Release once it was whole. + unsafe { *published } +} + +/// [`Who::key`] of the first fault this machine took: what a later one says +/// is decided by what the first one already broke. +static FIRST: AtomicU64 = AtomicU64::new(u64::MAX); + +/// How many strays — requesters no enumerated function is — are counted +/// apart; every one past them shares the last count. +const STRAYS: usize = 8; + +/// [`Who::key`] plus one of each stray counted apart, in the order each first +/// faulted; zero is free. +static STRAY_KEYS: [AtomicU64; STRAYS] = [const { AtomicU64::new(0) }; STRAYS]; + +/// The records read of each stray in [`STRAY_KEYS`], and of every later one. +static STRAY_FAULTS: [AtomicU32; STRAYS + 1] = [const { AtomicU32::new(0) }; STRAYS + 1]; + +/// This record's number among those of the stray `key`, from 1. +fn stray_fault(key: u64) -> u32 { + let at = STRAY_KEYS + .iter() + .position(|slot| match slot.compare_exchange(0, key + 1, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => true, + Err(held) => held == key + 1, + }) + .unwrap_or(STRAYS); + STRAY_FAULTS[at].fetch_add(1, Ordering::Relaxed) + 1 +} + +/// Every function this machine enumerated, before any unit is armed: the +/// handler reaches a faulting function's config space through this, with no lock. +pub fn describe(devices: &[PciDevice]) { + let functions: Vec = devices + .iter() + .map(|device| Function { + who: StreamId::pci(device.bus, device.dev, device.func), + config: DirectMap::phys_of(device.config_window().addr() as *const u8), + domain: AtomicU32::new(0), + faults: AtomicU32::new(0), + user_slot: AtomicU32::new(NO_SLOT), + }) + .collect(); + let functions: &'static [Function] = Box::leak(functions.into_boxed_slice()); + let first = FUNCTIONS.compare_exchange( + core::ptr::null_mut(), + Box::leak(Box::new(functions)), + Ordering::Release, + Ordering::Relaxed, + ); + assert!(first.is_ok(), "iommu: the fault handler's functions were described twice"); +} + +/// Record which domain a function moved to, for the flag the handler sets. +pub fn attached(stream: StreamId, domain: u16) { + if let Some(function) = find(Who::Function(stream)) { + function.domain.store(u32::from(domain), Ordering::Relaxed); + } +} + +/// Record that a process drives this function on `slot`, or no longer does. +/// +/// The handler reads it to decide what a fault on this stream is *terminal +/// for*: a kernel-driven function has nothing to hand a fault to and the +/// response is the halt in [`conclude`]; one a process drives has an owner, so +/// the record goes to that owner's claim and the machine stays up. +pub fn user_owned(stream: StreamId, slot: Option) { + if let Some(function) = find(Who::Function(stream)) { + function.user_slot.store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); + } +} + +fn find(who: Who) -> Option<&'static Function> { + match who { + Who::Function(stream) => functions().iter().find(|f| f.who == stream), + Who::Stream(_) => None, + } +} + +/// Whom a record names. +#[derive(Clone, Copy)] +pub enum Who { + /// A requester id: VT-d's source-id, or the function the IORT routes an + /// SMMU StreamID from. + Function(StreamId), + /// An SMMU StreamID no enumerated function is routed from. + // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. + #[allow(dead_code)] + Stream(u32), +} + +impl Who { + /// One number per `Who`, none of them `u64::MAX`. + const fn key(self) -> u64 { + match self { + Self::Function(stream) => stream.requester() as u64, + Self::Stream(stream) => 1 << 32 | stream as u64, + } + } +} + +impl core::fmt::Display for Who { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Function(stream) => write!(f, "{stream}"), + Self::Stream(stream) => write!(f, "{stream:#x}"), + } + } +} + +/// What the faulting transaction was, where the record says. +#[derive(Clone, Copy)] +pub enum Access { + Read, + Write, + /// An SMMU event that carries no transaction. + // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. + #[allow(dead_code)] + Unrecorded, +} + +impl core::fmt::Display for Access { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(match self { + Self::Read => "read", + Self::Write => "write", + Self::Unrecorded => "none", + }) + } +} + +/// One record a unit reported, decoded by its backend. +pub struct Fault { + pub who: Who, + pub address: u64, + pub access: Access, + /// The unit's own number for why, and its name. + pub reason: u8, + pub name: &'static str, +} + +/// `fault`, read off unit `unit`, whose count of reported faults is `count`: +/// its function stopped, it is handed to whoever drives that function, and +/// its line written, a stray's only at a power of two. `true` only for an enumerated function this kernel +/// drives, which the drain ends on with [`conclude`]. +pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { + let function = find(fault.who); + // First, before anything that can be slow or say no: a function that + // cannot master the bus raises no second fault, which is the whole of + // the ceiling on a storm. + if let Some(function) = function { + pci::stop_bus_mastering(config_window(function.config)); + } + let key = fault.who.key(); + let seen_here = match function { + Some(function) => function.faults.fetch_add(1, Ordering::Relaxed) + 1, + None => stray_fault(key), + }; + let _ = FIRST.compare_exchange(u64::MAX, key, Ordering::AcqRel, Ordering::Relaxed); + // Before the line, so `owner=` in it is what was actually told. + let owner = match function.map(|f| f.user_slot.load(Ordering::Acquire)) { + None => Owner::Nobody, + Some(NO_SLOT) => Owner::Kernel, + Some(slot) => Owner::Slot(slot as usize), + }; + if let Owner::Slot(slot) = owner { + crate::pcidev::note_fault(slot); + } + let count = count.fetch_add(1, Ordering::Relaxed) + 1; + // A stray's line is bounded by its count, never by the device: the next + // line it writes says how many were read in between. + if function.is_none() && !seen_here.is_power_of_two() { + return false; + } + log!( + // `owner=` first, because it is the only field that decides whether + // this machine is still running: `tests/common/serial.rs` reads + // `iommu: DMA FAULT owner=kernel` as a death and the other forms as + // records. The reason's name is the last word: every gate takes it + // from there. + "iommu: DMA FAULT owner={} unit{unit} stream={} addr={:#018x} access={} reason={:#04x} \ + domain={} bme={} unitfaults={count} streamfaults={seen_here} first={} {}", + owner, + fault.who, + fault.address, + fault.access, + fault.reason, + Blamed(function), + if function.is_some() { "cleared" } else { "unknown-function" }, + if FIRST.load(Ordering::Relaxed) == key { 'y' } else { 'n' }, + fault.name, + ); + matches!(owner, Owner::Kernel) +} + +/// The end of a drain that read `kernel_owned` faults on functions this kernel +/// drives: any at all halts the machine. +/// +/// The halt is the whole response and there is no recovery missing from it: a +/// faulting device reached an address this kernel never gave it, and nothing +/// here can know what else it already did. **So this halts and never panics** +/// — a report of one carries the fault record and `panic_reboot`'s arm line, +/// and no `panicked at` line; `capture` puts the fault on the panel first. +/// +/// **Only for a function this kernel drives.** A function a process drives has +/// an owner to refuse: its bus mastering is already gone by the time this is +/// reached, its claim answers every later call `Io`, and the machine — whose +/// other drivers are untouched — goes on. A requester nobody enumerated has +/// no driver here to be wrong, and the unit already refused what it sent. +pub fn conclude(kernel_owned: usize) { + if kernel_owned > 0 { + crate::drivers::panic_console::capture(); + crate::panic::halt_all_cpus(); + } +} + +/// The config window of the function at `phys`, from the address [`describe`] +/// published. +fn config_window(phys: u64) -> Mmio { + // SAFETY: `phys` is `DirectMap::phys_of` a config window the ECAM mapping + // holds for the machine's life, one page wide. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), PAGE_SIZE) } +} + +/// Who a fault was handed to, in the line. A word rather than a number, because +/// which of them it is decides whether this machine is still running. +#[derive(Clone, Copy)] +enum Owner { + /// An enumerated function only this kernel drives: the halt. + Kernel, + /// An enumerated function the process on this `pcidev` slot drives. + Slot(usize), + /// No enumerated function: a requester this kernel never took on. + Nobody, +} + +impl core::fmt::Display for Owner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Kernel => write!(f, "kernel"), + Self::Slot(slot) => write!(f, "slot{slot}"), + Self::Nobody => write!(f, "none"), + } + } +} + +/// What the handler can say about the faulting function's address space. The +/// middle answer is deliberately weak: a function the kernel never attached +/// carries domain id 0 whether it sits on the unit's default or on a domain an +/// actuator bound by hand, so the label claims only that nothing recorded one. +struct Blamed(Option<&'static Function>); + +impl core::fmt::Display for Blamed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0.map(|function| function.domain.load(Ordering::Relaxed)) { + None => write!(f, "unknown"), + Some(0) => write!(f, "unrecorded"), + Some(id) => write!(f, "{id}"), + } + } +} diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 3bcf02d17d1..003e4cd1423 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -1,6 +1,6 @@ //! The unit that decides what a device may reach. //! -//! Inventories the machine's IOMMU units, gives every enumerated PCI function an identity-mapped context entry, turns translation on, remaps every interrupt source through a source-id-verified table entry, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above `vtd/` stay backend-neutral so a second backend drops in without moving the seam. +//! Inventories the machine's IOMMU units, turns translation on, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above the backend stay backend-neutral, and what every backend decides alike is declared here once: a domain's addresses ([`window`]) and what a fault record ends ([`fault`]). //! //! The refusal is deliberately not yet built for a driver in this kernel: landing it before any userspace driver exists would cost every machine and protect nothing. A function a *process* drives is the other case and is refused ([`OwnSpace`], [`remapping`]), because a descriptor it writes a physical address into is an arbitrary read and write over all of memory, and a message it sends unremapped is any vector at any CPU; its message is its claim slot's own entry ([`Remapped`]). //! @@ -11,6 +11,9 @@ use crate::arch::iommu_unit as unit; +pub(crate) mod fault; +pub(crate) mod window; + /// The address width a device's translations cover. /// /// `Bits57` is omitted even where a unit advertises it, because it needs a fifth page-table level no machine in reach uses. @@ -428,7 +431,7 @@ pub fn remap_pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Delivery) { - unit::fault::user_owned(StreamId::pci(bus, device, function), slot); + fault::user_owned(StreamId::pci(bus, device, function), slot); } /// Reached from the IDT gate the unit's own `FEDATA` names. diff --git a/kernel/src/iommu/window.rs b/kernel/src/iommu/window.rs new file mode 100644 index 00000000000..74f31029fcc --- /dev/null +++ b/kernel/src/iommu/window.rs @@ -0,0 +1,156 @@ +//! A domain's device addresses — where they start, where they end and which +//! of them are handed out — for every backend. +//! +//! They start a quarter of the way up what the unit translates and above all +//! memory, so a descriptor still carrying one names nothing a domain maps and +//! faults rather than landing on a page. They end under the first root-bridge +//! window or reserved region over that start: a bridge may route a window +//! peer-to-peer before the unit sees the request (PCIe Base §2.4), and a +//! region firmware reserved is not a domain's (VT-d §3.16). An address is +//! handed out once, unmapped or not: a device holding a stale one would reach +//! whatever took its place. + +use crate::iommu::{IommuError, Iova}; +use crate::mm::PAGE_2M; + +/// One domain's device addresses. +#[derive(Clone, Copy)] +pub struct Window { + floor: u64, + ceiling: u64, + /// The first not yet handed out. + next: u64, +} + +/// A quarter of the way up `translatable` bits of device address. +pub const fn first_address(translatable: u8) -> u64 { + 1 << (translatable - 2) +} + +/// Where a window over `translatable` bits starting at `floor` ends: under the +/// first of `reserved` that reaches above `floor`; at or below `floor` where +/// one of them covers it. +const fn ceiling(translatable: u8, floor: u64, reserved: &[(u64, u64)]) -> u64 { + let mut ceiling = 1u64 << translatable; + let mut i = 0; + while i < reserved.len() { + let (start, end) = reserved[i]; + if end > floor && start < ceiling { + ceiling = start; + } + i += 1; + } + ceiling +} + +impl Window { + /// A new domain's window over `translatable` bits, clear of `reserved`, + /// with `room` bytes of it handed out first; refused where it would start + /// at or below memory or hold less than that, and at least one leaf. + pub fn new(translatable: u8, reserved: &[(u64, u64)], room: u64) -> Result<(Self, Iova), IommuError> { + let floor = first_address(translatable); + let top = crate::mm::pmm::top(); + if floor <= top { + return Err(IommuError::WindowBelowMemory { translatable, floor, top }); + } + let ceiling = ceiling(translatable, floor, reserved); + let mut window = Self::starting_at(floor, ceiling); + match window.reserve(room) { + Some(first) if ceiling.saturating_sub(floor) >= PAGE_2M => Ok((window, first)), + _ => Err(IommuError::NoRoom { floor, ceiling, room }), + } + } + + const fn starting_at(floor: u64, ceiling: u64) -> Self { + Self { floor, ceiling, next: floor } + } + + pub const fn floor(&self) -> u64 { + self.floor + } + + pub const fn ceiling(&self) -> u64 { + self.ceiling + } + + /// `bytes`, rounded up to whole 2 MiB leaves, of addresses never handed out. + pub const fn reserve(&mut self, bytes: u64) -> Option { + let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return None }; + let Some(end) = self.next.checked_add(span) else { return None }; + if end > self.ceiling { + return None; + } + let at = Iova::translated(self.next); + self.next = end; + Some(at) + } + + /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: + /// the only room a mapping may be placed in by address. + /// + /// `at` must itself be a leaf `reserve` could have returned, not merely + /// inside a handed-out span: a leaf's index floors to the enclosing leaf, + /// so an unaligned `at` this let through would place a mapping short of, + /// or overlapping, where the caller named. + pub const fn handed_out(&self, at: Iova, bytes: u64) -> bool { + if !at.raw().is_multiple_of(PAGE_2M) || at.raw() < self.floor { + return false; + } + let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return false }; + match at.raw().checked_add(span) { + Some(end) => end <= self.next, + None => false, + } + } +} + +/// [`Window`] and [`ceiling`] at their boundaries, at compile time: the +/// kernel binary has no test harness. +const _: () = { + const FLOOR: u64 = first_address(48); + let mut one = Window::starting_at(FLOOR, FLOOR + 2 * PAGE_2M); + assert!(matches!(one.reserve(1), Some(at) if at.raw() == FLOOR)); + // Exactly what one `reserve` handed out. + assert!(one.handed_out(Iova::translated(FLOOR), PAGE_2M)); + // Short of the floor, past what has been reserved, rounded up past it, + // off a leaf boundary, and past the end of the address space: nothing. + assert!(!one.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); + assert!(!one.handed_out(Iova::translated(FLOOR + 1), 0)); + assert!(!one.handed_out(Iova::translated(u64::MAX - PAGE_2M + 1), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR), u64::MAX)); + // Past the ceiling, and past the end of the address space, nothing. + assert!(one.reserve(PAGE_2M + 1).is_none()); + assert!(one.reserve(u64::MAX).is_none()); + assert!(matches!(one.reserve(PAGE_2M), Some(at) if at.raw() == FLOOR + PAGE_2M)); + // Two leaves handed out; the second is room in its own right, and a + // mid-span address off a leaf boundary is not. + assert!(one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR + 1), PAGE_2M)); + assert!(one.reserve(1).is_none()); +}; + +/// [`ceiling`] over the windows the T14's firmware declares, unsorted as it +/// declares them, and over `virt`'s: a 39-bit unit's window ends where the +/// first root-bridge window above its floor begins, one reaching over the +/// floor leaves it nothing, and windows all below the floor leave the whole +/// input. +const _: () = { + const FLOOR: u64 = first_address(39); + const T14: [(u64, u64); 7] = [ + (0xA200_0000, 0xBD00_0000), + (0x40_0000_0000, 0x60_3DC0_0000), + (0xA080_0000, 0xA200_0000), + (0xBD00_0000, 0xC000_0000), + (0xFF00_0000, 0xFFB8_0000), + (0xFFD3_A070, 0x1_0000_0000), + (0x60_3DC0_0000, 0x80_0000_0000), + ]; + assert!(FLOOR == 0x20_0000_0000); + assert!(ceiling(39, FLOOR, &T14) == 0x40_0000_0000); + assert!(ceiling(39, FLOOR, &[]) == 1 << 39); + assert!(ceiling(39, FLOOR, &[(0x10_0000_0000, FLOOR + 1)]) < FLOOR); + const VIRT: [(u64, u64); 2] = [(0x1000_0000, 0x3f00_0000), (0x80_0000_0000, 0x100_0000_0000)]; + assert!(ceiling(48, first_address(48), &VIRT) == 1 << 48); +}; diff --git a/tests/checks/serial.rs b/tests/checks/serial.rs index cba7757dcbc..d5d9091cacd 100644 --- a/tests/checks/serial.rs +++ b/tests/checks/serial.rs @@ -72,6 +72,15 @@ pub fn self_check() -> Result<(), String> { ) .must_be_clean() }), + ("must_be_clean on a DMA fault naming no enumerated function", false, &|| { + Serial::named( + "test capture", + "[ 0.001 cpu0 kernel] NVMe: found\n[ 4.100 cpu0 kernel] iommu: DMA FAULT \ + owner=none unit0 stream=0x18 addr=0x0000000000000000 access=none reason=0x04 \ + domain=unknown bme=unknown-function unitfaults=1 streamfaults=1 first=y C_BAD_STE\n", + ) + .must_be_clean() + }), ("must_be_clean on an empty capture", false, &|| dead.must_be_clean()), ]; @@ -168,6 +177,14 @@ pub fn self_check() -> Result<(), String> { addr=0x1000 access=read reason=0x06 read-permission", None, ), + // And on a requester no enumerated function is: the unit refused it, + // and the machine goes on. + ( + "[ 4.100 cpu0 kernel] iommu: DMA FAULT owner=none unit0 stream=00:1f.7 \ + addr=0x1000 access=write reason=0x02 domain=unknown bme=unknown-function \ + context-entry-not-present", + None, + ), ("[ 0.001 cpu0 kernel] EARLY PANIC: nothing is up yet", Some(Died::Kernel)), ( "[ 2.000 cpu1 kernel] DOUBLE PANIC: the cpu was already in Fatal; first: invalid \ diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 6df69de4ce2..d5138775c97 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -727,13 +727,18 @@ pub enum Profile { /// `HVC`, as under HVF. `virt_el1_smp`'s machine while a boot's last word /// can miss the console under HVF. VirtTcg, + /// [`Profile::Virt`] with its SMMUv3 and two of QEMU's `iommu-testdev`, a + /// function that writes where it is told to through the unit. + VirtSmmu, } impl Profile { /// The architecture this machine is. pub fn arch(self) -> Arch { match self { - Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg => Arch::Aarch64, + Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg | Self::VirtSmmu => { + Arch::Aarch64 + } Self::Headless | Self::HeadlessNoIommu | Self::HeadlessE1000e @@ -855,6 +860,8 @@ struct Shape { /// profile because absence is a shape and because the unit's own /// capabilities are what the kernel reads at boot. iommu: Option, + /// `virt`'s SMMUv3, which is a machine property rather than a device. + smmu: Smmu, /// A virtio-rng, which is firmware's alone: edk2's driver puts /// `EFI_RNG_PROTOCOL` behind it for the loader's seed, and the kernel /// drives no such device. `virt` has it because an HVF guest's CPU has no @@ -863,6 +870,15 @@ struct Shape { rng: bool, } +/// Whether `virt` has its SMMUv3. With it come two of QEMU's `iommu-testdev`, +/// whose writes are the only DMA a guest of this suite can aim at an address +/// of its choosing: a unit no function writes through is a unit no test reads. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Smmu { + Absent, + WithTestdev, +} + /// Where a machine's image and its DATA are. A size is stated because a /// structure sized per device block is bounded by it and by nothing else; the /// backing file is sparse either way. @@ -907,6 +923,7 @@ impl Profile { match self { Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg => Self::Virt.shape(), Self::VirtNoRng => Shape { rng: false, ..Self::Virt.shape() }, + Self::VirtSmmu => Shape { smmu: Smmu::WithTestdev, ..Self::Virt.shape() }, Self::Virt => Shape { vga: "std", panel: None, @@ -916,6 +933,7 @@ impl Profile { usb: &[], storage: Storage::Stick { nvme_bytes: 0 }, iommu: None, + smmu: Smmu::Absent, rng: true, }, Self::Headless => Shape { @@ -927,6 +945,7 @@ impl Profile { usb: &["usb-kbd,bus=xhci.0"], storage: Storage::Disk { data_bytes: NVME_SMALL }, iommu: Some(IOMMU_DEFAULT), + smmu: Smmu::Absent, rng: false, }, Self::Metal => Shape { @@ -942,6 +961,7 @@ impl Profile { usb: &[], storage: Storage::Stick { nvme_bytes: NVME_SMALL }, iommu: Some(IOMMU_DEFAULT), + smmu: Smmu::Absent, rng: false, }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, @@ -2056,15 +2076,22 @@ fn qemu_command( // needs the userspace half of the irqchip, and a machine with no unit has // no reason to be built differently from the one it has always been. let mut machine = match arch { - Arch::X86_64 => arch.machine().to_string(), + Arch::X86_64 => { + assert!(shape.smmu == Smmu::Absent, "a q35 has no SMMUv3"); + arch.machine().to_string() + } Arch::Aarch64 => { // The unit a profile declares is VT-d, which `virt` has none of. assert!(shape.iommu.is_none(), "`virt` has no VT-d"); - match options.profile { + let machine = match options.profile { Profile::VirtEl2 | Profile::VirtEl2NoVhe => { format!("{},gic-version=3,virtualization=on", arch.machine()) } _ => format!("{},gic-version=3", arch.machine()), + }; + match shape.smmu { + Smmu::Absent => machine, + Smmu::WithTestdev => format!("{machine},iommu=smmuv3"), } } }; @@ -2184,6 +2211,12 @@ fn qemu_command( if shape.rng { qemu.arg("-device").arg("virtio-rng-pci"); } + if shape.smmu == Smmu::WithTestdev { + // Two, the first enumerated below the second: the kernel's selftest + // routes nothing for the first, whose entry the table holds. + qemu.arg("-device").arg("iommu-testdev"); + qemu.arg("-device").arg("iommu-testdev"); + } // The NIC before the virtio block, so a profile that has one and not the // other still creates it after the unit and before everything else. diff --git a/tests/common/serial.rs b/tests/common/serial.rs index cd587c8bce1..a726f90ce09 100644 --- a/tests/common/serial.rs +++ b/tests/common/serial.rs @@ -96,10 +96,11 @@ pub(crate) const DEATHS: &[(&str, Died, Died)] = &[ // `machine_check_handler`, the one exception a Ring 3 frame does not make // the process's fault. Also `-> !`. ("MACHINE CHECK", Died::Kernel, Died::Kernel), - // kernel/src/arch/x86_64/vtd/fault.rs — a fault on a stream this kernel drives - // has nobody to hand it to, so the handler halts. One a *process* drives - // says `owner=slot` and the machine goes on, which is why the needle is - // the owner rather than the fault. + // kernel/src/iommu/fault.rs — a fault on a function this kernel drives has + // nobody to hand it to, so the handler halts. One a *process* drives says + // `owner=slot`, and one naming no enumerated function `owner=none`, and + // the machine goes on, which is why the needle is the owner rather than + // the fault. ("iommu: DMA FAULT owner=kernel", Died::Kernel, Died::Kernel), // kernel/src/main.rs — a panic that landed on a CPU already inside a fault // or a report. The rest of the line is `panic::last_words`: which of the @@ -348,12 +349,15 @@ impl Serial { /// all — so the only capture allowed to hold one is the capture of the test /// that staged it, and every other boot in the estate reds. const NEVER_CLEAN: &[&str] = &[ - // kernel/src/arch/x86_64/vtd/fault.rs — a function a *process* drives reached an + // kernel/src/iommu/fault.rs — a function a *process* drives reached an // address its own domain does not map. The machine goes on and the claim // refuses every later call, so this is not a death; it is a driver whose // descriptors are wrong, and a netstack that did it on every boot would // otherwise pass everywhere. "iommu: DMA FAULT owner=slot", + // The same file — a requester no enumerated function is: refused by the + // unit, and on a boot that staged none, a device nothing here accounts for. + "iommu: DMA FAULT owner=none", // kernel/src/log/console.rs — the stop waited out its whole budget for // `klogd` to let the console's wire go. super::power::WIRE_LATE, diff --git a/tests/toyos.rs b/tests/toyos.rs index d795f879202..159a5ed1bcd 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -287,6 +287,7 @@ const SCREEN_TESTS: &[(&str, qemu::Profile)] = &[ // and FP, which no boot under HVF runs. ("virt_jobs_at_el2", qemu::Profile::VirtEl2), ("virt_random_differs", qemu::Profile::Virt), + ("virt_smmu", qemu::Profile::VirtSmmu), ("virt_no_seed_refused", qemu::Profile::VirtNoRng), ]; @@ -2036,6 +2037,145 @@ fn virt_no_seed_refused(profile: qemu::Profile, test_config: &Path) -> Result<() Ok(()) } +/// The SMMUv3 armed from the IORT, judged by what two of QEMU's +/// `iommu-testdev` can and cannot write through it +/// (`kernel/src/arch/aarch64/smmu/selftest.rs`): `GBPA` read back aborting, +/// a `CMD_SYNC` consumed; one device's write refused on the entry its stream +/// starts with and nothing recorded, and landing where its own domain maps +/// it; the other's three, under a StreamID no function is routed from inside +/// the stream table, refused, the first two recorded as no function's and the +/// third counted and not written, and the machine going on; and the first's again once its domain takes that address back, +/// refused, recorded as the kernel's, and the machine halted on it. Each +/// record taken on the wired SPI and named. Read to the halt path's own line +/// past the stop, so nothing the guest could still say is missed. +fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { + const FAULT: &str = "iommu: DMA FAULT owner="; + const WENT_ON: &str = "smmu-selftest: the unrouted function's three events were read, and the machine goes on"; + // `panic_reboot::arm`'s line, which `halt_all_cpus` writes once every + // other CPU is stopped: one of its two heads. + let halted = |l: &str| l.contains("panic: rebooting") || l.contains("panic: holding this panel"); + let options = BootOptions { + profile, + kernel_params: &["smmu-selftest"], + ready_marker: "control registers: SCTLR_EL1=", + ..Default::default() + }; + let argv = qemu::profile_argv(&options); + for want in ["iommu=smmuv3", "iommu-testdev"] { + if !argv.iter().any(|a| a.contains(want)) { + return Err(format!("the machine has no {want}: {argv:?}")); + } + } + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[], options); + let rest = qemu.drain_until(Duration::from_secs(30), halted); + let serial = format!("{}\n{rest}", qemu.boot_log()); + let lines: Vec<&str> = serial.lines().collect(); + let at = |want: &str| lines.iter().position(|l| l.contains(want)); + let line = |want: &str| at(want).map(|i| lines[i].to_string()); + let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); + if let Some(failed) = line("smmu-selftest: FAIL") { + return Err(format!("{failed}\nserial:\n{serial}")); + } + let Some(stop) = lines.iter().position(|l| halted(l)) else { + return Err(format!("the machine never reached the halt path's line past the stop\nserial:\n{serial}")); + }; + let Some(armed) = line("armed, CR0ACK") else { + return Err(format!("the SMMUv3 was never armed\nserial:\n{serial}")); + }; + for want in ["every other entry invalid", "a CMD_SYNC consumed", "events on SPI 106"] { + if !armed.contains(want) { + return Err(format!("{want:?} not in {armed:?}")); + } + } + eprintln!(" [virt] {armed}"); + let table = armed.split_once("in a table of ").and_then(|(_, rest)| rest.split(',').next()?.parse::().ok()); + let Some(table) = table else { + return Err(format!("{armed:?} names no table size")); + }; + // `SMMU_GBPA.ABORT`, bit 20, as the unit reads it back. + let gbpa = line(", every transaction aborts while SMMUEN is clear") + .and_then(|l| l.split_once(": GBPA ").and_then(|(_, v)| hex(v.split(',').next()?))); + match gbpa { + Some(gbpa) if gbpa & 1 << 20 != 0 => eprintln!(" [virt] GBPA {gbpa:#x}: ABORT"), + _ => return Err(format!("GBPA does not read back aborting: {gbpa:?}\nserial:\n{serial}")), + } + let Some(unrouted) = line("is given no route") else { + return Err(format!("the selftest left no function unrouted\nserial:\n{serial}")); + }; + eprintln!(" [virt] {unrouted}"); + let Some(stream) = unrouted.split_once(", StreamID ").and_then(|(_, rest)| hex(rest.split(',').next()?)) else { + return Err(format!("{unrouted:?} names no StreamID")); + }; + if stream >= table { + return Err(format!("StreamID {stream:#x} is past a table of {table}: {unrouted:?}")); + } + for (said, verdict) in [ + ("on the entry its stream starts with", "refused"), + ("mapped to", "landed there"), + ("'s write 1 at ", "refused"), + ("'s write 2 at ", "refused"), + ("'s write 3 at ", "refused"), + ("which its domain no longer maps", "refused"), + ] { + let Some(found) = line(said) else { + return Err(format!("the selftest never said {said:?}\nserial:\n{serial}")); + }; + eprintln!(" [virt] {found}"); + if !found.ends_with(verdict) { + return Err(format!("{found}\nserial:\n{serial}")); + } + } + let Some(again) = line("which its domain no longer maps") else { + return Err(format!("the selftest never wrote where its domain took a mapping back\nserial:\n{serial}")); + }; + let aimed = again.split_once("smmu-selftest: ").and_then(|(_, said)| said.split_once("'s write at ")); + let Some((function, Some(address))) = + aimed.map(|(function, rest)| (function, rest.split(' ').next().and_then(hex))) + else { + return Err(format!("{again:?} names no function and address")); + }; + let faults: Vec = (0..lines.len()).filter(|&i| lines[i].contains(FAULT)).collect(); + for &fault in &faults { + eprintln!(" [virt] {}", lines[fault]); + } + // The unit's first four events, so the write on the entry its stream + // starts with was not recorded; the stray's third is counted and, not + // being a power of two, not written. + let wants = [ + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=1 streamfaults=1 ", "C_BAD_STE"), + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=2 streamfaults=2 ", "C_BAD_STE"), + ( + format!("owner=kernel unit0 stream={function} addr={address:#018x} access=write"), + " unitfaults=4 ", + "F_TRANSLATION", + ), + ]; + if faults.len() != wants.len() { + return Err(format!("{} events reached the handler, not {}\nserial:\n{serial}", faults.len(), wants.len())); + } + for (&fault, (named, count, name)) in faults.iter().zip(&wants) { + let fault = lines[fault]; + if !fault.contains(named.as_str()) || !fault.contains(count) || !fault.ends_with(name) { + return Err(format!("the event does not name {named:?},{count}and {name}: {fault}")); + } + } + // The machine went on past the unrouted records and stopped at the + // kernel's: the selftest's line between them, and the stop after the last. + let went_on = at(WENT_ON).ok_or_else(|| format!("the selftest never said {WENT_ON:?}\nserial:\n{serial}"))?; + if !(faults[1] < went_on && went_on < faults[2] && faults[2] < stop) { + return Err(format!( + "the unrouted records, the selftest going on, the kernel's record and the stop are out of order: \ + lines {}, {went_on}, {} and {stop}\nserial:\n{serial}", + faults[1], faults[2] + )); + } + // A halt, which writes the record and the stop's line and no panic. + if let Some(death) = lines.iter().find(|l| serial::died(l).is_some() && !l.contains("owner=kernel")) { + return Err(format!("the machine died of something other than the record: {death}\nserial:\n{serial}")); + } + Ok(()) +} + /// Everything the PL011 has carried on `qemu`'s boot: the capture each /// [`judge_virt_job`] after the first goes on from, since a drain reads past /// the marker it waited for. @@ -2766,6 +2906,7 @@ fn run_screen_test(name: &str, profile: qemu::Profile, test_config: &Path) -> Re "virt_jobs_at_el2" => virt_jobs_at_el2(profile), "virt_random_differs" => virt_random_differs(profile), "virt_no_seed_refused" => virt_no_seed_refused(profile, test_config), + "virt_smmu" => virt_smmu(profile, test_config), "virt_mask_windows" => virt_mask_windows(profile), "virt_irq_storm" => { // The CPU floods itself with SGIs until the timer has fired a diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs index dbfad87ce37..122892e9f68 100644 --- a/toyos-smmu/src/queue.rs +++ b/toyos-smmu/src/queue.rs @@ -74,6 +74,11 @@ const OVERFLOW: u32 = 1 << 31; pub struct Events(pub(crate) Indexes); impl Events { + /// The records a full queue holds. + pub const fn entries(self) -> u32 { + 1 << self.0.log2size + } + /// The entry the next record is read from. pub const fn slot(self, cons: u32) -> usize { self.0.slot(cons) @@ -194,6 +199,34 @@ pub enum Code { Other(u8), } +impl Code { + /// The event's mnemonic, as §7.3's headings name it. + pub const fn name(self) -> &'static str { + match self { + Self::UnsupportedTransaction => "F_UUT", + Self::BadStream => "C_BAD_STREAMID", + Self::EntryFetch => "F_STE_FETCH", + Self::BadEntry => "C_BAD_STE", + Self::AtsRequest => "F_BAD_ATS_TREQ", + Self::StreamDisabled => "F_STREAM_DISABLED", + Self::TranslatedForbidden => "F_TRANSL_FORBIDDEN", + Self::BadSubstream => "C_BAD_SUBSTREAMID", + Self::ContextFetch => "F_CD_FETCH", + Self::BadContext => "C_BAD_CD", + Self::WalkAbort => "F_WALK_EABT", + Self::Translation => "F_TRANSLATION", + Self::AddressSize => "F_ADDR_SIZE", + Self::AccessFlag => "F_ACCESS", + Self::Permission => "F_PERMISSION", + Self::TlbConflict => "F_TLB_CONFLICT", + Self::ConfigurationConflict => "F_CFG_CONFLICT", + Self::PageRequest => "E_PAGE_REQUEST", + Self::VmsFetch => "F_VMS_FETCH", + Self::Other(_) => "unnamed", + } + } +} + /// What a device tried, where the record says: the four translation-related /// faults carry it (§7.3.13 to §7.3.16). #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -209,6 +242,8 @@ pub struct Attempt { pub struct Event { /// `StreamID`, bits [63:32]. pub stream: u32, + /// The event's number, bits [7:0], which `code` names. + pub number: u8, pub code: Code, pub attempt: Option, } @@ -242,5 +277,5 @@ pub const fn event(record: [u64; 4]) -> Event { } _ => None, }; - Event { stream: (record[0] >> 32) as u32, code, attempt } + Event { stream: (record[0] >> 32) as u32, number: record[0] as u8, code, attempt } } diff --git a/toyos-smmu/src/unit.rs b/toyos-smmu/src/unit.rs index b706371d68d..f6ffe05782b 100644 --- a/toyos-smmu/src/unit.rs +++ b/toyos-smmu/src/unit.rs @@ -20,17 +20,24 @@ pub const IRQ_CTRL: usize = 0x050; pub const IRQ_CTRLACK: usize = 0x054; pub const GERROR: usize = 0x060; pub const GERRORN: usize = 0x064; +pub const GERROR_IRQ_CFG0: usize = 0x068; pub const STRTAB_BASE: usize = 0x080; pub const STRTAB_BASE_CFG: usize = 0x088; pub const CMDQ_BASE: usize = 0x090; pub const CMDQ_PROD: usize = 0x098; pub const CMDQ_CONS: usize = 0x09C; pub const EVENTQ_BASE: usize = 0x0A0; +pub const EVENTQ_IRQ_CFG0: usize = 0x0B0; +/// Present only where [`IDR0_PRI`] is set (§6.3.34). +pub const PRIQ_IRQ_CFG0: usize = 0x0D0; /// The event queue's indexes are in register page 1, 64 KiB above page 0 /// (§6.1, §6.2.2): `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS` at 0xA8 and 0xAC of it. pub const EVENTQ_PROD: usize = 0x1_00A8; pub const EVENTQ_CONS: usize = 0x1_00AC; +/// `SMMU_IDR0.PRI` [16] (§6.3.1): the unit has a PRI queue, and its registers. +pub const IDR0_PRI: u32 = 1 << 16; + /// `SMMU_CR0` (§6.3.9), and `SMMU_CR0ACK`, which reads each bit back once /// the unit has acted on it. pub const CR0_SMMUEN: u32 = 1 << 0; @@ -44,9 +51,10 @@ pub const CR0_CMDQEN: u32 = 1 << 3; pub const CR1_WRITE_BACK: u32 = 0b11 << 10 | 0b01 << 8 | 0b01 << 6 | 0b11 << 4 | 0b01 << 2 | 0b01; /// `SMMU_CR2` (§6.3.12): `RECINVSID` [1], so a transaction under a StreamID -/// past the stream table is recorded as well as aborted, and `PTM` [2], so -/// no CPU's broadcast TLB invalidation reaches the unit's entries: its ASIDs -/// are its own. `E2H` [0] stays clear. +/// past the stream table is recorded as well as aborted, as one under an +/// invalid entry in it is, and `PTM` [2], so no CPU's broadcast TLB +/// invalidation reaches the unit's entries: its ASIDs are its own. `E2H` [0] +/// stays clear. pub const CR2_RECORD_PRIVATE: u32 = 1 << 1 | 1 << 2; /// `SMMU_GBPA` (§6.3.15): what happens to every transaction while `SMMUEN` diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs index f25d80ae04d..75677363add 100644 --- a/toyos-smmu/tests/encodings.rs +++ b/toyos-smmu/tests/encodings.rs @@ -215,7 +215,7 @@ fn a_context_descriptors_ips_is_the_units_output_size_and_no_more_than_48_bits() /// (§6.3.130, §6.3.131). #[test] fn the_registers_sit_where_the_register_overview_has_them() { - let page0: [(usize, usize); 18] = [ + let page0: [(usize, usize); 21] = [ (0x0000, unit::IDR0), (0x0004, unit::IDR1), (0x0014, unit::IDR5), @@ -228,12 +228,15 @@ fn the_registers_sit_where_the_register_overview_has_them() { (0x0054, unit::IRQ_CTRLACK), (0x0060, unit::GERROR), (0x0064, unit::GERRORN), + (0x0068, unit::GERROR_IRQ_CFG0), (0x0080, unit::STRTAB_BASE), (0x0088, unit::STRTAB_BASE_CFG), (0x0090, unit::CMDQ_BASE), (0x0098, unit::CMDQ_PROD), (0x009C, unit::CMDQ_CONS), (0x00A0, unit::EVENTQ_BASE), + (0x00B0, unit::EVENTQ_IRQ_CFG0), + (0x00D0, unit::PRIQ_IRQ_CFG0), ]; for (specified, declared) in page0 { assert_eq!(declared, specified); @@ -254,6 +257,7 @@ fn the_control_values_hold_each_field_where_its_register_has_it() { ); // E2H clear, RECINVSID, PTM. assert_eq!(u64::from(unit::CR2_RECORD_PRIVATE), word(&[(0, 0, 0), (1, 1, 1), (2, 2, 1)])); + assert_eq!(u64::from(unit::IDR0_PRI), word(&[(16, 16, 1)])); assert_eq!(u64::from(unit::GBPA_ABORT | unit::GBPA_UPDATE), word(&[(20, 20, 1), (31, 31, 1)])); assert_eq!(u64::from(unit::IRQ_GERROR | unit::IRQ_EVENTQ), word(&[(0, 0, 1), (2, 2, 1)])); assert_eq!( @@ -489,6 +493,7 @@ fn a_translation_fault_names_its_stream_its_address_and_whether_it_wrote() { event(record(number, 0x0001_0008, 0xffff_8000_dead_b000, read)), Event { stream: 0x0001_0008, + number, code, attempt: Some(Attempt { address: 0xffff_8000_dead_b000, write: !read }) } @@ -522,8 +527,9 @@ fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { ]; for number in 0..=u8::MAX { let decoded = event(record(number, 7, 0x1000, false)); + assert_eq!(decoded.code.name() == "unnamed", !named.iter().any(|(n, _)| *n == number), "event {number:#x}"); let code = named.iter().find(|(n, _)| *n == number).map_or(Code::Other(number), |(_, code)| *code); - assert_eq!((decoded.stream, decoded.code), (7, code), "event {number:#x}"); + assert_eq!((decoded.stream, decoded.number, decoded.code), (7, number, code), "event {number:#x}"); assert_eq!(decoded.attempt.is_some(), (0x10..=0x13).contains(&number), "event {number:#x}"); } }