From 562fc3a9e368a941d651feb487b943f94cdb4509 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 09:31:04 +0200 Subject: [PATCH 1/4] The SMMUv3 comes up from the IORT with every stream aborting, and its events reach the handler on their wired SPI The AArch64 IOMMU stub becomes arch/aarch64/smmu/, programmed with toyos-smmu's encodings (G1, #798): - init finds the one SMMUv3 the IORT names, sets GBPA to abort before anything else is written (and reads it back), turns off a unit left enabled, probes IDR0/1/5, gives every enumerated function the IORT routes through the unit an aborting stream table entry, brings up the command queue (CMD_CFGI_ALL, CMD_TLBI_NSNH_ALL, then a CMD_SYNC it waits on), the event queue and its wired interrupt, and sets SMMUEN. Any refusal leaves the unit aborting and says why. - domain: create/map/map_at/place/unmap/attach behind the generic seam, one stage 1 context descriptor per domain under its own ASID, 2 MiB leaves, addresses from 2^46 up to the first root-bridge window, unmap invalidated by ASID behind a CMD_SYNC, attach a CMD_CFGI_STE behind one. - fault: the event queue drained from the SPI with no lock, each record's function stopped from mastering, a claim's fault handed to pcidev::note_fault, a kernel-owned one halting the machine after the line, in VT-d's line format. - irqchip routes that one SPI, edge-triggered, to the boot CPU; trap's irq() takes it under the preempt count. - toyos-smmu names each event by its mnemonic. virt_smmu boots virt with iommu=smmuv3 and QEMU's iommu-testdev under the smmu-selftest actuator: the device's write is refused on the entry it starts with, lands where its own domain maps it, and where the domain maps nothing is refused and recorded, the event reaching the handler on SPI 106 named F_TRANSLATION for that function and address. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- Cargo.lock | 2 + ...end-keeps-its-own-domain-address-window.md | 30 ++ kernel/Cargo.toml | 2 + kernel/src/actuator.rs | 7 + kernel/src/arch/aarch64/iommu_unit.rs | 89 ---- kernel/src/arch/aarch64/irqchip.rs | 59 ++- kernel/src/arch/aarch64/mod.rs | 4 +- kernel/src/arch/aarch64/smmu/domain.rs | 276 ++++++++++++ kernel/src/arch/aarch64/smmu/fault.rs | 229 ++++++++++ kernel/src/arch/aarch64/smmu/mod.rs | 416 ++++++++++++++++++ kernel/src/arch/aarch64/smmu/selftest.rs | 105 +++++ kernel/src/arch/aarch64/trap.rs | 8 + tests/common/qemu.rs | 36 +- tests/toyos.rs | 72 +++ toyos-smmu/src/queue.rs | 28 ++ toyos-smmu/tests/encodings.rs | 29 ++ 16 files changed, 1296 insertions(+), 96 deletions(-) create mode 100644 issues/each-iommu-backend-keeps-its-own-domain-address-window.md delete mode 100644 kernel/src/arch/aarch64/iommu_unit.rs create mode 100644 kernel/src/arch/aarch64/smmu/domain.rs create mode 100644 kernel/src/arch/aarch64/smmu/fault.rs create mode 100644 kernel/src/arch/aarch64/smmu/mod.rs create mode 100644 kernel/src/arch/aarch64/smmu/selftest.rs diff --git a/Cargo.lock b/Cargo.lock index 4e974e0480f..72d55046df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3050,10 +3050,12 @@ dependencies = [ "toyos-gpt", "toyos-hda", "toyos-pci", + "toyos-phys", "toyos-ps2", "toyos-quiesce", "toyos-random", "toyos-rootimage", + "toyos-smmu", "toyos-symbols", "toyos-tco", "toyos-tsc", diff --git a/issues/each-iommu-backend-keeps-its-own-domain-address-window.md b/issues/each-iommu-backend-keeps-its-own-domain-address-window.md new file mode 100644 index 00000000000..277b5021d83 --- /dev/null +++ b/issues/each-iommu-backend-keeps-its-own-domain-address-window.md @@ -0,0 +1,30 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# Each IOMMU backend keeps its own domain address window + +Where a device domain's addresses start, where they end and which of them +are handed out is one policy — a floor a quarter of the way up what the unit +translates and above all memory, a ceiling under the first root-bridge window +or reserved region over the floor, an address never handed out twice — and it +has two declarations: `Domain::new`, `Domain::reserve`, `Domain::handed_out` +and `ceiling` in `kernel/src/arch/x86_64/vtd/table.rs`, and `FLOOR`, +`Addresses` and `ceiling` in `kernel/src/arch/aarch64/smmu/domain.rs`. Nothing +holds the two equal; each carries its own compile-time checks. + +The SMMUv3 came in with the x86-64 side outside its fence, so its copy was +written beside VT-d's rather than hoisted above both. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, whose ITS stage edits +`kernel/src/iommu/mod.rs` and both backends' seam. + +**Exit.** One declaration above `kernel/src/arch/`, read by both backends: + +``` +rg -n 'fn handed_out|fn ceiling' kernel/src/arch +``` + +prints nothing. diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index bc74e4b7650..abd449f1a48 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -493,6 +493,8 @@ toyos-gicv3 = { path = "gicv3" } toyos-gpt = { path = "../toyos-gpt" } toyos-hda = { path = "../toyos-hda" } toyos-pci = { path = "pci" } +toyos-phys = { path = "../toyos-phys" } +toyos-smmu = { path = "../toyos-smmu" } toyos-tco = { path = "../toyos-tco" } toyos-tsc = { path = "../toyos-tsc" } toyos-ps2 = { path = "ps2" } diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 93117b8e7f9..5c0f8c1b03e 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -144,6 +144,13 @@ actuators! { #[allow(dead_code)] psci_withheld = "psci-withheld"; + /// Once the SMMUv3 is armed, have QEMU's `iommu-testdev` write on the + /// entry every stream starts with, on a domain at an address it maps, and + /// at one it does not. Judged by `virt_smmu`. + // The SMMUv3 is AArch64's alone, so x86-64 builds an accessor it never reads. + #[allow(dead_code)] + smmu_selftest = "smmu-selftest"; + /// Time the same read loop on every CPU, either side of the `mov cr0` that enables caching. control_regs_bench = "control-regs-bench"; diff --git a/kernel/src/arch/aarch64/iommu_unit.rs b/kernel/src/arch/aarch64/iommu_unit.rs deleted file mode 100644 index bb07a406314..00000000000 --- a/kernel/src/arch/aarch64/iommu_unit.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! The IOMMU: an SMMUv3 the IORT names, the port's stage 6. Until then no -//! unit exists, and [`init`] says so; a domain is refused rather than built. - -use crate::drivers::pci::PciDevice; -use crate::log; - -pub fn init( - _rsdp_addr: u64, - _devices: &[PciDevice], - _windows: &[toyos_abi::boot::RootBridgeWindow], -) { - log!("IOMMU: the SMMUv3 is the port's stage 6; no device is translated this boot"); -} - -pub mod domain { - use crate::iommu::{DomainId, IommuError, Iova, StreamId}; - - /// No unit is driven, so there is no domain to give. - pub fn create(_room: u64) -> Result<(DomainId, Iova), IommuError> { - Err(IommuError::NoUnit) - } - - pub fn map(_id: DomainId, _phys: u64, _bytes: u64) -> Result { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn map_at(_id: DomainId, _at: Iova, _phys: u64, _bytes: u64) -> Result<(), IommuError> { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn place(_id: DomainId, _at: Iova, _phys: u64, _bytes: u64) -> Result { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn unmap(_id: DomainId, _at: Iova, _bytes: u64) -> Result<(), IommuError> { - unreachable!("no domain exists: `create` refuses every one") - } - - pub fn attach(_stream: StreamId, _id: DomainId) { - unreachable!("no domain exists: `create` refuses every one") - } -} - -pub mod interrupt { - use crate::iommu::{Refused, StreamId}; - - pub struct Msi { - pub address: u32, - pub data: u32, - } - - pub struct Pin { - pub low: u32, - pub high: u32, - } - - /// No unit, so nothing remaps: callers deliver directly. - pub fn is_armed() -> bool { - false - } - - pub fn msi(_source: StreamId, _vector: u8, _dest: u32) -> Result { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn claim(_slot: usize, _source: StreamId, _vector: u8) -> Msi { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn release(_slot: usize, _source: StreamId) { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } - - pub fn pin(_apic_id: u8, _vector: u8, _dest: u32, _level: bool) -> Result { - unreachable!("no interrupt remapping without an IOMMU unit, and `is_armed` said so") - } -} - -pub mod fault { - use crate::iommu::StreamId; - - pub fn user_owned(_stream: StreamId, _slot: Option) { - owed!("SMMUv3 fault reporting", "stage 6") - } - - pub fn service() { - owed!("SMMUv3 fault reporting", "stage 6") - } -} diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index ebef83ce249..863ce2afb91 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -4,8 +4,9 @@ //! timer's EL1 virtual timer (Arm ARM K.a, chapter D12), whose PPI the GTDT //! names. //! -//! **What this kernel takes is SGIs and the timer's PPI, and nothing else.** -//! No SPI is routed and no LPI exists: a device's interrupt is a message the +//! **What this kernel takes is SGIs, the timer's PPI and the IOMMU's event +//! SPI ([`route_iommu_events`]), and nothing else.** No other SPI is routed +//! and no LPI exists: a device's interrupt is a message the //! GICv3 ITS translates, and every device that would take one is either a //! driver the small-kernel track moves out of the kernel or a claimed function, //! which the SMMUv3 of the port's stage 6 must translate first @@ -16,7 +17,7 @@ //! zero by the entry from EL2 the two count alike. It is level-triggered, so a //! handler that neither re-arms nor stops it takes it again at once. -use core::sync::atomic::{AtomicBool, AtomicU32, Ordering::Relaxed}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed}; use alloc::vec::Vec; @@ -62,6 +63,20 @@ const CTLR_GRP1: u32 = 1 << 1; const RWP: u32 = 1 << 31; /// `GICD_PIDR2.ArchRev`, bits 7:4: 3 is GICv3, 4 is GICv4. const GICD_PIDR2: u64 = 0xFFE8; +/// `GICD_TYPER.ITLinesNumber`, bits 4:0: the distributor takes INTIDs below +/// `32 * (ITLinesNumber + 1)`, and none from 1020. +const GICD_TYPER: u64 = 0x0004; +/// An SPI's bit in each of the word-per-32 registers, its byte in +/// `GICD_IPRIORITYR`, its two bits in `GICD_ICFGR` and its doubleword in +/// `GICD_IROUTER`. +const GICD_IGROUPR: u64 = 0x0080; +const GICD_ISENABLER: u64 = 0x0100; +const GICD_ICENABLER: u64 = 0x0180; +const GICD_ICPENDR: u64 = 0x0280; +const GICD_IPRIORITYR: u64 = 0x0400; +const GICD_ICFGR: u64 = 0x0C00; +const GICD_IGRPMODR: u64 = 0x0D00; +const GICD_IROUTER: u64 = 0x6000; /// A redistributor's frames: `RD_base`, then `SGI_base` 64 KiB above it. const GICR_CTLR: u64 = 0x0000; @@ -92,6 +107,11 @@ const SPURIOUS: u32 = 1023; static TIMER_INTID: AtomicU32 = AtomicU32::new(0); static TIMER_EDGE: AtomicBool = AtomicBool::new(false); +/// The distributor's physical base, from [`init`]. +static DISTRIBUTOR: AtomicU64 = AtomicU64::new(0); +/// The IOMMU's event SPI; zero until [`route_iommu_events`]. +static IOMMU_EVENTS: AtomicU32 = AtomicU32::new(0); + /// Wait until `done`, for at most `1 / per_second` of a second counted at the /// rate firmware states: the boot has no calibrated clock yet. fn settles(per_second: u64, what: &str, done: impl Fn() -> bool) { @@ -159,6 +179,7 @@ pub fn init(rsdp_addr: u64) -> Gic { } let gicd = gicd.expect("GIC: the MADT names no distributor"); let distributor = crate::mm::paging::map_mmio(gicd, FRAME, MmioPolicy::Uncacheable); + DISTRIBUTOR.store(gicd, Relaxed); let revision = distributor.read_u32(GICD_PIDR2) >> 4 & 0xF; assert!(revision >= 3, "GIC: GICD_PIDR2.ArchRev is {revision}, and this kernel drives a GICv3 or later"); @@ -277,6 +298,38 @@ pub(super) fn timer_intid() -> u32 { TIMER_INTID.load(Relaxed) } +/// Take SPI `intid` on this CPU at [`PRIORITY`], edge-triggered, as the +/// IOMMU's event interrupt: an SMMUv3's wired interrupts are pulses, and the +/// IORT names no trigger. Refused with the distributor's limit where it takes +/// no such SPI. +pub(super) fn route_iommu_events(intid: u32) -> Result<(), u32> { + let gicd = Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME); + let limit = (32 * ((gicd.read_u32(GICD_TYPER) & 0x1F) + 1)).min(1020); + if !(32..limit).contains(&intid) { + return Err(limit); + } + let (word, bit) = (u64::from(intid / 32) * 4, 1 << (intid % 32)); + gicd.write_u32(GICD_ICENABLER + word, bit); + settles(100, "GICD_CTLR after disabling an SPI", || gicd.read_u32(GICD_CTLR) & RWP == 0); + gicd.write_u32(GICD_IGROUPR + word, gicd.read_u32(GICD_IGROUPR + word) | bit); + gicd.write_u32(GICD_IGRPMODR + word, gicd.read_u32(GICD_IGRPMODR + word) & !bit); + let priority = GICD_IPRIORITYR + u64::from(intid & !3); + let shift = 8 * (intid % 4); + gicd.write_u32(priority, gicd.read_u32(priority) & !(0xFF << shift) | u32::from(PRIORITY) << shift); + let config = GICD_ICFGR + u64::from(intid / 16) * 4; + gicd.write_u32(config, gicd.read_u32(config) | 0b10 << (2 * (intid % 16))); + gicd.write_u64(GICD_IROUTER + 8 * u64::from(intid), toyos_gicv3::unpacked_affinity(cpu::hardware_id())); + gicd.write_u32(GICD_ICPENDR + word, bit); + IOMMU_EVENTS.store(intid, Relaxed); + gicd.write_u32(GICD_ISENABLER + word, bit); + Ok(()) +} + +/// The IOMMU's event SPI, or zero where [`route_iommu_events`] routed none. +pub(super) fn iommu_events() -> u32 { + IOMMU_EVENTS.load(Relaxed) +} + /// Write `ICC_SGI1R_EL1` whole: the SGI it names is raised, after every store /// before it. Only a `DSB` orders a system register write after stores to /// Normal memory, so without it a target could take the SGI before the store diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index aeb2aef53e2..a3c29dfc62d 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -23,7 +23,6 @@ pub mod entropy; pub mod entry; pub mod fpu; pub mod hw; -pub mod iommu_unit; pub mod irqchip; pub mod keyboard_controller; pub mod paging; @@ -33,12 +32,15 @@ pub mod pmu; pub mod power; pub mod psci; pub mod rtc; +pub mod smmu; pub mod smp; pub mod switch; pub mod tlb; pub mod trap; pub mod watchdog; +pub use smmu as iommu_unit; + /// The machine every program image this kernel loads must be built for. pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::Aarch64; diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs new file mode 100644 index 00000000000..8b438596a54 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -0,0 +1,276 @@ +//! A device's address space: one context descriptor, tagged with an ASID of +//! its own, over stage 1 tables that map 2 MiB blocks and nothing else +//! (`toyos_smmu::table`), and the device addresses it hands out. +//! +//! A domain's addresses start a quarter of the way up the 48-bit input, above +//! all memory, so a descriptor still carrying one names nothing a domain maps +//! by accident; they end below the first root-bridge window over that, which +//! a bridge may route peer-to-peer before the unit sees the request. An +//! address is handed out once: a device holding a stale one reaches whatever +//! took its place, so nothing takes it. +//! +//! A new leaf needs no invalidation — the unit caches no translation that +//! faulted — and a removed one is invalidated by the domain's ASID, behind a +//! `CMD_SYNC`, before [`unmap`] returns. + +use toyos_phys::Phys; +use toyos_smmu::config::{Cd, Ste}; +use toyos_smmu::queue::Command; +use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS}; +use toyos_smmu::Asid; + +use super::{window, Live, UNIT}; +use crate::iommu::{DomainId, IommuError, Iova, StreamId}; +use crate::log; +use crate::mm::{PAGE_2M, PAGE_SIZE}; + +/// Domain ids start at 1: `DomainId` is never 0. +const FIRST: u16 = 1; + +/// A quarter of the way up the input. +const FLOOR: u64 = 1 << (INPUT_BITS - 2); + +pub(super) struct Domain { + asid: Asid, + root: Phys<12>, + context: Phys<6>, + addresses: Addresses, +} + +/// A domain's device addresses, from [`FLOOR`] to its ceiling. +#[derive(Clone, Copy)] +struct Addresses { + ceiling: u64, + /// The first not yet handed out. + next: u64, +} + +impl Addresses { + /// `bytes`, rounded up to whole leaves, of addresses never handed out. + const fn reserve(&mut self, bytes: u64) -> Option { + let Some(end) = bytes.checked_next_multiple_of(PAGE_2M) else { return None }; + let Some(end) = self.next.checked_add(end) else { return None }; + if end > self.ceiling { + return None; + } + let at = Iova::translated(self.next); + self.next = end; + Some(at) + } + + /// Whether `bytes` at `at` is room [`Self::reserve`] handed out, starting + /// on a leaf it could have returned. + const fn handed_out(&self, at: Iova, bytes: u64) -> bool { + if !at.raw().is_multiple_of(PAGE_2M) || at.raw() < FLOOR { + return false; + } + match bytes.checked_next_multiple_of(PAGE_2M) { + Some(span) => match at.raw().checked_add(span) { + Some(end) => end <= self.next, + None => false, + }, + None => false, + } + } +} + +/// Where a domain's addresses end: under the input, and under the first of +/// `reserved` reaching above [`FLOOR`]; at or below it where one covers it. +const fn ceiling(reserved: &[(u64, u64)]) -> u64 { + let mut ceiling = 1 << INPUT_BITS; + let mut i = 0; + while i < reserved.len() { + let (start, end) = reserved[i]; + if end > FLOOR && start < ceiling { + ceiling = start; + } + i += 1; + } + ceiling +} + +/// [`Addresses`] and [`ceiling`] at their boundaries, at compile time: the +/// binary has no test harness. +const _: () = { + let mut one = Addresses { ceiling: FLOOR + 2 * PAGE_2M, next: FLOOR }; + assert!(matches!(one.reserve(1), Some(at) if at.raw() == FLOOR)); + assert!(one.handed_out(Iova::translated(FLOOR), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); + assert!(!one.handed_out(Iova::translated(FLOOR + 1), 0)); + assert!(!one.handed_out(Iova::translated(u64::MAX - PAGE_2M + 1), PAGE_2M)); + // Past the ceiling, and past the end of the address space, nothing. + assert!(one.reserve(PAGE_2M + 1).is_none()); + assert!(one.reserve(u64::MAX).is_none()); + assert!(matches!(one.reserve(PAGE_2M), Some(at) if at.raw() == FLOOR + PAGE_2M)); + assert!(one.reserve(1).is_none()); + // `virt`'s windows, all below the floor, leave the whole input. + assert!(ceiling(&[(0x1000_0000, 0x3f00_0000), (0x80_0000_0000, 0x100_0000_0000)]) == 1 << INPUT_BITS); + // A window over the floor ends the domain where it starts; one across it leaves nothing. + assert!(ceiling(&[(FLOOR + 4 * PAGE_2M, FLOOR + 8 * PAGE_2M)]) == FLOOR + 4 * PAGE_2M); + assert!(ceiling(&[(FLOOR - PAGE_2M, FLOOR + PAGE_2M)]) < FLOOR); +}; + +impl Live { + fn domain(&mut self, id: DomainId) -> &mut Domain { + &mut self.domains[usize::from(id.raw() - FIRST)] + } + + /// One 2 MiB block of `phys` at `at`, its tables grown on the way. + fn put(&mut self, root: Phys<12>, at: u64, phys: u64) { + let block = Phys::<21>::new(phys).expect("SMMU: a 2 MiB-aligned page of memory"); + let path = plan(at, Leaf::Memory(block, Access::ReadWrite), &self.unit).expect("SMMU: a handed-out address"); + let mut table = root; + let (leaf, walk) = path.indices().split_last().expect("a path names at least its leaf's table"); + for (level, index) in walk.iter().enumerate() { + let slot = window(table.get(), PAGE_SIZE).subregion(*index as u64 * 8, 8); + table = match entry(slot.read_u64(0), level) { + Entry::Table(below) => below, + Entry::Invalid => { + let below = Phys::new(self.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); + slot.write_u64(0, next(below, &self.unit).expect("SMMU: a table below the unit's output size")); + below + } + Entry::Mapped => panic!("SMMU: {at:#x}'s level {level} entry is a block, where a table goes"), + }; + } + let slot = window(table.get(), PAGE_SIZE).subregion(*leaf as u64 * 8, 8); + // A live leaf here is memory a holder still reaches: the caller takes + // a mapping back before it puts another at its address. + assert!(entry(slot.read_u64(0), walk.len()) == Entry::Invalid, "SMMU: {at:#x} was still mapped when a new leaf was written there"); + slot.write_u64(0, path.descriptor); + } + + /// Clear the 2 MiB block at `at`; `false` where nothing is mapped there. + fn take(&mut self, root: Phys<12>, at: u64) -> bool { + let any = Phys::<21>::new(0).expect("0 is aligned"); + let Some(path) = plan(at, Leaf::Memory(any, Access::Read), &self.unit) else { return false }; + let mut table = root; + let (leaf, walk) = path.indices().split_last().expect("a path names at least its leaf's table"); + for (level, index) in walk.iter().enumerate() { + match entry(window(table.get(), PAGE_SIZE).read_u64(*index as u64 * 8), level) { + Entry::Table(below) => table = below, + Entry::Invalid | Entry::Mapped => return false, + } + } + let slot = window(table.get(), PAGE_SIZE).subregion(*leaf as u64 * 8, 8); + if entry(slot.read_u64(0), walk.len()) != Entry::Mapped { + return false; + } + slot.write_u64(0, 0); + true + } +} + +/// A new domain, with `room` bytes of it handed out first: refused, with no id +/// spent, where it would have less than that. +pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { + let mut held = UNIT.lock(); + let live = held.as_mut().ok_or(IommuError::NoUnit)?; + let domains = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; + let id = u16::try_from(usize::from(FIRST) + live.domains.len()).map_err(|_| IommuError::DomainsExhausted(domains))?; + let asid = live.unit.asid(id).ok_or(IommuError::DomainsExhausted(domains))?; + let top = crate::mm::pmm::top(); + if FLOOR <= top { + return Err(IommuError::WindowBelowMemory { translatable: INPUT_BITS as u8, floor: FLOOR, top }); + } + let ceiling = ceiling(&live.reserved); + let mut addresses = Addresses { ceiling, next: FLOOR }; + // At least one leaf, whatever was asked: a domain with none is no domain. + let first = match addresses.reserve(room) { + Some(first) if ceiling >= FLOOR + PAGE_2M => first, + _ => return Err(IommuError::NoRoom { floor: FLOOR, ceiling, room }), + }; + let root = Phys::new(live.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); + let context = Phys::new(live.memory.alloc(64)).expect("SMMU: a descriptor below the unit's output size"); + let words = Cd::new(root, asid, &live.unit).expect("SMMU: a table below the unit's output size").words(); + let descriptor = window(context.get(), 64); + // `V` is in the first doubleword, so it goes last. + for (i, word) in words.iter().enumerate().skip(1) { + descriptor.write_u64(8 * i as u64, *word); + } + descriptor.write_u64(0, words[0]); + let domain = Domain { asid, root, context, addresses }; + log!( + "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {FLOOR:#x} to {ceiling:#x}", + root.get(), + context.get(), + asid.get() + ); + live.domains.push(domain); + Ok((DomainId::new(id), first)) +} + +pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { + if !phys.is_multiple_of(PAGE_2M) { + return Err(IommuError::Unaligned(phys)); + } + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let domain = live.domain(id); + let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling))?; + let root = domain.root; + for offset in (0..bytes).step_by(PAGE_2M as usize) { + live.put(root, at.raw() + offset, phys + offset); + } + log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + Ok(at) +} + +/// Put `bytes` at `phys` at `at` again, room this domain handed out before +/// and whose mapping was taken back. +pub fn map_at(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result<(), IommuError> { + place(id, at, phys, bytes)?; + log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x} again", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); + Ok(()) +} + +/// [`map_at`] without its record line, for a mapping a holder makes and takes +/// back as often as it likes. Room never handed out is a kernel bug: [`map`] +/// may yet hand it out. +pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result { + if !phys.is_multiple_of(PAGE_2M) { + return Err(IommuError::Unaligned(phys)); + } + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let domain = live.domain(id); + assert!(domain.addresses.handed_out(at, bytes), "iommu: domain{} never handed out {:#x}+{bytes:#x}", id.raw(), at.raw()); + let root = domain.root; + for offset in (0..bytes).step_by(PAGE_2M as usize) { + live.put(root, at.raw() + offset, phys + offset); + } + Ok(id.raw()) +} + +/// Take `bytes` at `at` back: no device reaches them once this returns. +pub fn unmap(id: DomainId, at: Iova, bytes: u64) -> Result<(), IommuError> { + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let (root, asid) = (live.domain(id).root, live.domain(id).asid); + let mut result = Ok(()); + for offset in (0..bytes).step_by(PAGE_2M as usize) { + let here = at.raw() + offset; + if !live.take(root, here) { + result = Err(IommuError::NotMapped(Iova::translated(here))); + break; + } + } + // Whatever was cleared before a refusal is gone from the unit too. + live.issue(&[Command::InvalidateAsid(asid)]); + result +} + +/// Move `function` onto domain `id`: translating through it, and only it, +/// once this returns. +pub fn attach(function: StreamId, id: DomainId) { + let mut held = UNIT.lock(); + let live = held.as_mut().expect("a domain exists only on an armed unit"); + let stream = live.stream(function); + let context = live.domain(id).context; + let ste = Ste::stage1(context, &live.unit).expect("SMMU: a descriptor below the unit's output size"); + live.write_entry(stream, ste); + super::fault::attached(stream, id.raw()); + log!("iommu: {function} moves to domain{}", id.raw()); +} diff --git a/kernel/src/arch/aarch64/smmu/fault.rs b/kernel/src/arch/aarch64/smmu/fault.rs new file mode 100644 index 00000000000..a7b637c07b1 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/fault.rs @@ -0,0 +1,229 @@ +//! The event queue, drained from the unit's wired event interrupt and never +//! polled. +//! +//! The handler allocates nothing and takes no lock: what it reads is +//! published once, whole, before the interrupt is enabled, and what it writes +//! is atomics. For each record it first clears Bus Master Enable on the +//! function the record names — a function that cannot master the bus raises +//! no second event, which is the ceiling on a storm (PCI 3.0 §6.2.2, bit 2 of +//! `COMMAND`) — then counts it and hands it to whoever drives the function. +//! +//! **Who that is decides what the record ends.** A stream a process drives +//! has an owner: `pcidev` is told, that claim refuses every later call, and +//! the machine goes on. A stream no process drives has nobody to hand it to, +//! so after the line the machine halts, as x86-64's VT-d handler does. + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; + +use toyos_smmu::queue::{event, Events}; +use toyos_smmu::unit as reg; + +use super::Registers; +use crate::drivers::pci::{self, PciDevice}; +use crate::iommu::StreamId; +use crate::log; +use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; + +/// A `pcidev` slot number no claim has: this function is not driven by one. +const NO_SLOT: u32 = u32::MAX; + +/// One function the unit translates for. +struct Function { + function: StreamId, + stream: u32, + /// Physical base of its config window, through which an event clears `BME`. + config: u64, + domain: AtomicU32, + faults: AtomicU32, + /// The `pcidev` slot a process drives this function on, or [`NO_SLOT`]. + user_slot: AtomicU32, +} + +/// What the handler reads, published once by [`arm`]. +struct Armed { + regs: Registers, + queue: Mmio, + events: Events, + functions: Box<[Function]>, + /// Events this unit has recorded for the life of the boot. + count: AtomicU32, + /// The StreamID of the first, or `u32::MAX`. + first: AtomicU32, +} + +static ARMED: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +fn armed() -> Option<&'static Armed> { + // SAFETY: non-null only as `arm` stored it: a leaked box, never freed, + // published with Release once it was whole. + unsafe { ARMED.load(Ordering::Acquire).as_ref() } +} + +/// Everything the handler reads, before the unit's event interrupt is +/// enabled: the registers, the queue and its indexes, and each function +/// `routes` names. +pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, devices: &[PciDevice], routes: &[(StreamId, u32)]) { + let functions: Vec = routes + .iter() + .map(|&(function, stream)| { + let device = devices + .iter() + .find(|d| StreamId::pci(d.bus, d.dev, d.func) == function) + .expect("a route is an enumerated function's"); + Function { + function, + stream, + config: DirectMap::phys_of(device.config_window().addr() as *const u8), + domain: AtomicU32::new(0), + faults: AtomicU32::new(0), + user_slot: AtomicU32::new(NO_SLOT), + } + }) + .collect(); + let armed = Box::new(Armed { + regs, + queue, + events, + functions: functions.into_boxed_slice(), + count: AtomicU32::new(0), + first: AtomicU32::new(u32::MAX), + }); + let first = ARMED.compare_exchange(core::ptr::null_mut(), Box::leak(armed), Ordering::Release, Ordering::Relaxed); + assert!(first.is_ok(), "SMMU: the event handler was armed twice"); +} + +fn find(stream: u32) -> Option<&'static Function> { + armed()?.functions.iter().find(|f| f.stream == stream) +} + +fn by_function(function: StreamId) -> Option<&'static Function> { + armed()?.functions.iter().find(|f| f.function == function) +} + +/// Record which domain stream `stream` moved to, for the line. +pub(super) fn attached(stream: u32, domain: u16) { + if let Some(function) = find(stream) { + function.domain.store(u32::from(domain), Ordering::Relaxed); + } +} + +/// Record that a process drives `function` on `slot`, or no longer does. +pub fn user_owned(function: StreamId, slot: Option) { + if let Some(function) = by_function(function) { + function.user_slot.store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); + } +} + +/// The events the unit has recorded, as the handler has read them. +#[cfg(feature = "boot-actuators")] +pub(super) fn recorded() -> u32 { + armed().map_or(0, |armed| armed.count.load(Ordering::Relaxed)) +} + +/// The unit's event interrupt. +pub fn service() { + let armed = armed().expect("SMMU: the event interrupt is enabled only once the handler is armed"); + let (regs, events) = (armed.regs, armed.events); + let mut kernel_owned = 0usize; + let mut prod = regs.read(reg::EVENTQ_PROD); + let mut cons = regs.read(reg::EVENTQ_CONS); + if events.overflowed(prod, cons) { + log!("iommu: the SMMUv3's event queue overflowed: earlier events are lost"); + } + // Every record the unit wrote before the last read of `PROD`, one that + // lands after it raising the interrupt again; at most twice a queue's + // worth, so a stream no `BME` stops cannot hold this CPU here. + let mut budget = 2u32 << super::EVENTS_LOG2; + while !events.is_empty(prod, cons) && budget > 0 { + budget -= 1; + let at = events.slot(cons) as u64 * 32; + let record = [0, 8, 16, 24].map(|offset| armed.queue.read_u64(at + offset)); + if record_one(armed, event(record)) { + kernel_owned += 1; + } + cons = events.after(cons, prod); + regs.write(reg::EVENTQ_CONS, cons); + prod = regs.read(reg::EVENTQ_PROD); + } + let errors = reg::active_errors(regs.read(reg::GERROR), regs.read(reg::GERRORN)); + if errors & reg::GERROR_EVENTQ_ABORT != 0 { + log!("iommu: the SMMUv3 could not write its event queue: events are lost (GERROR {errors:#x})"); + } + if kernel_owned > 0 { + // A function reached an address this kernel never gave it, and + // nothing here can know what else it did: the line is the report, + // and the machine stops. + crate::drivers::panic_console::capture(); + crate::panic::halt_all_cpus(); + } +} + +/// One record, handed to whoever drives the function it names; `true` where +/// that is nobody. +fn record_one(armed: &Armed, event: toyos_smmu::queue::Event) -> bool { + let function = find(event.stream); + // First, before anything that can be slow: the ceiling on a storm. + if let Some(function) = function { + pci::stop_bus_mastering(config_window(function.config)); + } + let faults = function.map_or(0, |f| f.faults.fetch_add(1, Ordering::Relaxed) + 1); + let count = armed.count.fetch_add(1, Ordering::Relaxed) + 1; + let first = armed.first.compare_exchange(u32::MAX, event.stream, Ordering::AcqRel, Ordering::Relaxed).is_ok(); + let owner = function.map(|f| f.user_slot.load(Ordering::Acquire)).filter(|slot| *slot != NO_SLOT); + if let Some(slot) = owner { + crate::pcidev::note_fault(slot as usize); + } + let (address, access) = match event.attempt { + Some(attempt) => (attempt.address, if attempt.write { "write" } else { "read" }), + None => (0, "none"), + }; + log!( + // `owner=` first: `tests/common/serial.rs` reads `owner=kernel` as a + // death and the other form as a record. The event's name is the last + // word. + "iommu: DMA FAULT owner={} unit0 stream={} addr={address:#018x} access={access} domain={} bme={} \ + unitfaults={count} streamfaults={faults} first={} reason={:?} {}", + Owner(owner), + Who(function, event.stream), + function.map_or(0, |f| f.domain.load(Ordering::Relaxed)), + if function.is_some() { "cleared" } else { "unknown-function" }, + if first { 'y' } else { 'n' }, + event.code, + event.code.name(), + ); + owner.is_none() +} + +/// The function's config window, from the address [`arm`] published. +fn config_window(phys: u64) -> Mmio { + // SAFETY: `phys` is `DirectMap::phys_of` a config window the ECAM mapping + // holds for the machine's life, one page wide. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), PAGE_SIZE) } +} + +/// Who an event was handed to. +struct Owner(Option); + +impl core::fmt::Display for Owner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + None => write!(f, "kernel"), + Some(slot) => write!(f, "slot{slot}"), + } + } +} + +/// The function an event names, as `pci::enumerate` prints it, or the raw +/// StreamID where it is none this unit translates for. +struct Who(Option<&'static Function>, u32); + +impl core::fmt::Display for Who { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + Some(function) => write!(f, "{}", function.function), + None => write!(f, "{:#x}", self.1), + } + } +} diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs new file mode 100644 index 00000000000..8aba96fe553 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -0,0 +1,416 @@ +//! The IOMMU: the SMMUv3 the IORT names (Arm IHI 0070 H.a), programmed with +//! `toyos_smmu`'s encodings. Every register and Arm name of the unit is at or +//! below this module. +//! +//! [`init`] arms the unit with every stream aborting and no bypass anywhere: +//! `GBPA` aborts before anything else is written, so nothing passes while the +//! unit is programmed; every enumerated function's stream has an entry that +//! aborts, and the unit is enabled. A stream translates only once +//! [`domain::attach`] puts it on a domain of its own, through stage 1 alone; +//! no domain maps memory by identity, so a function no driver attached +//! reaches nothing. A unit this kernel cannot program is left aborting every +//! transaction, and no domain is given. +//! +//! The unit reads what it is given coherently — `toyos_smmu::unit::probe` +//! refuses one that does not — so a store it reads is published by the order +//! [`Mmio`] gives it ahead of the register write or the command that names it. +//! +//! One lock, [`UNIT`], over the unit's registers, queues, tables and domains. +//! The event interrupt takes none ([`fault`]). + +pub mod domain; +pub mod fault; +#[cfg(feature = "boot-actuators")] +mod selftest; + +use alloc::vec::Vec; + +use toyos_acpi::{IortRefused, Node, Route, TableError}; +use toyos_phys::Phys; +use toyos_smmu::config::Ste; +use toyos_smmu::queue::{Command, Commands, Signal}; +use toyos_smmu::unit::{self as reg, Unit}; + +use crate::drivers::acpi::direct_phys; +use crate::drivers::pci::PciDevice; +use crate::iommu::StreamId; +use crate::log; +use crate::mm::pmm::{self, PhysPage}; +use crate::mm::policy::MmioPolicy; +use crate::mm::{DirectMap, Mmio, PAGE_2M}; +use crate::sync::Lock; +use crate::time::{Duration, Tripwire}; + +/// Register pages 0 and 1, 64 KiB each (§6.1). +const REGISTER_PAGE: u64 = 0x1_0000; + +/// The largest queues this kernel makes, as log2 of their entries: 4 KiB each. +const COMMANDS_LOG2: u8 = 8; +const EVENTS_LOG2: u8 = 7; + +/// A stream table no larger than one 2 MiB page. +const STREAMS_LOG2_MAX: u32 = 15; + +/// How long the unit is given to take a register write or consume its +/// commands before the kernel panics: a unit half-way through either is one +/// whose reach nothing can state. +const UNIT_TIMEOUT: Tripwire = Tripwire::absurd( + Duration::from_secs(1), + "a unit that takes no register write or command for a second is one whose reach nothing can state", +); + +static UNIT: Lock> = Lock::new(None); + +/// The unit's two register pages, addressed by `toyos_smmu::unit`'s offsets. +#[derive(Clone, Copy)] +struct Registers(Mmio); + +impl Registers { + fn read(self, at: usize) -> u32 { + self.0.read_u32(at as u64) + } + + fn write(self, at: usize, value: u32) { + self.0.write_u32(at as u64, value); + } + + fn write64(self, at: usize, value: u64) { + self.0.write_u64(at as u64, value); + } + + /// Wait until `done`, panicking past [`UNIT_TIMEOUT`] or on a command the + /// unit stopped its queue at. + fn wait(self, what: &str, done: impl Fn() -> bool) { + let deadline = crate::clock::nanos_since_boot() + UNIT_TIMEOUT.nanos(); + while !done() { + let errors = reg::active_errors(self.read(reg::GERROR), self.read(reg::GERRORN)); + if errors & reg::GERROR_CMDQ != 0 { + let cons = self.read(reg::CMDQ_CONS); + panic!("SMMU: the command queue stopped at CONS {cons:#x}: {:?}", reg::command_error(cons)); + } + assert!( + crate::clock::nanos_since_boot() < deadline, + "SMMU: {what} not seen within {UNIT_TIMEOUT}; GERROR {errors:#x}" + ); + core::hint::spin_loop(); + } + } + + /// Write `value` to the control register at `at` and wait for its + /// acknowledgement register, the word above it, to read it back. + fn control(self, at: usize, value: u32, what: &str) { + self.write(at, value); + self.wait(what, || self.read(at + 4) == value); + } +} + +/// The armed unit and everything it reads. +struct Live { + regs: Registers, + unit: Unit, + commands: Commands, + queue: Mmio, + /// `SMMU_CMDQ_PROD` as last written. + prod: u32, + streams: Mmio, + memory: Memory, + /// Each enumerated function the IORT routes through this unit, and its + /// StreamID. + routes: Vec<(StreamId, u32)>, + domains: Vec, + /// What no domain's addresses may reach, as `(start, end)`. + reserved: Vec<(u64, u64)>, +} + +impl Live { + /// `commands`, then a `CMD_SYNC`, consumed: every one has taken effect + /// when this returns. + fn issue(&mut self, commands: &[Command]) { + let (regs, queue) = (self.regs, self.commands); + for command in commands.iter().copied().chain([Command::Sync(Signal::None)]) { + let prod = self.prod; + regs.wait("room in its command queue", || !queue.is_full(prod, regs.read(reg::CMDQ_CONS))); + let at = queue.slot(prod) as u64 * 16; + let [low, high] = command.words(); + self.queue.write_u64(at, low); + self.queue.write_u64(at + 8, high); + self.prod = queue.after(prod); + } + let prod = self.prod; + regs.write(reg::CMDQ_PROD, prod); + regs.wait("its CMD_SYNC consumed", || queue.is_empty(prod, regs.read(reg::CMDQ_CONS))); + } + + /// The StreamID the IORT gives `function`'s requests through this unit. + fn stream(&self, function: StreamId) -> u32 { + let route = self.routes.iter().find(|(rid, _)| *rid == function); + route.unwrap_or_else(|| panic!("SMMU: {function} is no function the IORT routes through this unit")).1 + } + + /// Point stream `stream`'s entry at `ste` and have the unit forget what + /// it held for the old one. The first doubleword goes last: it holds `V`, + /// `Config` and the context pointer, and the others are read only under + /// the configuration it names. + fn write_entry(&mut self, stream: u32, ste: Ste) { + let words = ste.words(); + let at = u64::from(stream) * 64; + for (i, word) in words.iter().enumerate().skip(1) { + self.streams.write_u64(at + 8 * i as u64, *word); + } + self.streams.write_u64(at, words[0]); + self.issue(&[Command::ForgetStream(stream)]); + } +} + +/// What the unit reads, carved from 2 MiB pages and never given back: a page +/// the unit may still walk is not a page to free. +struct Memory { + pages: Vec, + /// Bytes taken from the newest page; full to begin with. + used: u64, +} + +impl Memory { + const fn new() -> Self { + Self { pages: Vec::new(), used: PAGE_2M } + } + + /// `bytes` of zeroes, aligned to their own size: a power of two from 64 + /// bytes to 2 MiB. + fn alloc(&mut self, bytes: u64) -> u64 { + assert!(bytes.is_power_of_two() && (64..=PAGE_2M).contains(&bytes), "SMMU: {bytes:#x} bytes of tables"); + let mut at = self.used.next_multiple_of(bytes); + if at + bytes > PAGE_2M { + self.pages.push(pmm::alloc_page().expect("SMMU: no memory for the unit's tables")); + at = 0; + } + self.used = at + bytes; + let phys = self.pages.last().expect("a page was just pushed").direct_map().phys() + at; + let zeroed = window(phys, bytes); + for offset in (0..bytes).step_by(8) { + zeroed.write_u64(offset, 0); + } + phys + } +} + +/// `bytes` of [`Memory`] at `phys`. +fn window(phys: u64, bytes: u64) -> Mmio { + // SAFETY: every caller names memory `Memory::alloc` handed out, which is + // never freed and which the direct map covers for the machine's life. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } +} + +/// The SMMUv3, armed; a refusal leaves it aborting what it is given, or says +/// why it could not be told to. +pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::RootBridgeWindow]) { + let iort = match toyos_acpi::iort(direct_phys(), rsdp_addr) { + Ok(iort) => iort, + Err(IortRefused::Table(TableError::Absent)) => { + log!("IOMMU: no IORT, so no SMMUv3: no device is translated this boot"); + return; + } + Err(why) => { + log!("IOMMU: the IORT is refused: {why:?}; no device is translated this boot"); + return; + } + }; + let mut units = iort.nodes().filter_map(|node| match node { + Node::Smmuv3(smmu) => Some(smmu), + _ => None, + }); + let Some(smmu) = units.next() else { + log!("IOMMU: the IORT names no SMMUv3: no device is translated this boot"); + return; + }; + if units.next().is_some() { + log!("IOMMU: the IORT names more than one SMMUv3, and this kernel drives one: none is touched"); + return; + } + if !smmu.base.is_multiple_of(REGISTER_PAGE) { + log!("IOMMU: the SMMUv3's registers at {:#x} are not on a 64 KiB page: not touched", smmu.base); + return; + } + let regs = Registers(crate::mm::paging::map_mmio(smmu.base, 2 * REGISTER_PAGE, MmioPolicy::Uncacheable)); + abort_unprogrammed(regs, smmu.base); + + let segment = u32::from(crate::pcidev::segment()); + let mut routes = Vec::new(); + for device in devices { + let function = StreamId::pci(device.bus, device.dev, device.func); + match iort.route(segment, function.requester()) { + Ok(Route::Translated { smmu: by, stream, .. }) if by.base == smmu.base => routes.push((function, stream)), + Ok(route) => log!("IOMMU: {function} is not routed through the SMMUv3: {route:?}"), + Err(why) => log!("IOMMU: {function}'s IORT route is refused: {why:?}"), + } + } + let reserved = windows.iter().map(|w| (w.base, w.end())).collect(); + let Some(live) = program(regs, smmu.base, smmu.coherent_override, smmu.event, routes, reserved, devices) else { + return; + }; + *UNIT.lock() = Some(live); + #[cfg(feature = "boot-actuators")] + if crate::actuator::smmu_selftest() { + selftest::run(devices); + } +} + +/// `GBPA` aborting and the unit disabled: from here until `SMMUEN`, and for +/// good if it is never set, every transaction aborts. +fn abort_unprogrammed(regs: Registers, base: u64) { + let gbpa = || regs.read(reg::GBPA); + regs.wait("GBPA free to update", || gbpa() & reg::GBPA_UPDATE == 0); + regs.write(reg::GBPA, gbpa() | reg::GBPA_ABORT | reg::GBPA_UPDATE); + regs.wait("GBPA updated", || gbpa() & reg::GBPA_UPDATE == 0); + assert!( + gbpa() & reg::GBPA_ABORT != 0, + "SMMU: GBPA reads {:#x} after ABORT was written: transactions bypass while SMMUEN is clear", + gbpa() + ); + let cr0 = regs.read(reg::CR0); + if cr0 != 0 { + log!("IOMMU: the SMMUv3 at {base:#x} was handed over with CR0 {cr0:#x}; it goes off first"); + regs.control(reg::CR0, 0, "CR0 cleared"); + } + log!("IOMMU: SMMUv3 at {base:#x}: GBPA {:#x}, every transaction aborts while SMMUEN is clear", gbpa()); +} + +/// The unit's tables, queues and interrupt, and `SMMUEN`; `None`, with the +/// reason logged, where it cannot be, the unit left as +/// [`abort_unprogrammed`] left it. +fn program( + regs: Registers, + base: u64, + coherent_override: bool, + event: Option, + routes: Vec<(StreamId, u32)>, + reserved: Vec<(u64, u64)>, + devices: &[PciDevice], +) -> Option { + let refused = |why: core::fmt::Arguments<'_>| log!("IOMMU: the SMMUv3 at {base:#x} is left aborting: {why}"); + let (idr0, idr1, idr5) = (regs.read(reg::IDR0), regs.read(reg::IDR1), regs.read(reg::IDR5)); + let unit = match reg::probe(idr0, idr1, idr5, coherent_override) { + Ok(unit) => unit, + Err(lacks) => { + refused(format_args!("it lacks {lacks:?} (IDR0 {idr0:#x}, IDR1 {idr1:#x}, IDR5 {idr5:#x})")); + return None; + } + }; + // Every table and leaf below is in memory this bounds, so none of + // `toyos_smmu`'s answers past the unit's output size can come back. + let top = pmm::top(); + let last_page = Phys::new((top - 1) & !(crate::mm::PAGE_SIZE - 1)); + if last_page.and_then(|page| toyos_smmu::table::next(page, &unit)).is_none() { + refused(format_args!("memory reaches {top:#x}, past what it outputs (IDR5 {idr5:#x})")); + return None; + } + let Some(event) = event else { + refused(format_args!("the IORT gives its event queue no wired interrupt")); + return None; + }; + let highest = routes.iter().map(|(_, stream)| *stream).max().unwrap_or(0); + let log2 = u32::BITS - highest.leading_zeros(); + if log2 > STREAMS_LOG2_MAX || log2 > u32::from(unit.stream_bits) { + refused(format_args!("StreamID {highest:#x} needs a table of 2^{log2} entries")); + return None; + } + if let Err(limit) = super::irqchip::route_iommu_events(event.get()) { + refused(format_args!("its event interrupt {event} is no SPI the distributor takes below {limit}")); + return None; + } + + let mut memory = Memory::new(); + let table = memory.alloc(64 << log2); + let streams = window(table, 64 << log2); + for stream in 0..1u64 << log2 { + for (i, word) in Ste::ABORT.words().iter().enumerate() { + streams.write_u64(stream * 64 + 8 * i as u64, *word); + } + } + let (commands_log2, events_log2) = + (unit.command_queue_log2.min(COMMANDS_LOG2), unit.event_queue_log2.min(EVENTS_LOG2)); + let commands_at = memory.alloc(16 << commands_log2); + let events_at = memory.alloc(32 << events_log2); + let (Some((table_base, table_cfg)), Some((commands_base, commands)), Some((events_base, events))) = ( + Phys::new(table).and_then(|at| unit.stream_table(at, log2 as u8)), + Phys::new(commands_at).and_then(|at| unit.command_queue(at, commands_log2)), + Phys::new(events_at).and_then(|at| unit.event_queue(at, events_log2)), + ) else { + unreachable!("SMMU: each is aligned to its size, inside the unit's sizes, and below its output size"); + }; + + regs.write(reg::CR1, reg::CR1_WRITE_BACK); + regs.write(reg::CR2, reg::CR2_RECORD_PRIVATE); + regs.write64(reg::STRTAB_BASE, table_base); + regs.write(reg::STRTAB_BASE_CFG, table_cfg); + regs.write64(reg::CMDQ_BASE, commands_base); + regs.write(reg::CMDQ_PROD, 0); + regs.write(reg::CMDQ_CONS, 0); + regs.control(reg::CR0, reg::CR0_CMDQEN, "its command queue enabled"); + let mut live = Live { + regs, + unit, + commands, + queue: window(commands_at, 16 << commands_log2), + prod: 0, + streams, + memory, + routes, + domains: Vec::new(), + reserved, + }; + live.issue(&[Command::ForgetAll, Command::InvalidateAll]); + + regs.write64(reg::EVENTQ_BASE, events_base); + regs.write(reg::EVENTQ_PROD, 0); + regs.write(reg::EVENTQ_CONS, 0); + fault::arm(regs, window(events_at, 32 << events_log2), events, devices, &live.routes); + regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN, "its event queue enabled"); + regs.control(reg::IRQ_CTRL, reg::IRQ_EVENTQ, "its event interrupt enabled"); + regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN | reg::CR0_SMMUEN, "SMMUEN"); + + log!( + "IOMMU: SMMUv3 at {base:#x} armed, CR0ACK {:#x}: {} functions' streams in a table of {}, every entry \ + aborting; a CMD_SYNC consumed; events on SPI {event}", + regs.read(reg::CR0ACK), + live.routes.len(), + 1u32 << log2, + ); + Some(live) +} + +pub mod interrupt { + use crate::iommu::{Refused, StreamId}; + + pub struct Msi { + pub address: u32, + pub data: u32, + } + + pub struct Pin { + pub low: u32, + pub high: u32, + } + + /// The SMMUv3 remaps no interrupt: a message is an ITS's to translate, + /// and nothing here drives one. + pub fn is_armed() -> bool { + false + } + + pub fn msi(_source: StreamId, _vector: u8, _dest: u32) -> Result { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn claim(_slot: usize, _source: StreamId, _vector: u8) -> Msi { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn release(_slot: usize, _source: StreamId) { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } + + pub fn pin(_apic_id: u8, _vector: u8, _dest: u32, _level: bool) -> Result { + unreachable!("no interrupt remapping on this machine, and `is_armed` said so") + } +} diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs new file mode 100644 index 00000000000..ed8859d3984 --- /dev/null +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -0,0 +1,105 @@ +//! `smmu-selftest`: QEMU's `iommu-testdev` (1b36:0005), whose one register +//! read makes a DMA write of a known word at a device address and reads it +//! back at a physical one, asked three things of the armed unit: +//! +//! 1. its write, on the entry every stream starts with, is refused; +//! 2. on a domain of its own, a write at an address the domain maps lands at +//! the memory mapped there and nowhere else; +//! 3. a write at an address the domain does not map is refused and recorded: +//! the event interrupt reaches the handler, which names it and, the +//! function being no process's, halts the machine. +//! +//! Each says what it saw on its own line; the harness judges them, and the +//! third by the handler's. + +use crate::drivers::pci::PciDevice; +use crate::iommu::OwnSpace; +use crate::log; +use crate::mm::policy::MmioPolicy; +use crate::mm::{Mmio, PAGE_2M}; +use crate::time::{Duration, Tripwire}; + +/// `hw/misc/iommu-testdev.h`: Red Hat's vendor and its test device. +const VENDOR: u16 = 0x1b36; +const DEVICE: u16 = 0x0005; + +/// Its BAR0: reading `TRIGGERING` runs an armed DMA; `GVA` is the device +/// address written, `GPA` the physical one read back, `LEN` the bytes, +/// `DBELL` bit 0 arms, and `RESULT` says how it went. +const TRIGGERING: u64 = 0x00; +const GVA_LO: u64 = 0x04; +const GVA_HI: u64 = 0x08; +const LEN: u64 = 0x0c; +const RESULT: u64 = 0x10; +const DBELL: u64 = 0x14; +const GPA_LO: u64 = 0x1c; +const GPA_HI: u64 = 0x20; +const BAR_BYTES: u64 = 0x1000; + +/// `RESULT`: the word landed at `GPA`; the write was refused. +const LANDED: u32 = 0; +const REFUSED: u32 = 0xdead_0002; + +/// How long the event is given to reach this CPU once the write is refused. +const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( + Duration::from_secs(1), + "the unit records the event and pulses its interrupt while the refused write is still being made", +); + +/// The device's write of four bytes at `at`, read back at `phys`: its `RESULT`. +fn write(regs: Mmio, at: u64, phys: u64) -> u32 { + regs.write_u32(GVA_LO, at as u32); + regs.write_u32(GVA_HI, (at >> 32) as u32); + regs.write_u32(GPA_LO, phys as u32); + regs.write_u32(GPA_HI, (phys >> 32) as u32); + regs.write_u32(LEN, 4); + regs.write_u32(DBELL, 1); + regs.read_u32(TRIGGERING); + regs.read_u32(RESULT) +} + +pub(super) fn run(devices: &[PciDevice]) { + let device = devices + .iter() + .find(|d| d.is_id(VENDOR, DEVICE)) + .expect("smmu-selftest: no iommu-testdev (1b36:0005) on this machine"); + let bar = device.memory_bar(0).expect("smmu-selftest: the iommu-testdev's BAR0 is unassigned").address(); + let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); + // No Bus Master Enable: the device writes through the unit's address + // space whatever its `COMMAND` says. + device.enable_memory_space(); + let page = crate::mm::pmm::alloc_page().expect("smmu-selftest: no page to aim at"); + let phys = page.direct_map().phys(); + let function = crate::iommu::StreamId::pci(device.bus, device.dev, device.func); + + let aborted = write(regs, phys, phys); + log!( + "smmu-selftest: {function}'s write at {phys:#x}, on the entry every stream starts with, answered {aborted:#x}: {}", + if aborted == REFUSED { "refused" } else { "FAIL" } + ); + + let (space, at) = OwnSpace::create(PAGE_2M).expect("smmu-selftest: a domain on the armed unit"); + space.place(at, phys, PAGE_2M).expect("smmu-selftest: the domain's own first leaf"); + space.attach(device.bus, device.dev, device.func); + let landed = write(regs, at + 0x40, phys + 0x40); + log!( + "smmu-selftest: {function}'s write at {:#x}, mapped to {:#x}, answered {landed:#x}: {}", + at + 0x40, + phys + 0x40, + if landed == LANDED { "landed there" } else { "FAIL" } + ); + + let unmapped = at + PAGE_2M + 0x40; + log!("smmu-selftest: {function} writes at {unmapped:#x}, which its domain does not map"); + let before = super::fault::recorded(); + let refused = write(regs, unmapped, phys + 0x40); + let deadline = crate::clock::nanos_since_boot() + EVENT_TIMEOUT.nanos(); + while super::fault::recorded() == before { + assert!( + crate::clock::nanos_since_boot() < deadline, + "smmu-selftest: FAIL: the write at {unmapped:#x} answered {refused:#x}, and no event reached this CPU within {EVENT_TIMEOUT}" + ); + core::hint::spin_loop(); + } + panic!("smmu-selftest: FAIL: the handler read the event of a function no process drives and the machine went on"); +} diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 2ac9947151c..41881d73319 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -219,6 +219,14 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { storm::sgi(); irqchip::end(intid); } + // Zero until routed, which the kick's arm above takes first. + intid if intid == irqchip::iommu_events() => { + percpu::irq_took(Source::DmaFault); + percpu::preempt_count_up(); + crate::iommu::fault_interrupt(); + percpu::preempt_count_down(); + irqchip::end(intid); + } _ => { percpu::irq_took(Source::Unclaimed); UNCLAIMED.fetch_add(1, Relaxed); diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 6df69de4ce2..8c0339617d9 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -727,13 +727,18 @@ pub enum Profile { /// `HVC`, as under HVF. `virt_el1_smp`'s machine while a boot's last word /// can miss the console under HVF. VirtTcg, + /// [`Profile::Virt`] with its SMMUv3 and QEMU's `iommu-testdev`, a function + /// that writes where it is told to through the unit. + VirtSmmu, } impl Profile { /// The architecture this machine is. pub fn arch(self) -> Arch { match self { - Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg => Arch::Aarch64, + Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg | Self::VirtSmmu => { + Arch::Aarch64 + } Self::Headless | Self::HeadlessNoIommu | Self::HeadlessE1000e @@ -855,6 +860,8 @@ struct Shape { /// profile because absence is a shape and because the unit's own /// capabilities are what the kernel reads at boot. iommu: Option, + /// `virt`'s SMMUv3, which is a machine property rather than a device. + smmu: Smmu, /// A virtio-rng, which is firmware's alone: edk2's driver puts /// `EFI_RNG_PROTOCOL` behind it for the loader's seed, and the kernel /// drives no such device. `virt` has it because an HVF guest's CPU has no @@ -863,6 +870,15 @@ struct Shape { rng: bool, } +/// Whether `virt` has its SMMUv3. With it comes QEMU's `iommu-testdev`, whose +/// writes are the only DMA a guest of this suite can aim at an address of its +/// choosing: a unit no function writes through is a unit no test reads. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Smmu { + Absent, + WithTestdev, +} + /// Where a machine's image and its DATA are. A size is stated because a /// structure sized per device block is bounded by it and by nothing else; the /// backing file is sparse either way. @@ -907,6 +923,7 @@ impl Profile { match self { Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg => Self::Virt.shape(), Self::VirtNoRng => Shape { rng: false, ..Self::Virt.shape() }, + Self::VirtSmmu => Shape { smmu: Smmu::WithTestdev, ..Self::Virt.shape() }, Self::Virt => Shape { vga: "std", panel: None, @@ -916,6 +933,7 @@ impl Profile { usb: &[], storage: Storage::Stick { nvme_bytes: 0 }, iommu: None, + smmu: Smmu::Absent, rng: true, }, Self::Headless => Shape { @@ -927,6 +945,7 @@ impl Profile { usb: &["usb-kbd,bus=xhci.0"], storage: Storage::Disk { data_bytes: NVME_SMALL }, iommu: Some(IOMMU_DEFAULT), + smmu: Smmu::Absent, rng: false, }, Self::Metal => Shape { @@ -942,6 +961,7 @@ impl Profile { usb: &[], storage: Storage::Stick { nvme_bytes: NVME_SMALL }, iommu: Some(IOMMU_DEFAULT), + smmu: Smmu::Absent, rng: false, }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, @@ -2056,15 +2076,22 @@ fn qemu_command( // needs the userspace half of the irqchip, and a machine with no unit has // no reason to be built differently from the one it has always been. let mut machine = match arch { - Arch::X86_64 => arch.machine().to_string(), + Arch::X86_64 => { + assert!(shape.smmu == Smmu::Absent, "a q35 has no SMMUv3"); + arch.machine().to_string() + } Arch::Aarch64 => { // The unit a profile declares is VT-d, which `virt` has none of. assert!(shape.iommu.is_none(), "`virt` has no VT-d"); - match options.profile { + let machine = match options.profile { Profile::VirtEl2 | Profile::VirtEl2NoVhe => { format!("{},gic-version=3,virtualization=on", arch.machine()) } _ => format!("{},gic-version=3", arch.machine()), + }; + match shape.smmu { + Smmu::Absent => machine, + Smmu::WithTestdev => format!("{machine},iommu=smmuv3"), } } }; @@ -2184,6 +2211,9 @@ fn qemu_command( if shape.rng { qemu.arg("-device").arg("virtio-rng-pci"); } + if shape.smmu == Smmu::WithTestdev { + qemu.arg("-device").arg("iommu-testdev"); + } // The NIC before the virtio block, so a profile that has one and not the // other still creates it after the unit and before everything else. diff --git a/tests/toyos.rs b/tests/toyos.rs index 562ca05ee8a..b906c498d65 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -279,6 +279,7 @@ const SCREEN_TESTS: &[(&str, qemu::Profile)] = &[ // and FP, which no boot under HVF runs. ("virt_jobs_at_el2", qemu::Profile::VirtEl2), ("virt_random_differs", qemu::Profile::Virt), + ("virt_smmu", qemu::Profile::VirtSmmu), ("virt_no_seed_refused", qemu::Profile::VirtNoRng), ]; @@ -2009,6 +2010,76 @@ fn virt_no_seed_refused(profile: qemu::Profile, test_config: &Path) -> Result<() Ok(()) } +/// The SMMUv3 armed from the IORT, judged by what QEMU's `iommu-testdev` can +/// and cannot write through it (`kernel/src/arch/aarch64/smmu/selftest.rs`): +/// `GBPA` read back aborting, every stream's entry aborting, a `CMD_SYNC` +/// consumed; the device's write refused on the entry it starts with, landing +/// where its own domain maps it, and refused where the domain maps nothing — +/// that one recorded, its event taken on the wired SPI and named, and the +/// machine halted for a function no process drives. +fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { + const FAULT: &str = "iommu: DMA FAULT owner=kernel"; + let options = BootOptions { + profile, + kernel_params: &["smmu-selftest"], + ready_marker: "control registers: SCTLR_EL1=", + ..Default::default() + }; + let argv = qemu::profile_argv(&options); + for want in ["iommu=smmuv3", "iommu-testdev"] { + if !argv.iter().any(|a| a.contains(want)) { + return Err(format!("the machine has no {want}: {argv:?}")); + } + } + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[], options); + let rest = qemu.drain_until(Duration::from_secs(30), |l| l.contains(FAULT)); + let serial = format!("{}\n{rest}", qemu.boot_log()); + let line = |want: &str| serial.lines().find(|l| l.contains(want)).map(str::to_string); + let Some(armed) = line("armed, CR0ACK") else { + return Err(format!("the SMMUv3 was never armed\nserial:\n{serial}")); + }; + for want in ["every entry aborting", "a CMD_SYNC consumed", "events on SPI 106"] { + if !armed.contains(want) { + return Err(format!("{want:?} not in {armed:?}")); + } + } + eprintln!(" [virt] {armed}"); + // `SMMU_GBPA.ABORT`, bit 20, as the unit reads it back. + let gbpa = line(", every transaction aborts while SMMUEN is clear") + .and_then(|l| l.split_once(": GBPA 0x").and_then(|(_, v)| u32::from_str_radix(v.split(',').next()?, 16).ok())); + match gbpa { + Some(gbpa) if gbpa & 1 << 20 != 0 => eprintln!(" [virt] GBPA {gbpa:#x}: ABORT"), + _ => return Err(format!("GBPA does not read back aborting: {gbpa:?}\nserial:\n{serial}")), + } + for (said, verdict) in [("on the entry every stream starts with", "refused"), ("mapped to", "landed there")] { + let Some(found) = line(said) else { + return Err(format!("the selftest never said {said:?}\nserial:\n{serial}")); + }; + eprintln!(" [virt] {found}"); + if !found.ends_with(verdict) { + return Err(format!("{found}\nserial:\n{serial}")); + } + } + let Some(aimed) = line("which its domain does not map") else { + return Err(format!("the selftest never aimed a write past its domain\nserial:\n{serial}")); + }; + let aimed_at = aimed.split_once("smmu-selftest: ").and_then(|(_, said)| said.split_once(" writes at 0x")); + let Some((function, Ok(at))) = aimed_at.map(|(function, rest)| { + (function, u64::from_str_radix(rest.split(',').next().unwrap_or_default(), 16)) + }) else { + return Err(format!("{aimed:?} names no function and address")); + }; + let Some(fault) = line(FAULT) else { + return Err(format!("no event reached the handler\nserial:\n{serial}")); + }; + eprintln!(" [virt] {fault}"); + let named = format!("stream={function} addr={at:#018x} access=write"); + if !fault.contains(&named) || !fault.ends_with("F_TRANSLATION") { + return Err(format!("the event does not name {named:?} and F_TRANSLATION: {fault}")); + } + Ok(()) +} + /// Everything the PL011 has carried on `qemu`'s boot: the capture each /// [`judge_virt_job`] after the first goes on from, since a drain reads past /// the marker it waited for. @@ -2733,6 +2804,7 @@ fn run_screen_test(name: &str, profile: qemu::Profile, test_config: &Path) -> Re "virt_jobs_at_el2" => virt_jobs_at_el2(profile), "virt_random_differs" => virt_random_differs(profile), "virt_no_seed_refused" => virt_no_seed_refused(profile, test_config), + "virt_smmu" => virt_smmu(profile, test_config), "virt_mask_windows" => virt_mask_windows(profile), "virt_irq_storm" => { // The CPU floods itself with SGIs until the timer has fired a diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs index dbfad87ce37..63106680031 100644 --- a/toyos-smmu/src/queue.rs +++ b/toyos-smmu/src/queue.rs @@ -194,6 +194,34 @@ pub enum Code { Other(u8), } +impl Code { + /// The event's mnemonic, as §7.3's headings name it. + pub const fn name(self) -> &'static str { + match self { + Self::UnsupportedTransaction => "F_UUT", + Self::BadStream => "C_BAD_STREAMID", + Self::EntryFetch => "F_STE_FETCH", + Self::BadEntry => "C_BAD_STE", + Self::AtsRequest => "F_BAD_ATS_TREQ", + Self::StreamDisabled => "F_STREAM_DISABLED", + Self::TranslatedForbidden => "F_TRANSL_FORBIDDEN", + Self::BadSubstream => "C_BAD_SUBSTREAMID", + Self::ContextFetch => "F_CD_FETCH", + Self::BadContext => "C_BAD_CD", + Self::WalkAbort => "F_WALK_EABT", + Self::Translation => "F_TRANSLATION", + Self::AddressSize => "F_ADDR_SIZE", + Self::AccessFlag => "F_ACCESS", + Self::Permission => "F_PERMISSION", + Self::TlbConflict => "F_TLB_CONFLICT", + Self::ConfigurationConflict => "F_CFG_CONFLICT", + Self::PageRequest => "E_PAGE_REQUEST", + Self::VmsFetch => "F_VMS_FETCH", + Self::Other(_) => "unnamed", + } + } +} + /// What a device tried, where the record says: the four translation-related /// faults carry it (§7.3.13 to §7.3.16). #[derive(Clone, Copy, PartialEq, Eq, Debug)] diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs index f25d80ae04d..ff58b28810b 100644 --- a/toyos-smmu/tests/encodings.rs +++ b/toyos-smmu/tests/encodings.rs @@ -522,12 +522,41 @@ fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { ]; for number in 0..=u8::MAX { let decoded = event(record(number, 7, 0x1000, false)); + assert_eq!(decoded.code.name() == "unnamed", !named.iter().any(|(n, _)| *n == number), "event {number:#x}"); let code = named.iter().find(|(n, _)| *n == number).map_or(Code::Other(number), |(_, code)| *code); assert_eq!((decoded.stream, decoded.code), (7, code), "event {number:#x}"); assert_eq!(decoded.attempt.is_some(), (0x10..=0x13).contains(&number), "event {number:#x}"); } } +#[test] +fn each_event_is_named_by_its_own_mnemonic() { + let mnemonics = [ + (0x01, "F_UUT"), + (0x02, "C_BAD_STREAMID"), + (0x03, "F_STE_FETCH"), + (0x04, "C_BAD_STE"), + (0x05, "F_BAD_ATS_TREQ"), + (0x06, "F_STREAM_DISABLED"), + (0x07, "F_TRANSL_FORBIDDEN"), + (0x08, "C_BAD_SUBSTREAMID"), + (0x09, "F_CD_FETCH"), + (0x0A, "C_BAD_CD"), + (0x0B, "F_WALK_EABT"), + (0x10, "F_TRANSLATION"), + (0x11, "F_ADDR_SIZE"), + (0x12, "F_ACCESS"), + (0x13, "F_PERMISSION"), + (0x20, "F_TLB_CONFLICT"), + (0x21, "F_CFG_CONFLICT"), + (0x24, "E_PAGE_REQUEST"), + (0x25, "F_VMS_FETCH"), + ]; + for (number, mnemonic) in mnemonics { + assert_eq!(event(record(number, 7, 0x1000, false)).code.name(), mnemonic, "event {number:#x}"); + } +} + // --- the stage 1 tables (DDI 0487 M.d §D8.3.1) ----------------------------- /// A device address whose four table indexes are 0x1a5, 0x0f0, 0x00f and 0x155. From f24e70907ca290d5157d234e5a3bc24bea14505e Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 10:30:55 +0200 Subject: [PATCH 2/4] One fault policy and one address window for both IOMMU backends; every SMMUv3 the IORT names aborts before a refusal, its interrupts write nowhere, and every unrouted stream has one answer The review of round 1 found the SMMUv3 bring-up's fault policy and domain window to be second declarations of VT-d's, and four holes in the unit's programming. This answers each. One home for what both backends decide alike, above kernel/src/arch/: - kernel/src/iommu/fault.rs holds the policy VT-d's handler applied: the enumerated functions published once, BME cleared first, the per-function and per-unit counts, the first faulting function, owner-or-halt, and the `iommu: DMA FAULT` line. vtd/fault.rs keeps its recording registers and reason names; smmu/fault.rs keeps its event queue. The line's format is VT-d's, unchanged; the SMMUv3's now carries `reason=0x..` and VT-d's `domain=` answer too. FIRST's address, reason and unit were written and never read, and go; pci::NO_FUNCTION, their only sentinel, goes with them. - kernel/src/iommu/window.rs holds a domain's addresses: the floor a quarter up what the unit translates and above memory, the ceiling under the first reserved window over it, reserve and handed_out, and both backends' compile-time checks. VT-d keeps translatable_bits (SAGAW against MGAW). issues/each-iommu-backend-keeps-its-own-domain-address-window.md is met: `rg -n 'fn handed_out|fn ceiling' kernel/src/arch` prints nothing. The SMMUv3: - Every SMMUv3 the IORT names gets GBPA.ABORT before a machine with more than one is refused: GBPA's reset value is IMPLEMENTATION DEFINED. - IRQ_CTRL is cleared, acked, and GERROR_IRQ_CFG0, EVENTQ_IRQ_CFG0 and, where IDR0.PRI is set, PRIQ_IRQ_CFG0 are zeroed before anything else, as Linux's arm_smmu_setup_msis does: a non-zero ADDR makes the interrupt an MSI write no STE governs, each resets UNKNOWN, and each is writable only with its interrupt off (IHI 0070 H.a 3.18.2, 6.3.21, 6.3.30, 6.3.34). - Every stream no enumerated function is routed from is aborted and recorded, inside the stream table (an entry left invalid: C_BAD_STE) or past it (CR2.RECINVSID: C_BAD_STREAMID), and halts the machine, as VT-d's unenumerated requester does. Silence for both was tried first and is not holdable: QEMU 11.1.1's hw/arm/smmuv3.c stores CR2 and never reads RECINVSID, so an out-of-range StreamID is recorded there whatever CR2 says; measured, a C_BAD_STREAMID with RECINVSID clear. - The event queue drains until it reads empty: the unit raises its interrupt only as the queue goes from empty to non-empty (3.18.2), so a record left behind raises nothing. An overflow is reported where it is seen, mid-drain included. - The edge trigger now cites IHI 0070 H.a 3.18.2 ("interrupt outputs are effectively edge-triggered") and 12.4 ("required to be edge-triggered or MSIs"). The selftest takes a second iommu-testdev, below the first, which the actuator leaves unrouted, and writes four times: on the entry the routed one starts with (refused, unrecorded), on its domain (lands), there again after `unmap` (refused, recorded), and the unrouted one inside the table (refused, recorded). The last two are made with interrupts masked so one drain reads both; virt_smmu reads them as the unit's first and second events. The write after `unmap` reds without its CMD_TLBI_NH_ASID: QEMU's SMMUv3 keeps an IOTLB. Issues: the SMMUv3's GERROR is filed as read and never delivered, owned by stage 6's ITS work; a late write from a released function halting the machine is filed against stage 6's claim through an SMMUv3 domain; the descriptor half of the SMMUv3-and-ITS issue is re-owned to stage 4's owed break-before-make work on paging.rs, and GICD_TYPER's two decoders are added to its distributor half. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...m-a-released-function-halts-the-machine.md | 34 +++ ...end-keeps-its-own-domain-address-window.md | 30 -- ...e-what-the-aarch64-kernel-also-declares.md | 21 +- ...bal-errors-are-read-and-never-delivered.md | 32 ++ kernel/src/actuator.rs | 7 +- kernel/src/arch/aarch64/irqchip.rs | 6 +- kernel/src/arch/aarch64/smmu/domain.rs | 113 +------ kernel/src/arch/aarch64/smmu/fault.rs | 201 ++---------- kernel/src/arch/aarch64/smmu/mod.rs | 102 +++++-- kernel/src/arch/aarch64/smmu/selftest.rs | 97 ++++-- kernel/src/arch/x86_64/vtd/domain.rs | 13 +- kernel/src/arch/x86_64/vtd/fault.rs | 285 ++---------------- kernel/src/arch/x86_64/vtd/mod.rs | 2 +- kernel/src/arch/x86_64/vtd/table.rs | 205 +++---------- kernel/src/drivers/pci.rs | 3 - kernel/src/iommu/fault.rs | 272 +++++++++++++++++ kernel/src/iommu/mod.rs | 7 +- kernel/src/iommu/window.rs | 156 ++++++++++ tests/common/qemu.rs | 13 +- tests/toyos.rs | 81 +++-- toyos-smmu/src/queue.rs | 4 +- toyos-smmu/src/unit.rs | 14 +- toyos-smmu/tests/encodings.rs | 9 +- 23 files changed, 846 insertions(+), 861 deletions(-) create mode 100644 issues/a-late-write-from-a-released-function-halts-the-machine.md delete mode 100644 issues/each-iommu-backend-keeps-its-own-domain-address-window.md create mode 100644 issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md create mode 100644 kernel/src/iommu/fault.rs create mode 100644 kernel/src/iommu/window.rs diff --git a/issues/a-late-write-from-a-released-function-halts-the-machine.md b/issues/a-late-write-from-a-released-function-halts-the-machine.md new file mode 100644 index 00000000000..80b27463406 --- /dev/null +++ b/issues/a-late-write-from-a-released-function-halts-the-machine.md @@ -0,0 +1,34 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# A late write from a released function halts the machine + +`kernel/src/pcidev/mod.rs`'s `tear_down` clears the function's ownership +(`crate::iommu::note_user_owned(…, None)`) before its reset has quiesced what +it already issued: Bus Master Enable is cleared first, so it starts nothing +new, and its grants stay mapped until it is quiet, so nothing it issued lands +in a page handed on. A transaction already in flight that faults after the +ownership is cleared — at an address a grant never covered, or one an +`unmap` before the release took back — is read by the fault handler as a +function's no process drives: `owner=kernel`, and `crate::iommu::fault::conclude` +halts the machine. One process's bug then takes the machine down, which is the +thing a claim exists to prevent. + +This holds on x86-64 under VT-d today, and on AArch64 it is reachable once a +function can be claimed through an SMMUv3 domain. No test makes a write +outlive its function's release; QEMU's devices finish a DMA inside the access +that starts it, so no guest of this suite can. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, its claim of a function +through an SMMUv3 domain (the stage's exit: netd claims its NIC), which makes +the AArch64 half reachable and lands before it. + +**Exit.** A fault on a function between its release and its reset's quiet is +handed to the released claim's record and not to the kernel, the ownership +cleared only once the reset is quiet; a test that makes such a write — a +claimed function's grant unmapped while it is aimed there, its claim released +before the write lands — sees the record and a machine still running, and +reds with the ownership cleared where it is today. diff --git a/issues/each-iommu-backend-keeps-its-own-domain-address-window.md b/issues/each-iommu-backend-keeps-its-own-domain-address-window.md deleted file mode 100644 index 277b5021d83..00000000000 --- a/issues/each-iommu-backend-keeps-its-own-domain-address-window.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-10 ---- - -# Each IOMMU backend keeps its own domain address window - -Where a device domain's addresses start, where they end and which of them -are handed out is one policy — a floor a quarter of the way up what the unit -translates and above all memory, a ceiling under the first root-bridge window -or reserved region over the floor, an address never handed out twice — and it -has two declarations: `Domain::new`, `Domain::reserve`, `Domain::handed_out` -and `ceiling` in `kernel/src/arch/x86_64/vtd/table.rs`, and `FLOOR`, -`Addresses` and `ceiling` in `kernel/src/arch/aarch64/smmu/domain.rs`. Nothing -holds the two equal; each carries its own compile-time checks. - -The SMMUv3 came in with the x86-64 side outside its fence, so its copy was -written beside VT-d's rather than hoisted above both. - -**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, whose ITS stage edits -`kernel/src/iommu/mod.rs` and both backends' seam. - -**Exit.** One declaration above `kernel/src/arch/`, read by both backends: - -``` -rg -n 'fn handed_out|fn ceiling' kernel/src/arch -``` - -prints nothing. diff --git a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md index 06244c6ad8e..0f17a3bdc83 100644 --- a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md +++ b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md @@ -28,14 +28,20 @@ the two registers' offsets are `kernel/src/arch/aarch64/irqchip.rs`'s: one register's fields have two crates. `toyos_its::TRANSLATION_FRAME` and `toyos_gicv3::FRAME` are both the GIC's 64 KiB frame. -**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which -writes these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns -the descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for -every CPU, owns the redistributor. +**The distributor.** `toyos-its/src/lpi.rs`'s `Layout::new` decodes +`GICD_TYPER`'s `LPIS`, `IDbits` and `num_LPIs`, and +`kernel/src/arch/aarch64/irqchip.rs`'s `route_iommu_events` decodes its +`ITLinesNumber`: one register's fields have two homes. + +**Owner.** `issues/toyos-runs-on-arm64.md`. The descriptor is stage 4's, its +owed break-before-make ordering of a live entry's replacement, which rewrites +how `kernel/src/arch/aarch64/paging.rs` writes an entry. The redistributor and +the distributor are stage 6's ITS work, which edits `irqchip.rs` and reads +`GICR_TYPER` for every CPU and `GICD_TYPER` for its LPIs. **Exit.** Each bit and the level index has one declaration that the kernel's tables, the loader's and the unit's are all written from, and each -redistributor register's fields have one crate: +redistributor or distributor register's fields have one home: ``` rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_SHAREABLE|AF|PXN|UXN): u64' \ @@ -43,5 +49,6 @@ rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_ rg -n 'fn index' kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs ``` -print nothing, and `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one -directory. +print nothing, `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one +directory, and `rg -l 'ITLinesNumber|IDbits|num_LPIs' toyos-its/src kernel/src +kernel/gicv3/src` names files of one crate. diff --git a/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md b/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md new file mode 100644 index 00000000000..462fdf12d04 --- /dev/null +++ b/issues/the-smmuv3s-global-errors-are-read-and-never-delivered.md @@ -0,0 +1,32 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# The SMMUv3's global errors are read and never delivered + +`kernel/src/arch/aarch64/smmu/` reads `SMMU_GERROR` against `SMMU_GERRORN` +only where something else brings it there: in every bounded wait on a register +or the command queue, and after each drain of the event queue. Its own wired +interrupt, the IORT's `GERR` GSIV, is routed nowhere and +`SMMU_IRQ_CTRL.GERROR_IRQEN` stays clear, so an error the unit raises between +those reads goes unseen until the next one, and one it raises on a quiet +machine is never seen (IHI 0070 H.a §7.5 lists them): + +- `SFM_ERR`: the unit entered Service Failure Mode, terminating every client + transaction and no longer accessing its queues (§12.3). +- `EVENTQ_ABT_ERR`: a write to the event queue aborted, and events were lost + with no record of them; read only after a drain, which a lost record does + not start. +- `CMDQ_ERR` outside a wait: none can arise there today, since every command + is issued under a wait for its `CMD_SYNC`. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, its ITS work: it edits +`kernel/src/arch/aarch64/irqchip.rs` and `trap.rs`, where a second SPI is +routed and taken. + +**Exit.** `SMMU_IRQ_CTRL.GERROR_IRQEN` is set and the `GERR` SPI reaches a +handler that names each active error and acknowledges it in `SMMU_GERRORN`; +`virt_smmu`, or a test beside it, raises one — a `CMD_SYNC` behind an illegal +command is QEMU's `CMDQ_ERR` — and reds with the routing deleted. diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 5c0f8c1b03e..475067388e6 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -144,9 +144,10 @@ actuators! { #[allow(dead_code)] psci_withheld = "psci-withheld"; - /// Once the SMMUv3 is armed, have QEMU's `iommu-testdev` write on the - /// entry every stream starts with, on a domain at an address it maps, and - /// at one it does not. Judged by `virt_smmu`. + /// Give the `iommu-testdev` off bus 0 no route, and once the SMMUv3 is + /// armed have it and the one on bus 0 write on no domain, then the one on + /// bus 0 on a domain at an address it maps, and there again once it is + /// taken back. Judged by `virt_smmu`. // The SMMUv3 is AArch64's alone, so x86-64 builds an accessor it never reads. #[allow(dead_code)] smmu_selftest = "smmu-selftest"; diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 863ce2afb91..04ec59f72e6 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -299,9 +299,9 @@ pub(super) fn timer_intid() -> u32 { } /// Take SPI `intid` on this CPU at [`PRIORITY`], edge-triggered, as the -/// IOMMU's event interrupt: an SMMUv3's wired interrupts are pulses, and the -/// IORT names no trigger. Refused with the distributor's limit where it takes -/// no such SPI. +/// IOMMU's event interrupt: an SMMUv3's wired interrupts are edge-triggered +/// (IHI 0070 H.a §3.18.2, §12.4), and the IORT names no trigger. Refused +/// with the distributor's limit where it takes no such SPI. pub(super) fn route_iommu_events(intid: u32) -> Result<(), u32> { let gicd = Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME); let limit = (32 * ((gicd.read_u32(GICD_TYPER) & 0x1F) + 1)).min(1020); diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs index 8b438596a54..4b61ff63cb7 100644 --- a/kernel/src/arch/aarch64/smmu/domain.rs +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -1,13 +1,7 @@ //! A device's address space: one context descriptor, tagged with an ASID of //! its own, over stage 1 tables that map 2 MiB blocks and nothing else -//! (`toyos_smmu::table`), and the device addresses it hands out. -//! -//! A domain's addresses start a quarter of the way up the 48-bit input, above -//! all memory, so a descriptor still carrying one names nothing a domain maps -//! by accident; they end below the first root-bridge window over that, which -//! a bridge may route peer-to-peer before the unit sees the request. An -//! address is handed out once: a device holding a stale one reaches whatever -//! took its place, so nothing takes it. +//! (`toyos_smmu::table`), and the device addresses it hands out, over the +//! unit's 48-bit input (`crate::iommu::window`). //! //! A new leaf needs no invalidation — the unit caches no translation that //! faulted — and a removed one is invalidated by the domain's ASID, behind a @@ -20,6 +14,7 @@ use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS}; use toyos_smmu::Asid; use super::{window, Live, UNIT}; +use crate::iommu::window::Window; use crate::iommu::{DomainId, IommuError, Iova, StreamId}; use crate::log; use crate::mm::{PAGE_2M, PAGE_SIZE}; @@ -27,91 +22,13 @@ use crate::mm::{PAGE_2M, PAGE_SIZE}; /// Domain ids start at 1: `DomainId` is never 0. const FIRST: u16 = 1; -/// A quarter of the way up the input. -const FLOOR: u64 = 1 << (INPUT_BITS - 2); - pub(super) struct Domain { asid: Asid, root: Phys<12>, context: Phys<6>, - addresses: Addresses, + addresses: Window, } -/// A domain's device addresses, from [`FLOOR`] to its ceiling. -#[derive(Clone, Copy)] -struct Addresses { - ceiling: u64, - /// The first not yet handed out. - next: u64, -} - -impl Addresses { - /// `bytes`, rounded up to whole leaves, of addresses never handed out. - const fn reserve(&mut self, bytes: u64) -> Option { - let Some(end) = bytes.checked_next_multiple_of(PAGE_2M) else { return None }; - let Some(end) = self.next.checked_add(end) else { return None }; - if end > self.ceiling { - return None; - } - let at = Iova::translated(self.next); - self.next = end; - Some(at) - } - - /// Whether `bytes` at `at` is room [`Self::reserve`] handed out, starting - /// on a leaf it could have returned. - const fn handed_out(&self, at: Iova, bytes: u64) -> bool { - if !at.raw().is_multiple_of(PAGE_2M) || at.raw() < FLOOR { - return false; - } - match bytes.checked_next_multiple_of(PAGE_2M) { - Some(span) => match at.raw().checked_add(span) { - Some(end) => end <= self.next, - None => false, - }, - None => false, - } - } -} - -/// Where a domain's addresses end: under the input, and under the first of -/// `reserved` reaching above [`FLOOR`]; at or below it where one covers it. -const fn ceiling(reserved: &[(u64, u64)]) -> u64 { - let mut ceiling = 1 << INPUT_BITS; - let mut i = 0; - while i < reserved.len() { - let (start, end) = reserved[i]; - if end > FLOOR && start < ceiling { - ceiling = start; - } - i += 1; - } - ceiling -} - -/// [`Addresses`] and [`ceiling`] at their boundaries, at compile time: the -/// binary has no test harness. -const _: () = { - let mut one = Addresses { ceiling: FLOOR + 2 * PAGE_2M, next: FLOOR }; - assert!(matches!(one.reserve(1), Some(at) if at.raw() == FLOOR)); - assert!(one.handed_out(Iova::translated(FLOOR), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - assert!(!one.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); - assert!(!one.handed_out(Iova::translated(FLOOR + 1), 0)); - assert!(!one.handed_out(Iova::translated(u64::MAX - PAGE_2M + 1), PAGE_2M)); - // Past the ceiling, and past the end of the address space, nothing. - assert!(one.reserve(PAGE_2M + 1).is_none()); - assert!(one.reserve(u64::MAX).is_none()); - assert!(matches!(one.reserve(PAGE_2M), Some(at) if at.raw() == FLOOR + PAGE_2M)); - assert!(one.reserve(1).is_none()); - // `virt`'s windows, all below the floor, leave the whole input. - assert!(ceiling(&[(0x1000_0000, 0x3f00_0000), (0x80_0000_0000, 0x100_0000_0000)]) == 1 << INPUT_BITS); - // A window over the floor ends the domain where it starts; one across it leaves nothing. - assert!(ceiling(&[(FLOOR + 4 * PAGE_2M, FLOOR + 8 * PAGE_2M)]) == FLOOR + 4 * PAGE_2M); - assert!(ceiling(&[(FLOOR - PAGE_2M, FLOOR + PAGE_2M)]) < FLOOR); -}; - impl Live { fn domain(&mut self, id: DomainId) -> &mut Domain { &mut self.domains[usize::from(id.raw() - FIRST)] @@ -171,17 +88,7 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { let domains = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; let id = u16::try_from(usize::from(FIRST) + live.domains.len()).map_err(|_| IommuError::DomainsExhausted(domains))?; let asid = live.unit.asid(id).ok_or(IommuError::DomainsExhausted(domains))?; - let top = crate::mm::pmm::top(); - if FLOOR <= top { - return Err(IommuError::WindowBelowMemory { translatable: INPUT_BITS as u8, floor: FLOOR, top }); - } - let ceiling = ceiling(&live.reserved); - let mut addresses = Addresses { ceiling, next: FLOOR }; - // At least one leaf, whatever was asked: a domain with none is no domain. - let first = match addresses.reserve(room) { - Some(first) if ceiling >= FLOOR + PAGE_2M => first, - _ => return Err(IommuError::NoRoom { floor: FLOOR, ceiling, room }), - }; + let (addresses, first) = Window::new(INPUT_BITS as u8, &live.reserved, room)?; let root = Phys::new(live.memory.alloc(PAGE_SIZE)).expect("SMMU: a table below the unit's output size"); let context = Phys::new(live.memory.alloc(64)).expect("SMMU: a descriptor below the unit's output size"); let words = Cd::new(root, asid, &live.unit).expect("SMMU: a table below the unit's output size").words(); @@ -193,10 +100,12 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { descriptor.write_u64(0, words[0]); let domain = Domain { asid, root, context, addresses }; log!( - "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {FLOOR:#x} to {ceiling:#x}", + "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {:#x} to {:#x}", root.get(), context.get(), - asid.get() + asid.get(), + addresses.floor(), + addresses.ceiling() ); live.domains.push(domain); Ok((DomainId::new(id), first)) @@ -209,7 +118,7 @@ pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { let mut held = UNIT.lock(); let live = held.as_mut().expect("a domain exists only on an armed unit"); let domain = live.domain(id); - let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling))?; + let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling()))?; let root = domain.root; for offset in (0..bytes).step_by(PAGE_2M as usize) { live.put(root, at.raw() + offset, phys + offset); @@ -271,6 +180,6 @@ pub fn attach(function: StreamId, id: DomainId) { let context = live.domain(id).context; let ste = Ste::stage1(context, &live.unit).expect("SMMU: a descriptor below the unit's output size"); live.write_entry(stream, ste); - super::fault::attached(stream, id.raw()); + crate::iommu::fault::attached(function, id.raw()); log!("iommu: {function} moves to domain{}", id.raw()); } diff --git a/kernel/src/arch/aarch64/smmu/fault.rs b/kernel/src/arch/aarch64/smmu/fault.rs index a7b637c07b1..8c273b10cdc 100644 --- a/kernel/src/arch/aarch64/smmu/fault.rs +++ b/kernel/src/arch/aarch64/smmu/fault.rs @@ -3,54 +3,35 @@ //! //! The handler allocates nothing and takes no lock: what it reads is //! published once, whole, before the interrupt is enabled, and what it writes -//! is atomics. For each record it first clears Bus Master Enable on the -//! function the record names — a function that cannot master the bus raises -//! no second event, which is the ceiling on a storm (PCI 3.0 §6.2.2, bit 2 of -//! `COMMAND`) — then counts it and hands it to whoever drives the function. +//! is atomics. What a record ends is `crate::iommu::fault`'s, the policy every +//! backend applies; this reads the queue into it. //! -//! **Who that is decides what the record ends.** A stream a process drives -//! has an owner: `pcidev` is told, that claim refuses every later call, and -//! the machine goes on. A stream no process drives has nobody to hand it to, -//! so after the line the machine halts, as x86-64's VT-d handler does. +//! The unit pulses its interrupt only as the queue goes from empty to +//! non-empty (IHI 0070 H.a §3.18.2), so a drain ends only on a queue it read +//! empty: a record left behind raises nothing. use alloc::boxed::Box; -use alloc::vec::Vec; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; use toyos_smmu::queue::{event, Events}; use toyos_smmu::unit as reg; use super::Registers; -use crate::drivers::pci::{self, PciDevice}; +use crate::iommu::fault::{self as policy, Access, Fault, Who}; use crate::iommu::StreamId; use crate::log; -use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; - -/// A `pcidev` slot number no claim has: this function is not driven by one. -const NO_SLOT: u32 = u32::MAX; - -/// One function the unit translates for. -struct Function { - function: StreamId, - stream: u32, - /// Physical base of its config window, through which an event clears `BME`. - config: u64, - domain: AtomicU32, - faults: AtomicU32, - /// The `pcidev` slot a process drives this function on, or [`NO_SLOT`]. - user_slot: AtomicU32, -} +use crate::mm::Mmio; /// What the handler reads, published once by [`arm`]. struct Armed { regs: Registers, queue: Mmio, events: Events, - functions: Box<[Function]>, + /// Each enumerated function the IORT routes through the unit, and its + /// StreamID. + routes: Box<[(StreamId, u32)]>, /// Events this unit has recorded for the life of the boot. - count: AtomicU32, - /// The StreamID of the first, or `u32::MAX`. - first: AtomicU32, + faults: AtomicU32, } static ARMED: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); @@ -62,64 +43,18 @@ fn armed() -> Option<&'static Armed> { } /// Everything the handler reads, before the unit's event interrupt is -/// enabled: the registers, the queue and its indexes, and each function -/// `routes` names. -pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, devices: &[PciDevice], routes: &[(StreamId, u32)]) { - let functions: Vec = routes - .iter() - .map(|&(function, stream)| { - let device = devices - .iter() - .find(|d| StreamId::pci(d.bus, d.dev, d.func) == function) - .expect("a route is an enumerated function's"); - Function { - function, - stream, - config: DirectMap::phys_of(device.config_window().addr() as *const u8), - domain: AtomicU32::new(0), - faults: AtomicU32::new(0), - user_slot: AtomicU32::new(NO_SLOT), - } - }) - .collect(); - let armed = Box::new(Armed { - regs, - queue, - events, - functions: functions.into_boxed_slice(), - count: AtomicU32::new(0), - first: AtomicU32::new(u32::MAX), - }); +/// enabled: the registers, the queue and its indexes, and the routes. +pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, routes: &[(StreamId, u32)]) { + let armed = + Box::new(Armed { regs, queue, events, routes: routes.into(), faults: AtomicU32::new(0) }); let first = ARMED.compare_exchange(core::ptr::null_mut(), Box::leak(armed), Ordering::Release, Ordering::Relaxed); assert!(first.is_ok(), "SMMU: the event handler was armed twice"); } -fn find(stream: u32) -> Option<&'static Function> { - armed()?.functions.iter().find(|f| f.stream == stream) -} - -fn by_function(function: StreamId) -> Option<&'static Function> { - armed()?.functions.iter().find(|f| f.function == function) -} - -/// Record which domain stream `stream` moved to, for the line. -pub(super) fn attached(stream: u32, domain: u16) { - if let Some(function) = find(stream) { - function.domain.store(u32::from(domain), Ordering::Relaxed); - } -} - -/// Record that a process drives `function` on `slot`, or no longer does. -pub fn user_owned(function: StreamId, slot: Option) { - if let Some(function) = by_function(function) { - function.user_slot.store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); - } -} - /// The events the unit has recorded, as the handler has read them. #[cfg(feature = "boot-actuators")] pub(super) fn recorded() -> u32 { - armed().map_or(0, |armed| armed.count.load(Ordering::Relaxed)) + armed().map_or(0, |armed| armed.faults.load(Ordering::Relaxed)) } /// The unit's event interrupt. @@ -127,20 +62,24 @@ pub fn service() { let armed = armed().expect("SMMU: the event interrupt is enabled only once the handler is armed"); let (regs, events) = (armed.regs, armed.events); let mut kernel_owned = 0usize; - let mut prod = regs.read(reg::EVENTQ_PROD); let mut cons = regs.read(reg::EVENTQ_CONS); - if events.overflowed(prod, cons) { - log!("iommu: the SMMUv3's event queue overflowed: earlier events are lost"); - } - // Every record the unit wrote before the last read of `PROD`, one that - // lands after it raising the interrupt again; at most twice a queue's - // worth, so a stream no `BME` stops cannot hold this CPU here. - let mut budget = 2u32 << super::EVENTS_LOG2; - while !events.is_empty(prod, cons) && budget > 0 { - budget -= 1; + let mut prod = regs.read(reg::EVENTQ_PROD); + while !events.is_empty(prod, cons) { + if events.overflowed(prod, cons) { + log!("iommu: the SMMUv3's event queue overflowed: earlier events are lost"); + } let at = events.slot(cons) as u64 * 32; - let record = [0, 8, 16, 24].map(|offset| armed.queue.read_u64(at + offset)); - if record_one(armed, event(record)) { + let record = event([0, 8, 16, 24].map(|offset| armed.queue.read_u64(at + offset))); + let who = match armed.routes.iter().find(|(_, stream)| *stream == record.stream) { + Some(&(function, _)) => Who::Function(function), + None => Who::Stream(record.stream), + }; + let (address, access) = match record.attempt { + Some(attempt) => (attempt.address, if attempt.write { Access::Write } else { Access::Read }), + None => (0, Access::Unrecorded), + }; + let fault = Fault { who, address, access, reason: record.number, name: record.code.name() }; + if policy::report(0, &armed.faults, fault) { kernel_owned += 1; } cons = events.after(cons, prod); @@ -151,79 +90,5 @@ pub fn service() { if errors & reg::GERROR_EVENTQ_ABORT != 0 { log!("iommu: the SMMUv3 could not write its event queue: events are lost (GERROR {errors:#x})"); } - if kernel_owned > 0 { - // A function reached an address this kernel never gave it, and - // nothing here can know what else it did: the line is the report, - // and the machine stops. - crate::drivers::panic_console::capture(); - crate::panic::halt_all_cpus(); - } -} - -/// One record, handed to whoever drives the function it names; `true` where -/// that is nobody. -fn record_one(armed: &Armed, event: toyos_smmu::queue::Event) -> bool { - let function = find(event.stream); - // First, before anything that can be slow: the ceiling on a storm. - if let Some(function) = function { - pci::stop_bus_mastering(config_window(function.config)); - } - let faults = function.map_or(0, |f| f.faults.fetch_add(1, Ordering::Relaxed) + 1); - let count = armed.count.fetch_add(1, Ordering::Relaxed) + 1; - let first = armed.first.compare_exchange(u32::MAX, event.stream, Ordering::AcqRel, Ordering::Relaxed).is_ok(); - let owner = function.map(|f| f.user_slot.load(Ordering::Acquire)).filter(|slot| *slot != NO_SLOT); - if let Some(slot) = owner { - crate::pcidev::note_fault(slot as usize); - } - let (address, access) = match event.attempt { - Some(attempt) => (attempt.address, if attempt.write { "write" } else { "read" }), - None => (0, "none"), - }; - log!( - // `owner=` first: `tests/common/serial.rs` reads `owner=kernel` as a - // death and the other form as a record. The event's name is the last - // word. - "iommu: DMA FAULT owner={} unit0 stream={} addr={address:#018x} access={access} domain={} bme={} \ - unitfaults={count} streamfaults={faults} first={} reason={:?} {}", - Owner(owner), - Who(function, event.stream), - function.map_or(0, |f| f.domain.load(Ordering::Relaxed)), - if function.is_some() { "cleared" } else { "unknown-function" }, - if first { 'y' } else { 'n' }, - event.code, - event.code.name(), - ); - owner.is_none() -} - -/// The function's config window, from the address [`arm`] published. -fn config_window(phys: u64) -> Mmio { - // SAFETY: `phys` is `DirectMap::phys_of` a config window the ECAM mapping - // holds for the machine's life, one page wide. - unsafe { Mmio::over_phys(DirectMap::from_phys(phys), PAGE_SIZE) } -} - -/// Who an event was handed to. -struct Owner(Option); - -impl core::fmt::Display for Owner { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self.0 { - None => write!(f, "kernel"), - Some(slot) => write!(f, "slot{slot}"), - } - } -} - -/// The function an event names, as `pci::enumerate` prints it, or the raw -/// StreamID where it is none this unit translates for. -struct Who(Option<&'static Function>, u32); - -impl core::fmt::Display for Who { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self.0 { - Some(function) => write!(f, "{}", function.function), - None => write!(f, "{:#x}", self.1), - } - } + policy::conclude(kernel_owned); } diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index 8aba96fe553..6df76be26e4 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -4,12 +4,17 @@ //! //! [`init`] arms the unit with every stream aborting and no bypass anywhere: //! `GBPA` aborts before anything else is written, so nothing passes while the -//! unit is programmed; every enumerated function's stream has an entry that -//! aborts, and the unit is enabled. A stream translates only once +//! unit is programmed; and the unit is enabled. A stream translates only once //! [`domain::attach`] puts it on a domain of its own, through stage 1 alone; -//! no domain maps memory by identity, so a function no driver attached -//! reaches nothing. A unit this kernel cannot program is left aborting every -//! transaction, and no domain is given. +//! no domain maps memory by identity. An enumerated function's stream on no +//! domain aborts and records nothing (`Ste::ABORT`): firmware may leave one +//! mastering the bus until its driver resets it. **Every stream no +//! enumerated function is routed from has one answer**, inside the stream +//! table or past it: it aborts and is recorded — `C_BAD_STE` on an entry left +//! invalid, `C_BAD_STREAMID` under `CR2.RECINVSID` — and, nobody driving it, +//! halts the machine, as an unenumerated requester does on VT-d. A unit this +//! kernel cannot program, and every unit of a machine whose IORT names more +//! than one, is left aborting every transaction, and no domain is given. //! //! The unit reads what it is given coherently — `toyos_smmu::unit::probe` //! refuses one that does not — so a store it reads is published by the order @@ -201,8 +206,8 @@ fn window(phys: u64, bytes: u64) -> Mmio { unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } } -/// The SMMUv3, armed; a refusal leaves it aborting what it is given, or says -/// why it could not be told to. +/// The SMMUv3, armed; a refusal leaves every unit the IORT names aborting +/// what it is given, or says why one could not be told to. pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::RootBridgeWindow]) { let iort = match toyos_acpi::iort(direct_phys(), rsdp_addr) { Ok(iort) => iort, @@ -215,37 +220,57 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::R return; } }; - let mut units = iort.nodes().filter_map(|node| match node { - Node::Smmuv3(smmu) => Some(smmu), - _ => None, - }); - let Some(smmu) = units.next() else { - log!("IOMMU: the IORT names no SMMUv3: no device is translated this boot"); - return; - }; - if units.next().is_some() { - log!("IOMMU: the IORT names more than one SMMUv3, and this kernel drives one: none is touched"); - return; - } - if !smmu.base.is_multiple_of(REGISTER_PAGE) { - log!("IOMMU: the SMMUv3's registers at {:#x} are not on a 64 KiB page: not touched", smmu.base); - return; + let named: Vec<_> = iort + .nodes() + .filter_map(|node| match node { + Node::Smmuv3(smmu) => Some(smmu), + _ => None, + }) + .collect(); + // Every unit, before any refusal: `GBPA`'s reset value is IMPLEMENTATION + // DEFINED, and one firmware left bypassing gives its devices all memory. + let mut aborting = Vec::new(); + for smmu in &named { + if !smmu.base.is_multiple_of(REGISTER_PAGE) { + log!("IOMMU: the SMMUv3's registers at {:#x} are not on a 64 KiB page: not touched", smmu.base); + continue; + } + let regs = Registers(crate::mm::paging::map_mmio(smmu.base, 2 * REGISTER_PAGE, MmioPolicy::Uncacheable)); + abort_unprogrammed(regs, smmu.base); + aborting.push((smmu, regs)); } - let regs = Registers(crate::mm::paging::map_mmio(smmu.base, 2 * REGISTER_PAGE, MmioPolicy::Uncacheable)); - abort_unprogrammed(regs, smmu.base); + let (smmu, regs) = match (named.len(), aborting.pop()) { + (0, _) => { + log!("IOMMU: the IORT names no SMMUv3: no device is translated this boot"); + return; + } + (1, Some(one)) => one, + (1, None) => return, + (count, _) => { + log!("IOMMU: the IORT names {count} SMMUv3s, and this kernel drives one: each is left aborting"); + return; + } + }; + crate::iommu::fault::describe(devices); let segment = u32::from(crate::pcidev::segment()); let mut routes = Vec::new(); for device in devices { let function = StreamId::pci(device.bus, device.dev, device.func); match iort.route(segment, function.requester()) { + #[cfg(feature = "boot-actuators")] + Ok(Route::Translated { stream, .. }) + if crate::actuator::smmu_selftest() && selftest::stands_unrouted(device, devices) => + { + log!("smmu-selftest: {function}, StreamID {stream:#x}, is given no route, as a function never enumerated is not"); + } Ok(Route::Translated { smmu: by, stream, .. }) if by.base == smmu.base => routes.push((function, stream)), Ok(route) => log!("IOMMU: {function} is not routed through the SMMUv3: {route:?}"), Err(why) => log!("IOMMU: {function}'s IORT route is refused: {why:?}"), } } let reserved = windows.iter().map(|w| (w.base, w.end())).collect(); - let Some(live) = program(regs, smmu.base, smmu.coherent_override, smmu.event, routes, reserved, devices) else { + let Some(live) = program(regs, smmu.base, smmu.coherent_override, smmu.event, routes, reserved) else { return; }; *UNIT.lock() = Some(live); @@ -255,8 +280,9 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::R } } -/// `GBPA` aborting and the unit disabled: from here until `SMMUEN`, and for -/// good if it is never set, every transaction aborts. +/// `GBPA` aborting, the unit disabled and none of its interrupts able to +/// write: from here until `SMMUEN`, and for good if it is never set, every +/// transaction aborts. fn abort_unprogrammed(regs: Registers, base: u64) { let gbpa = || regs.read(reg::GBPA); regs.wait("GBPA free to update", || gbpa() & reg::GBPA_UPDATE == 0); @@ -272,6 +298,15 @@ fn abort_unprogrammed(regs: Registers, base: u64) { log!("IOMMU: the SMMUv3 at {base:#x} was handed over with CR0 {cr0:#x}; it goes off first"); regs.control(reg::CR0, 0, "CR0 cleared"); } + // An `*_IRQ_CFG0` address that is not zero makes its interrupt a write + // there, which no stream table entry governs; each resets UNKNOWN and is + // written only with its interrupt off (§3.18.2, §6.3.21, §6.3.30, §6.3.34). + regs.control(reg::IRQ_CTRL, 0, "its interrupts disabled"); + regs.write64(reg::GERROR_IRQ_CFG0, 0); + regs.write64(reg::EVENTQ_IRQ_CFG0, 0); + if regs.read(reg::IDR0) & reg::IDR0_PRI != 0 { + regs.write64(reg::PRIQ_IRQ_CFG0, 0); + } log!("IOMMU: SMMUv3 at {base:#x}: GBPA {:#x}, every transaction aborts while SMMUEN is clear", gbpa()); } @@ -285,7 +320,6 @@ fn program( event: Option, routes: Vec<(StreamId, u32)>, reserved: Vec<(u64, u64)>, - devices: &[PciDevice], ) -> Option { let refused = |why: core::fmt::Arguments<'_>| log!("IOMMU: the SMMUv3 at {base:#x} is left aborting: {why}"); let (idr0, idr1, idr5) = (regs.read(reg::IDR0), regs.read(reg::IDR1), regs.read(reg::IDR5)); @@ -322,9 +356,11 @@ fn program( let mut memory = Memory::new(); let table = memory.alloc(64 << log2); let streams = window(table, 64 << log2); - for stream in 0..1u64 << log2 { + // Every other entry stays zero, invalid: a stream no function is routed + // from is recorded, as one past the table is. + for (_, stream) in &routes { for (i, word) in Ste::ABORT.words().iter().enumerate() { - streams.write_u64(stream * 64 + 8 * i as u64, *word); + streams.write_u64(u64::from(*stream) * 64 + 8 * i as u64, *word); } } let (commands_log2, events_log2) = @@ -364,14 +400,14 @@ fn program( regs.write64(reg::EVENTQ_BASE, events_base); regs.write(reg::EVENTQ_PROD, 0); regs.write(reg::EVENTQ_CONS, 0); - fault::arm(regs, window(events_at, 32 << events_log2), events, devices, &live.routes); + fault::arm(regs, window(events_at, 32 << events_log2), events, &live.routes); regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN, "its event queue enabled"); regs.control(reg::IRQ_CTRL, reg::IRQ_EVENTQ, "its event interrupt enabled"); regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN | reg::CR0_SMMUEN, "SMMUEN"); log!( - "IOMMU: SMMUv3 at {base:#x} armed, CR0ACK {:#x}: {} functions' streams in a table of {}, every entry \ - aborting; a CMD_SYNC consumed; events on SPI {event}", + "IOMMU: SMMUv3 at {base:#x} armed, CR0ACK {:#x}: {} functions' streams aborting in a table of {}, every \ + other entry invalid; a CMD_SYNC consumed; events on SPI {event}", regs.read(reg::CR0ACK), live.routes.len(), 1u32 << log2, diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs index ed8859d3984..d6cab2e2cfd 100644 --- a/kernel/src/arch/aarch64/smmu/selftest.rs +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -1,16 +1,22 @@ //! `smmu-selftest`: QEMU's `iommu-testdev` (1b36:0005), whose one register //! read makes a DMA write of a known word at a device address and reads it -//! back at a physical one, asked three things of the armed unit: +//! back at a physical one. Two of them, asked four things of the armed unit: //! -//! 1. its write, on the entry every stream starts with, is refused; -//! 2. on a domain of its own, a write at an address the domain maps lands at -//! the memory mapped there and nowhere else; -//! 3. a write at an address the domain does not map is refused and recorded: -//! the event interrupt reaches the handler, which names it and, the -//! function being no process's, halts the machine. +//! 1. the last one writes on the entry its stream starts with, and is refused +//! with nothing recorded; +//! 2. on a domain of its own, it writes at an address the domain maps, and +//! lands at the memory mapped there and nowhere else; +//! 3. once the domain takes that address back, it writes there again, and is +//! refused and recorded; +//! 4. the other, which [`super::init`] gives no route, as it gives none to a +//! function it never enumerated, writes under a StreamID inside the stream +//! table, and is refused and recorded. //! -//! Each says what it saw on its own line; the harness judges them, and the -//! third by the handler's. +//! The last two are made with this CPU's interrupts masked, so one drain reads +//! both: the event interrupt reaches the handler, which names each and, their +//! functions being no process's, halts the machine. Each step says what it +//! saw on its own line; the harness judges them, and the records by the +//! handler's, whose count says the first step recorded nothing. use crate::drivers::pci::PciDevice; use crate::iommu::OwnSpace; @@ -40,12 +46,28 @@ const BAR_BYTES: u64 = 0x1000; const LANDED: u32 = 0; const REFUSED: u32 = 0xdead_0002; -/// How long the event is given to reach this CPU once the write is refused. +/// How long the two events are given to reach this CPU once it unmasks. const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( Duration::from_secs(1), - "the unit records the event and pulses its interrupt while the refused write is still being made", + "the unit records each event and pulses its interrupt while the refused write is still being made", ); +/// A test device below another: [`super::init`] routes nothing for it, and +/// the stream table, sized to the last one's StreamID, holds its entry. +pub(super) fn stands_unrouted(device: &PciDevice, devices: &[PciDevice]) -> bool { + let at = |d: &PciDevice| (d.bus, d.dev, d.func); + device.is_id(VENDOR, DEVICE) && devices.iter().any(|d| d.is_id(VENDOR, DEVICE) && at(d) > at(device)) +} + +/// `device`'s registers, its memory decode on. No Bus Master Enable: the +/// device writes through the unit's address space whatever its `COMMAND` says. +fn registers(device: &PciDevice) -> Mmio { + let bar = device.memory_bar(0).expect("smmu-selftest: an iommu-testdev's BAR0 is unassigned").address(); + let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); + device.enable_memory_space(); + regs +} + /// The device's write of four bytes at `at`, read back at `phys`: its `RESULT`. fn write(regs: Mmio, at: u64, phys: u64) -> u32 { regs.write_u32(GVA_LO, at as u32); @@ -58,24 +80,29 @@ fn write(regs: Mmio, at: u64, phys: u64) -> u32 { regs.read_u32(RESULT) } +fn verdict(result: u32, want: u32) -> &'static str { + match (result == want, want) { + (true, REFUSED) => "refused", + (true, _) => "landed there", + (false, _) => "FAIL", + } +} + pub(super) fn run(devices: &[PciDevice]) { - let device = devices - .iter() - .find(|d| d.is_id(VENDOR, DEVICE)) - .expect("smmu-selftest: no iommu-testdev (1b36:0005) on this machine"); - let bar = device.memory_bar(0).expect("smmu-selftest: the iommu-testdev's BAR0 is unassigned").address(); - let regs = crate::mm::paging::map_mmio(bar, BAR_BYTES, MmioPolicy::Uncacheable); - // No Bus Master Enable: the device writes through the unit's address - // space whatever its `COMMAND` says. - device.enable_memory_space(); + let mut testdevs = devices.iter().filter(|d| d.is_id(VENDOR, DEVICE)); + let (Some(stray), Some(device), None) = (testdevs.next(), testdevs.next(), testdevs.next()) else { + panic!("smmu-selftest: this machine has not two iommu-testdevs (1b36:0005)"); + }; + let (regs, stray_regs) = (registers(device), registers(stray)); let page = crate::mm::pmm::alloc_page().expect("smmu-selftest: no page to aim at"); let phys = page.direct_map().phys(); let function = crate::iommu::StreamId::pci(device.bus, device.dev, device.func); + let unrouted = crate::iommu::StreamId::pci(stray.bus, stray.dev, stray.func); let aborted = write(regs, phys, phys); log!( - "smmu-selftest: {function}'s write at {phys:#x}, on the entry every stream starts with, answered {aborted:#x}: {}", - if aborted == REFUSED { "refused" } else { "FAIL" } + "smmu-selftest: {function}'s write at {phys:#x}, on the entry its stream starts with, answered {aborted:#x}: {}", + verdict(aborted, REFUSED) ); let (space, at) = OwnSpace::create(PAGE_2M).expect("smmu-selftest: a domain on the armed unit"); @@ -86,20 +113,32 @@ pub(super) fn run(devices: &[PciDevice]) { "smmu-selftest: {function}'s write at {:#x}, mapped to {:#x}, answered {landed:#x}: {}", at + 0x40, phys + 0x40, - if landed == LANDED { "landed there" } else { "FAIL" } + verdict(landed, LANDED) ); + space.unmap(at, PAGE_2M).expect("smmu-selftest: the leaf it placed"); - let unmapped = at + PAGE_2M + 0x40; - log!("smmu-selftest: {function} writes at {unmapped:#x}, which its domain does not map"); let before = super::fault::recorded(); - let refused = write(regs, unmapped, phys + 0x40); + let masked = crate::arch::IrqGuard::close(); + let taken_back = write(regs, at + 0x40, phys + 0x40); + let stray_wrote = write(stray_regs, phys, phys); + log!( + "smmu-selftest: {function}'s write at {:#x} again, which its domain no longer maps, answered {taken_back:#x}: {}", + at + 0x40, + verdict(taken_back, REFUSED) + ); + log!( + "smmu-selftest: {unrouted}'s write at {phys:#x}, under no route, answered {stray_wrote:#x}: {}", + verdict(stray_wrote, REFUSED) + ); + drop(masked); let deadline = crate::clock::nanos_since_boot() + EVENT_TIMEOUT.nanos(); - while super::fault::recorded() == before { + while super::fault::recorded() < before + 2 { assert!( crate::clock::nanos_since_boot() < deadline, - "smmu-selftest: FAIL: the write at {unmapped:#x} answered {refused:#x}, and no event reached this CPU within {EVENT_TIMEOUT}" + "smmu-selftest: FAIL: {} of the two refused writes' events reached this CPU within {EVENT_TIMEOUT}", + super::fault::recorded() - before ); core::hint::spin_loop(); } - panic!("smmu-selftest: FAIL: the handler read the event of a function no process drives and the machine went on"); + panic!("smmu-selftest: FAIL: the handler read the events of functions no process drives and the machine went on"); } diff --git a/kernel/src/arch/x86_64/vtd/domain.rs b/kernel/src/arch/x86_64/vtd/domain.rs index b0027c93edd..35283ea9c3a 100644 --- a/kernel/src/arch/x86_64/vtd/domain.rs +++ b/kernel/src/arch/x86_64/vtd/domain.rs @@ -78,15 +78,14 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { if u32::from(id) >= ceiling { return Err(IommuError::DomainsExhausted(ceiling)); } - let mut domain = Domain::new(&mut TABLES.lock(), id, width, mgaw, &domains.reserved, room)?; - let first = domain.reserve(room).expect("`Domain::new` refuses a domain without the room"); + let (domain, first) = Domain::new(&mut TABLES.lock(), id, width, mgaw, &domains.reserved, room)?; log!( "iommu: domain{id} root={:#x} aw={} mgaw={} addresses from {:#x} to {:#x}", domain.root().phys(), width.bits(), mgaw, - domain.floor(), - domain.ceiling() + domain.window().floor(), + domain.window().ceiling() ); domains.live.push(domain); Ok((DomainId::new(id), first)) @@ -98,7 +97,7 @@ pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { } let mut domains = DOMAINS.lock(); let domain = domains.at(id); - let at = domain.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.ceiling()))?; + let at = domain.window_mut().reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.window().ceiling()))?; let (did, domain) = (domain.id(), *domain); let mut units = UNITS.lock(); table::map(&mut TABLES.lock(), &domain, at, phys, bytes); @@ -138,7 +137,7 @@ pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result = AtomicPtr::new(core::ptr::null_mut()); - -fn functions() -> &'static [Function] { - let published = FUNCTIONS.load(Ordering::Acquire); - if published.is_null() { - return &[]; - } - // SAFETY: non-null only as `describe` stored it: a leaked box, never freed - // or written again, published with Release once it was whole. - unsafe { *published } -} - -/// The first fault this machine took, whole: what a later one says is decided -/// by what the first one already broke. -struct FirstFault { - who: AtomicU32, - address: AtomicU64, - reason: AtomicU32, - unit: AtomicU32, -} - -static FIRST: FirstFault = FirstFault { - who: AtomicU32::new(pci::NO_FUNCTION), - address: AtomicU64::new(0), - reason: AtomicU32::new(0), - unit: AtomicU32::new(0), -}; - -/// Every function this machine enumerated, before any unit is armed: the -/// handler reaches a faulting function's config space through this, with no lock. -pub fn describe(devices: &[PciDevice]) { - let functions: Vec = devices - .iter() - .map(|device| Function { - who: StreamId::pci(device.bus, device.dev, device.func).requester(), - config: DirectMap::phys_of(device.config_window().addr() as *const u8), - domain: AtomicU32::new(0), - faults: AtomicU32::new(0), - user_slot: AtomicU32::new(NO_SLOT), - }) - .collect(); - let functions: &'static [Function] = Box::leak(functions.into_boxed_slice()); - let first = FUNCTIONS.compare_exchange( - core::ptr::null_mut(), - Box::leak(Box::new(functions)), - Ordering::Release, - Ordering::Relaxed, - ); - assert!(first.is_ok(), "iommu: the fault handler's functions were described twice"); -} - -/// Record which domain a function moved to, for the flag the handler sets. -pub fn attached(stream: StreamId, domain: u16) { - if let Some(slot) = find(u32::from(stream.requester())) { - slot.domain.store(u32::from(domain), Ordering::Relaxed); - } -} - -/// Record that a process drives this function on `slot`, or no longer does. -/// -/// The handler reads it to decide what a fault on this stream is *terminal -/// for*: a kernel-driven function has nothing to hand a fault to and the -/// response is the halt below; one a process drives has an owner, so the record -/// goes to that owner's claim and the machine stays up. -pub fn user_owned(stream: StreamId, slot: Option) { - if let Some(function) = find(u32::from(stream.requester())) { - function - .user_slot - .store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); - } -} - -fn find(requester: u32) -> Option<&'static Function> { - functions().iter().find(|f| u32::from(f.who) == requester) -} - /// A unit's fault-record location and count: `CAP.FRO` and `CAP.NFR`. #[derive(Clone, Copy)] pub struct Records { @@ -198,18 +92,12 @@ pub fn arm(index: usize, regs: Mmio, found: Records, vector: u8) { regs.write_u32(FECTL_REG, 0); } -const CONFIG_WINDOW: u64 = crate::mm::PAGE_SIZE; - // The window an armed unit's registers live in, from the address `arm` published. fn window(phys: u64) -> Mmio { - window_of(phys, REGISTER_WINDOW) -} - -fn window_of(phys: u64, size: u64) -> Mmio { // SAFETY: `phys` came from `DirectMap::phys_of` over a window - // `paging::map_mmio` produced at boot and never unmapped, `size` bytes wide; - // this is the same window read back. - unsafe { Mmio::over_phys(DirectMap::from_phys(phys), size) } + // `paging::map_mmio` produced at boot and never unmapped, `REGISTER_WINDOW` + // bytes wide; this is the same window read back. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), REGISTER_WINDOW) } } /// The unit raised its fault event. @@ -227,26 +115,7 @@ pub fn service() { unit.count.load(Ordering::Relaxed), ); } - - if kernel_owned > 0 { - // capture() puts the fault on the panel before the halt takes the - // machine down. - // - // The refusal is the whole response and there is no recovery missing - // from it: a faulting device reached an address this kernel never gave - // it, and nothing here can know what else it already did. **So this - // path halts and never panics** — a report of one carries the fault - // record and `panic_reboot`'s arm line, and no `panicked at` line. - // - // **Only for a stream this kernel drives.** A function a process drives - // has an owner to refuse: its bus mastering is already gone by the time - // this is reached, its claim answers every later call `Io`, and the - // machine — whose other drivers are untouched — goes on. Halting for - // that would be one process's bug taking the whole machine down, which - // is the thing moving a driver out was for. - crate::drivers::panic_console::capture(); - crate::panic::halt_all_cpus(); - } + policy::conclude(kernel_owned); crate::arch::apic::eoi(); } @@ -275,40 +144,16 @@ fn drain(index: usize, regs: Mmio, records: u64, count: u32) -> usize { (high & 0x7) as u8, ); let reason = ((high >> 32) & 0xFF) as u8; - let address = regs.read_u64(record) & !0xFFF; - // First, before anything that can be slow or say no: a function that - // cannot master the bus raises no second fault, which is the whole of - // the ceiling on a storm. - let stopped = stop(stream); - let seen_here = note(stream); - latch(index, stream, address, reason); - // Before the line, so `owner=` in it is what was actually told. - let owner = owner_of(stream); - if let Some(slot) = owner { - crate::pcidev::note_fault(slot); - } - let count = UNITS[index].faults.fetch_add(1, Ordering::Relaxed) + 1; - log!( - // `owner=` first, because it is the only field that decides whether - // this machine is still running: `tests/common/serial.rs` reads - // `iommu: DMA FAULT owner=kernel` as a death and the other form as - // a record. The reason's name is the last word: every gate takes it - // from there. - "iommu: DMA FAULT owner={} unit{index} stream={stream} addr={address:#018x} \ - access={} reason={reason:#04x} domain={} bme={} unitfaults={count} \ - streamfaults={seen_here} first={} {}", - match owner { - Some(slot) => Owner::Process(slot), - None => Owner::Kernel, - }, - if high & (1u64 << 62) != 0 { "read" } else { "write" }, - blamed(stream), - if stopped { "cleared" } else { "unknown-function" }, - yn(FIRST.who.load(Ordering::Relaxed) == u32::from(stream.requester())), - reason_name(reason), - ); + let fault = Fault { + who: Who::Function(stream), + address: regs.read_u64(record) & !0xFFF, + access: if high & (1u64 << 62) != 0 { Access::Read } else { Access::Write }, + reason, + name: reason_name(reason), + }; + let unowned = policy::report(index, &UNITS[index].faults, fault); clear_record(regs, record); - if owner.is_none() { + if unowned { seen += 1; } } @@ -320,100 +165,6 @@ fn clear_record(regs: Mmio, record: u64) { regs.write_u32(record + 12, RECORD_FAULT); } -/// The `pcidev` slot a process drives this stream on, or `None` for a stream -/// this kernel drives — including one it never enumerated, which nothing can be -/// handed to either. -fn owner_of(stream: StreamId) -> Option { - match find(u32::from(stream.requester()))?.user_slot.load(Ordering::Acquire) { - NO_SLOT => None, - slot => Some(slot as usize), - } -} - -/// Who a fault was handed to, in the line. A word rather than a number, because -/// which of the two it is decides whether this machine is still running. -enum Owner { - Kernel, - Process(usize), -} - -impl core::fmt::Display for Owner { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Kernel => write!(f, "kernel"), - Self::Process(slot) => write!(f, "slot{slot}"), - } - } -} - -/// Clear Bus Master Enable on whoever faulted; `false` where this machine -/// enumerated no function with that requester id, and the line says so. -fn stop(stream: StreamId) -> bool { - let Some(slot) = find(u32::from(stream.requester())) else { - return false; - }; - pci::stop_bus_mastering(window_of(slot.config, CONFIG_WINDOW)); - true -} - -fn note(stream: StreamId) -> u32 { - find(u32::from(stream.requester())) - .map_or(0, |slot| slot.faults.fetch_add(1, Ordering::Relaxed) + 1) -} - -/// What the handler can say about the faulting function's address space. The -/// middle answer is deliberately weak: a function the kernel never attached -/// carries domain id 0 whether it sits on the identity domain or on one an -/// actuator bound by hand, so the label claims only that nothing recorded one. -enum Blamed { - Own(u32), - Unrecorded, - Unknown, -} - -impl core::fmt::Display for Blamed { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Own(id) => write!(f, "{id}"), - Self::Unrecorded => write!(f, "unrecorded"), - Self::Unknown => write!(f, "unknown"), - } - } -} - -fn blamed(stream: StreamId) -> Blamed { - match find(u32::from(stream.requester())) { - None => Blamed::Unknown, - Some(slot) => match slot.domain.load(Ordering::Relaxed) { - 0 => Blamed::Unrecorded, - id => Blamed::Own(id), - }, - } -} - -/// Take the first fault whole, once: a second finds `who` taken and leaves it. -fn latch(unit: usize, stream: StreamId, address: u64, reason: u8) { - let taken = FIRST.who.compare_exchange( - pci::NO_FUNCTION, - u32::from(stream.requester()), - Ordering::AcqRel, - Ordering::Relaxed, - ); - if taken.is_ok() { - FIRST.address.store(address, Ordering::Relaxed); - FIRST.reason.store(u32::from(reason), Ordering::Relaxed); - FIRST.unit.store(unit as u32, Ordering::Relaxed); - } -} - -fn yn(v: bool) -> char { - if v { - 'y' - } else { - 'n' - } -} - // Only reasons where Linux's dma_remap_fault_reasons and QEMU's VTD_FR_* // agree get a name; 0x0c and 0x0d disagree and are left unnamed. fn reason_name(reason: u8) -> &'static str { diff --git a/kernel/src/arch/x86_64/vtd/mod.rs b/kernel/src/arch/x86_64/vtd/mod.rs index 9dc03c9ca9f..df574044349 100644 --- a/kernel/src/arch/x86_64/vtd/mod.rs +++ b/kernel/src/arch/x86_64/vtd/mod.rs @@ -156,7 +156,7 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) // Before any unit is armed: the handler reaches a faulting function's // config space through this and cannot take a lock to find it. - fault::describe(devices); + crate::iommu::fault::describe(devices); let mut units = 0usize; let mut regions = 0usize; diff --git a/kernel/src/arch/x86_64/vtd/table.rs b/kernel/src/arch/x86_64/vtd/table.rs index 39e2fc2d44a..6b44d87cd7e 100644 --- a/kernel/src/arch/x86_64/vtd/table.rs +++ b/kernel/src/arch/x86_64/vtd/table.rs @@ -7,6 +7,7 @@ use alloc::vec::Vec; +use crate::iommu::window::{self, Window}; use crate::iommu::{AddressWidth, IommuError, Iova, StreamId}; use crate::mm::pmm::{self, PhysPage}; use crate::mm::{DirectMap, Mmio, PAGE_2M}; @@ -218,37 +219,13 @@ pub fn identity_domain(tables: &mut Tables, width: AddressWidth, top: u64) -> (T (root, frames) } -/// One domain's second-level tables, its id, and how far up its addresses have been handed out. +/// One domain's second-level tables, its id, and its addresses. #[derive(Clone, Copy)] pub struct Domain { root: Table, id: u16, width: AddressWidth, - /// Where its addresses start: [`Domain::first_address`] of what its unit - /// translates, which is not [`AddressWidth::bits`] — see - /// [`Domain::translatable_bits`]. - floor: u64, - /// Where they end: [`ceiling`]. - ceiling: u64, - next: u64, -} - -/// Where a domain's addresses end: under what its unit translates, and under -/// the first of `reserved` that reaches above `floor` — a root bridge's window, -/// which a bridge may route peer-to-peer before the unit sees the request -/// (PCIe Base §2.4), or a region firmware reserved (VT-d §3.16). At or below -/// `floor` where one of them covers it. -const fn ceiling(translatable: u8, floor: u64, reserved: &[(u64, u64)]) -> u64 { - let mut ceiling = 1u64 << translatable; - let mut i = 0; - while i < reserved.len() { - let (start, end) = reserved[i]; - if end > floor && start < ceiling { - ceiling = start; - } - i += 1; - } - ceiling + window: Window, } impl Domain { @@ -266,13 +243,8 @@ impl Domain { if mgaw < width.bits() { mgaw } else { width.bits() } } - /// A quarter of the way up what this domain can translate. - const fn first_address(translatable: u8) -> u64 { - 1 << (translatable - 2) - } - - /// A domain with `room` bytes of addresses between its floor and its - /// [`ceiling`], or the reason it has not. + /// A domain with `room` bytes of its [`Window`] handed out at the address + /// answered, or the reason it has not. pub fn new( tables: &mut Tables, id: u16, @@ -280,21 +252,9 @@ impl Domain { mgaw: u8, reserved: &[(u64, u64)], room: u64, - ) -> Result { - let translatable = Self::translatable_bits(width, mgaw); - let floor = Self::first_address(translatable); - // Above memory, so a descriptor still carrying one of these names - // nothing this domain maps and faults rather than landing on a page. - let top = crate::mm::pmm::top(); - if floor <= top { - return Err(IommuError::WindowBelowMemory { translatable, floor, top }); - } - let ceiling = ceiling(translatable, floor, reserved); - // At least one leaf, whatever was asked: a domain with none is no domain. - if ceiling.saturating_sub(floor) < room.next_multiple_of(PAGE_2M).max(PAGE_2M) { - return Err(IommuError::NoRoom { floor, ceiling, room }); - } - Ok(Self { root: tables.alloc(), id, width, floor, ceiling, next: floor }) + ) -> Result<(Self, Iova), IommuError> { + let (window, first) = Window::new(Self::translatable_bits(width, mgaw), reserved, room)?; + Ok((Self { root: tables.alloc(), id, width, window }, first)) } pub fn root(&self) -> Table { @@ -305,101 +265,32 @@ impl Domain { self.id } - pub const fn floor(&self) -> u64 { - self.floor + pub const fn window(&self) -> &Window { + &self.window } - pub const fn ceiling(&self) -> u64 { - self.ceiling - } - - /// Reserve room for `bytes`, rounded up to whole leaves. An address is - /// never handed out twice, unmapped or not: a device holding a stale one - /// would reach whatever took its place. - pub fn reserve(&mut self, bytes: u64) -> Option { - let span = bytes.next_multiple_of(PAGE_2M); - let end = self.next.checked_add(span)?; - if end > self.ceiling() { - return None; - } - let at = Iova::translated(self.next); - self.next = end; - Some(at) - } - - /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: - /// the only room a mapping may be placed in by address. - /// - /// `at` must itself be a leaf `reserve` could have returned, not merely - /// inside a handed-out span: [`map_2m`]'s index floors to the enclosing - /// leaf, so an unaligned `at` this let through would silently place a - /// mapping short of, or overlapping, where the caller named. - pub const fn handed_out(&self, at: Iova, bytes: u64) -> bool { - if !at.raw().is_multiple_of(PAGE_2M) { - return false; - } - let span = bytes.next_multiple_of(PAGE_2M); - if at.raw() < self.floor() { - return false; - } - match at.raw().checked_add(span) { - Some(end) => end <= self.next, - None => false, - } + pub fn window_mut(&mut self) -> &mut Window { + &mut self.window } } -/// [`Domain::handed_out`] is a pure predicate on a `Copy` struct: checked here -/// at compile time rather than under a test harness this no-`std` binary has -/// none of. -const _: () = { - const ROOT: Table = Table { phys: 0 }; - // `translatable = 48` puts `floor()` (a quarter of `1 << 48`) at `1 << 46`, - // itself far past `PAGE_2M`-aligned. - const FLOOR: u64 = Domain::first_address(48); - const ONE_LEAF: Domain = Domain { - root: ROOT, - id: KERNEL_DOMAIN + 1, - width: AddressWidth::Bits48, - floor: FLOOR, - ceiling: 1 << 48, - next: FLOOR + PAGE_2M, - }; - const TWO_LEAVES: Domain = Domain { next: FLOOR + 2 * PAGE_2M, ..ONE_LEAF }; - - // Exactly what one `reserve(PAGE_2M)` handed out. - assert!(ONE_LEAF.handed_out(Iova::translated(FLOOR), PAGE_2M)); - // Short of the floor: nothing this domain has ever reserved. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); - // Past what has been reserved so far. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - // A byte count is rounded up to the leaf it needs, not truncated to fit. - assert!(!ONE_LEAF.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); - // Two leaves handed out; the second is room in its own right. - assert!(TWO_LEAVES.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); - // Mid-span but off a leaf boundary: inside the handed-out range without - // being an address `reserve` ever returned. - assert!(!TWO_LEAVES.handed_out(Iova::translated(FLOOR + 1), PAGE_2M)); -}; - -/// [`ceiling`] over the windows the T14's firmware declares, unsorted as it -/// declares them: a 39-bit unit's domain ends where the first window above its -/// floor begins, and a window reaching over the floor leaves it nothing. +/// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the +/// smaller of the two. +/// +/// Asserted here rather than in a guest because no guest holds the shape: QEMU's +/// `intel-iommu` derives both fields from one `aw-bits` property, so its model +/// cannot report a `SAGAW` wider than its `MGAW` at all. const _: () = { - const FLOOR: u64 = Domain::first_address(39); - const T14: [(u64, u64); 7] = [ - (0xA200_0000, 0xBD00_0000), - (0x40_0000_0000, 0x60_3DC0_0000), - (0xA080_0000, 0xA200_0000), - (0xBD00_0000, 0xC000_0000), - (0xFF00_0000, 0xFFB8_0000), - (0xFFD3_A070, 0x1_0000_0000), - (0x60_3DC0_0000, 0x80_0000_0000), - ]; - assert!(FLOOR == 0x20_0000_0000); - assert!(ceiling(39, FLOOR, &T14) == 0x40_0000_0000); - assert!(ceiling(39, FLOOR, &[]) == 1 << 39); - assert!(ceiling(39, FLOOR, &[(0x10_0000_0000, FLOOR + 1)]) < FLOOR); + assert!(Domain::translatable_bits(AddressWidth::Bits48, 39) == 39); + assert!(window::first_address(39) < 1 << 39); + // What a window placed by the table depth alone would be, against the + // ceiling such a unit reports. + assert!(window::first_address(48) >= 1 << 39); + // A unit whose two limits agree is unchanged by any of this. + assert!(Domain::translatable_bits(AddressWidth::Bits48, 48) == 48); + assert!(Domain::translatable_bits(AddressWidth::Bits39, 39) == 39); + // And tables shallower than `MGAW` bind it the other way round. + assert!(Domain::translatable_bits(AddressWidth::Bits39, 48) == 39); }; pub fn map(tables: &mut Tables, domain: &Domain, at: Iova, phys: u64, bytes: u64) { @@ -508,10 +399,11 @@ impl Displaced { } } -/// The entry that moves `requester`'s present entry `old` onto `domain`, and -/// what that leaves the unit holding: the id `old` named, never `domain`'s. -const fn rebind(old: (u64, u64), requester: u16, domain: &Domain) -> ((u64, u64), Displaced) { - let new = context_entry(domain.root, domain.id, domain.width); +/// The entry that moves `requester`'s present entry `old` onto domain `id` +/// at `root`, and what that leaves the unit holding: the id `old` named, never +/// `id`. +const fn rebind(old: (u64, u64), requester: u16, root: Table, id: u16, width: AddressWidth) -> ((u64, u64), Displaced) { + let new = context_entry(root, id, width); (new, Displaced { requester, domain: (old.1 >> 8) as u16 }) } @@ -549,7 +441,7 @@ pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { "iommu: {stream} has no context entry to move — it was not enumerated when its unit \ was programmed" ); - let (new, displaced) = rebind(old, stream.requester(), domain); + let (new, displaced) = rebind(old, stream.requester(), domain.root, domain.id, domain.width); context.replace_pair(index, old, new); displaced } @@ -558,35 +450,8 @@ pub fn bind(root: Table, stream: StreamId, domain: &Domain) -> Displaced { /// domain's id, the only one its cached entry can match (§6.5.2.1), and its new /// entry names the domain it moved to. const _: () = { - const OWN: Domain = Domain { - root: Table { phys: 0x5000 }, - id: u16::MAX, - width: AddressWidth::Bits39, - floor: 0, - ceiling: 0, - next: 0, - }; let identity = context_entry(Table { phys: 0x3000 }, KERNEL_DOMAIN, AddressWidth::Bits48); - let (new, displaced) = rebind(identity, 0x00F8, &OWN); + let (new, displaced) = rebind(identity, 0x00F8, Table { phys: 0x5000 }, u16::MAX, AddressWidth::Bits39); assert!(displaced.domain == KERNEL_DOMAIN && displaced.requester == 0x00F8); assert!(new.0 == 0x5000 | PRESENT && new.1 == (u16::MAX as u64) << 8 | 1); }; - -/// A unit whose `MGAW` is narrower than its `SAGAW` gets its window under the -/// smaller of the two. -/// -/// Asserted here rather than in a guest because no guest holds the shape: QEMU's -/// `intel-iommu` derives both fields from one `aw-bits` property, so its model -/// cannot report a `SAGAW` wider than its `MGAW` at all. -const _: () = { - assert!(Domain::translatable_bits(AddressWidth::Bits48, 39) == 39); - assert!(Domain::first_address(39) < 1 << 39); - // What a window placed by the table depth alone would be, against the - // ceiling such a unit reports. - assert!(Domain::first_address(48) >= 1 << 39); - // A unit whose two limits agree is unchanged by any of this. - assert!(Domain::translatable_bits(AddressWidth::Bits48, 48) == 48); - assert!(Domain::translatable_bits(AddressWidth::Bits39, 39) == 39); - // And tables shallower than `MGAW` bind it the other way round. - assert!(Domain::translatable_bits(AddressWidth::Bits39, 48) == 39); -}; diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index 665088e21bf..adfa2773c63 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -29,9 +29,6 @@ pub const MSIX_ENTRY: u16 = 0; // the message and for the unit to put in an entry. pub(crate) const MSG_DEST: u32 = 0; -/// No requester id: a bus/device/function is sixteen bits, so this is none of them. -pub(crate) const NO_FUNCTION: u32 = u32::MAX; - /// Why a walk of a function's capability list answered no capability. pub enum NoCapability { /// The walk reached the list's terminator and nothing on it carried the id. diff --git a/kernel/src/iommu/fault.rs b/kernel/src/iommu/fault.rs new file mode 100644 index 00000000000..979c8102ea6 --- /dev/null +++ b/kernel/src/iommu/fault.rs @@ -0,0 +1,272 @@ +//! What a unit's fault handler does with each record it reads, whatever the +//! unit: bounded, allocating nothing and taking no lock. Function state lives +//! in a slice of exactly the enumerated functions, published once before any +//! unit is armed. +//! +//! Whatever the stream, the same things happen first: Bus Master Enable +//! cleared on the function that faulted, the first faulting function latched, +//! and a count kept per unit and per function. Clearing `BME` is also the +//! ceiling on a storm, since a function that cannot master the bus cannot +//! raise a second fault (PCI 3.0 §6.2.2, bit 2 of `COMMAND`). +//! +//! **What differs is who the fault is handed to.** A stream every driver of +//! which is in this kernel has nobody, so the terminal action is a halt — the +//! last thing that happens rather than the first. A stream a process drives has +//! an owner: `pcidev` is told, that claim refuses every later call, and the +//! machine goes on, because one process's bug taking the machine down is the +//! thing moving a driver out of the kernel was for. + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; + +use crate::drivers::pci::{self, PciDevice}; +use crate::iommu::StreamId; +use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; + +/// A `pcidev` slot number no claim has: this function is not driven by one. +const NO_SLOT: u32 = u32::MAX; + +/// One enumerated function, published before any unit is armed. +struct Function { + who: StreamId, + /// Physical base of its config window, through which a fault clears `BME`. + config: u64, + domain: AtomicU32, + // Faults a unit has reported against it; non-zero is the per-domain flag. + faults: AtomicU32, + /// The `pcidev` slot a process drives this function on, or [`NO_SLOT`]. + /// What decides whether a fault on it is terminal for the machine. + user_slot: AtomicU32, +} + +/// Exactly the functions this machine enumerated, leaked once by [`describe`]: +/// null until then, and never written again. +static FUNCTIONS: AtomicPtr<&'static [Function]> = AtomicPtr::new(core::ptr::null_mut()); + +fn functions() -> &'static [Function] { + let published = FUNCTIONS.load(Ordering::Acquire); + if published.is_null() { + return &[]; + } + // SAFETY: non-null only as `describe` stored it: a leaked box, never freed + // or written again, published with Release once it was whole. + unsafe { *published } +} + +/// [`Who::key`] of the first fault this machine took: what a later one says +/// is decided by what the first one already broke. +static FIRST: AtomicU64 = AtomicU64::new(u64::MAX); + +/// Every function this machine enumerated, before any unit is armed: the +/// handler reaches a faulting function's config space through this, with no lock. +pub fn describe(devices: &[PciDevice]) { + let functions: Vec = devices + .iter() + .map(|device| Function { + who: StreamId::pci(device.bus, device.dev, device.func), + config: DirectMap::phys_of(device.config_window().addr() as *const u8), + domain: AtomicU32::new(0), + faults: AtomicU32::new(0), + user_slot: AtomicU32::new(NO_SLOT), + }) + .collect(); + let functions: &'static [Function] = Box::leak(functions.into_boxed_slice()); + let first = FUNCTIONS.compare_exchange( + core::ptr::null_mut(), + Box::leak(Box::new(functions)), + Ordering::Release, + Ordering::Relaxed, + ); + assert!(first.is_ok(), "iommu: the fault handler's functions were described twice"); +} + +/// Record which domain a function moved to, for the flag the handler sets. +pub fn attached(stream: StreamId, domain: u16) { + if let Some(function) = find(Who::Function(stream)) { + function.domain.store(u32::from(domain), Ordering::Relaxed); + } +} + +/// Record that a process drives this function on `slot`, or no longer does. +/// +/// The handler reads it to decide what a fault on this stream is *terminal +/// for*: a kernel-driven function has nothing to hand a fault to and the +/// response is the halt in [`conclude`]; one a process drives has an owner, so +/// the record goes to that owner's claim and the machine stays up. +pub fn user_owned(stream: StreamId, slot: Option) { + if let Some(function) = find(Who::Function(stream)) { + function.user_slot.store(slot.map_or(NO_SLOT, |slot| slot as u32), Ordering::Release); + } +} + +fn find(who: Who) -> Option<&'static Function> { + match who { + Who::Function(stream) => functions().iter().find(|f| f.who == stream), + Who::Stream(_) => None, + } +} + +/// Whom a record names. +#[derive(Clone, Copy)] +pub enum Who { + /// A requester id: VT-d's source-id, or the function the IORT routes an + /// SMMU StreamID from. + Function(StreamId), + /// An SMMU StreamID no enumerated function is routed from. + // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. + #[allow(dead_code)] + Stream(u32), +} + +impl Who { + /// One number per `Who`, none of them `u64::MAX`. + const fn key(self) -> u64 { + match self { + Self::Function(stream) => stream.requester() as u64, + Self::Stream(stream) => 1 << 32 | stream as u64, + } + } +} + +impl core::fmt::Display for Who { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Function(stream) => write!(f, "{stream}"), + Self::Stream(stream) => write!(f, "{stream:#x}"), + } + } +} + +/// What the faulting transaction was, where the record says. +#[derive(Clone, Copy)] +pub enum Access { + Read, + Write, + /// An SMMU event that carries no transaction. + // The SMMUv3 is AArch64's alone, so x86-64 builds a variant it never makes. + #[allow(dead_code)] + Unrecorded, +} + +impl core::fmt::Display for Access { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(match self { + Self::Read => "read", + Self::Write => "write", + Self::Unrecorded => "none", + }) + } +} + +/// One record a unit reported, decoded by its backend. +pub struct Fault { + pub who: Who, + pub address: u64, + pub access: Access, + /// The unit's own number for why, and its name. + pub reason: u8, + pub name: &'static str, +} + +/// `fault`, read off unit `unit`, whose count of reported faults is `count`: +/// its function stopped, it is handed to whoever drives that function, and +/// its line written. `true` where that is nobody, which the drain ends on +/// with [`conclude`]. +pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { + let function = find(fault.who); + // First, before anything that can be slow or say no: a function that + // cannot master the bus raises no second fault, which is the whole of + // the ceiling on a storm. + if let Some(function) = function { + pci::stop_bus_mastering(config_window(function.config)); + } + let seen_here = function.map_or(0, |f| f.faults.fetch_add(1, Ordering::Relaxed) + 1); + let key = fault.who.key(); + let _ = FIRST.compare_exchange(u64::MAX, key, Ordering::AcqRel, Ordering::Relaxed); + // Before the line, so `owner=` in it is what was actually told. + let owner = function.and_then(|f| match f.user_slot.load(Ordering::Acquire) { + NO_SLOT => None, + slot => Some(slot as usize), + }); + if let Some(slot) = owner { + crate::pcidev::note_fault(slot); + } + let count = count.fetch_add(1, Ordering::Relaxed) + 1; + log!( + // `owner=` first, because it is the only field that decides whether + // this machine is still running: `tests/common/serial.rs` reads + // `iommu: DMA FAULT owner=kernel` as a death and the other form as + // a record. The reason's name is the last word: every gate takes it + // from there. + "iommu: DMA FAULT owner={} unit{unit} stream={} addr={:#018x} access={} reason={:#04x} \ + domain={} bme={} unitfaults={count} streamfaults={seen_here} first={} {}", + Owner(owner), + fault.who, + fault.address, + fault.access, + fault.reason, + Blamed(function), + if function.is_some() { "cleared" } else { "unknown-function" }, + if FIRST.load(Ordering::Relaxed) == key { 'y' } else { 'n' }, + fault.name, + ); + owner.is_none() +} + +/// The end of a drain that read `kernel_owned` faults nobody owns: any at all +/// halts the machine. +/// +/// The halt is the whole response and there is no recovery missing from it: a +/// faulting device reached an address this kernel never gave it, and nothing +/// here can know what else it already did. **So this halts and never panics** +/// — a report of one carries the fault record and `panic_reboot`'s arm line, +/// and no `panicked at` line; `capture` puts the fault on the panel first. +/// +/// **Only for a stream this kernel drives.** A function a process drives has +/// an owner to refuse: its bus mastering is already gone by the time this is +/// reached, its claim answers every later call `Io`, and the machine — whose +/// other drivers are untouched — goes on. +pub fn conclude(kernel_owned: usize) { + if kernel_owned > 0 { + crate::drivers::panic_console::capture(); + crate::panic::halt_all_cpus(); + } +} + +/// The config window of the function at `phys`, from the address [`describe`] +/// published. +fn config_window(phys: u64) -> Mmio { + // SAFETY: `phys` is `DirectMap::phys_of` a config window the ECAM mapping + // holds for the machine's life, one page wide. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), PAGE_SIZE) } +} + +/// Who a fault was handed to, in the line. A word rather than a number, because +/// which of the two it is decides whether this machine is still running. +struct Owner(Option); + +impl core::fmt::Display for Owner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + None => write!(f, "kernel"), + Some(slot) => write!(f, "slot{slot}"), + } + } +} + +/// What the handler can say about the faulting function's address space. The +/// middle answer is deliberately weak: a function the kernel never attached +/// carries domain id 0 whether it sits on the unit's default or on a domain an +/// actuator bound by hand, so the label claims only that nothing recorded one. +struct Blamed(Option<&'static Function>); + +impl core::fmt::Display for Blamed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0.map(|function| function.domain.load(Ordering::Relaxed)) { + None => write!(f, "unknown"), + Some(0) => write!(f, "unrecorded"), + Some(id) => write!(f, "{id}"), + } + } +} diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 3bcf02d17d1..003e4cd1423 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -1,6 +1,6 @@ //! The unit that decides what a device may reach. //! -//! Inventories the machine's IOMMU units, gives every enumerated PCI function an identity-mapped context entry, turns translation on, remaps every interrupt source through a source-id-verified table entry, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above `vtd/` stay backend-neutral so a second backend drops in without moving the seam. +//! Inventories the machine's IOMMU units, turns translation on, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above the backend stay backend-neutral, and what every backend decides alike is declared here once: a domain's addresses ([`window`]) and what a fault record ends ([`fault`]). //! //! The refusal is deliberately not yet built for a driver in this kernel: landing it before any userspace driver exists would cost every machine and protect nothing. A function a *process* drives is the other case and is refused ([`OwnSpace`], [`remapping`]), because a descriptor it writes a physical address into is an arbitrary read and write over all of memory, and a message it sends unremapped is any vector at any CPU; its message is its claim slot's own entry ([`Remapped`]). //! @@ -11,6 +11,9 @@ use crate::arch::iommu_unit as unit; +pub(crate) mod fault; +pub(crate) mod window; + /// The address width a device's translations cover. /// /// `Bits57` is omitted even where a unit advertises it, because it needs a fifth page-table level no machine in reach uses. @@ -428,7 +431,7 @@ pub fn remap_pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Delivery) { - unit::fault::user_owned(StreamId::pci(bus, device, function), slot); + fault::user_owned(StreamId::pci(bus, device, function), slot); } /// Reached from the IDT gate the unit's own `FEDATA` names. diff --git a/kernel/src/iommu/window.rs b/kernel/src/iommu/window.rs new file mode 100644 index 00000000000..74f31029fcc --- /dev/null +++ b/kernel/src/iommu/window.rs @@ -0,0 +1,156 @@ +//! A domain's device addresses — where they start, where they end and which +//! of them are handed out — for every backend. +//! +//! They start a quarter of the way up what the unit translates and above all +//! memory, so a descriptor still carrying one names nothing a domain maps and +//! faults rather than landing on a page. They end under the first root-bridge +//! window or reserved region over that start: a bridge may route a window +//! peer-to-peer before the unit sees the request (PCIe Base §2.4), and a +//! region firmware reserved is not a domain's (VT-d §3.16). An address is +//! handed out once, unmapped or not: a device holding a stale one would reach +//! whatever took its place. + +use crate::iommu::{IommuError, Iova}; +use crate::mm::PAGE_2M; + +/// One domain's device addresses. +#[derive(Clone, Copy)] +pub struct Window { + floor: u64, + ceiling: u64, + /// The first not yet handed out. + next: u64, +} + +/// A quarter of the way up `translatable` bits of device address. +pub const fn first_address(translatable: u8) -> u64 { + 1 << (translatable - 2) +} + +/// Where a window over `translatable` bits starting at `floor` ends: under the +/// first of `reserved` that reaches above `floor`; at or below `floor` where +/// one of them covers it. +const fn ceiling(translatable: u8, floor: u64, reserved: &[(u64, u64)]) -> u64 { + let mut ceiling = 1u64 << translatable; + let mut i = 0; + while i < reserved.len() { + let (start, end) = reserved[i]; + if end > floor && start < ceiling { + ceiling = start; + } + i += 1; + } + ceiling +} + +impl Window { + /// A new domain's window over `translatable` bits, clear of `reserved`, + /// with `room` bytes of it handed out first; refused where it would start + /// at or below memory or hold less than that, and at least one leaf. + pub fn new(translatable: u8, reserved: &[(u64, u64)], room: u64) -> Result<(Self, Iova), IommuError> { + let floor = first_address(translatable); + let top = crate::mm::pmm::top(); + if floor <= top { + return Err(IommuError::WindowBelowMemory { translatable, floor, top }); + } + let ceiling = ceiling(translatable, floor, reserved); + let mut window = Self::starting_at(floor, ceiling); + match window.reserve(room) { + Some(first) if ceiling.saturating_sub(floor) >= PAGE_2M => Ok((window, first)), + _ => Err(IommuError::NoRoom { floor, ceiling, room }), + } + } + + const fn starting_at(floor: u64, ceiling: u64) -> Self { + Self { floor, ceiling, next: floor } + } + + pub const fn floor(&self) -> u64 { + self.floor + } + + pub const fn ceiling(&self) -> u64 { + self.ceiling + } + + /// `bytes`, rounded up to whole 2 MiB leaves, of addresses never handed out. + pub const fn reserve(&mut self, bytes: u64) -> Option { + let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return None }; + let Some(end) = self.next.checked_add(span) else { return None }; + if end > self.ceiling { + return None; + } + let at = Iova::translated(self.next); + self.next = end; + Some(at) + } + + /// Whether `bytes` at `at` is room [`Self::reserve`] already handed out: + /// the only room a mapping may be placed in by address. + /// + /// `at` must itself be a leaf `reserve` could have returned, not merely + /// inside a handed-out span: a leaf's index floors to the enclosing leaf, + /// so an unaligned `at` this let through would place a mapping short of, + /// or overlapping, where the caller named. + pub const fn handed_out(&self, at: Iova, bytes: u64) -> bool { + if !at.raw().is_multiple_of(PAGE_2M) || at.raw() < self.floor { + return false; + } + let Some(span) = bytes.checked_next_multiple_of(PAGE_2M) else { return false }; + match at.raw().checked_add(span) { + Some(end) => end <= self.next, + None => false, + } + } +} + +/// [`Window`] and [`ceiling`] at their boundaries, at compile time: the +/// kernel binary has no test harness. +const _: () = { + const FLOOR: u64 = first_address(48); + let mut one = Window::starting_at(FLOOR, FLOOR + 2 * PAGE_2M); + assert!(matches!(one.reserve(1), Some(at) if at.raw() == FLOOR)); + // Exactly what one `reserve` handed out. + assert!(one.handed_out(Iova::translated(FLOOR), PAGE_2M)); + // Short of the floor, past what has been reserved, rounded up past it, + // off a leaf boundary, and past the end of the address space: nothing. + assert!(!one.handed_out(Iova::translated(FLOOR - PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR), PAGE_2M + 1)); + assert!(!one.handed_out(Iova::translated(FLOOR + 1), 0)); + assert!(!one.handed_out(Iova::translated(u64::MAX - PAGE_2M + 1), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR), u64::MAX)); + // Past the ceiling, and past the end of the address space, nothing. + assert!(one.reserve(PAGE_2M + 1).is_none()); + assert!(one.reserve(u64::MAX).is_none()); + assert!(matches!(one.reserve(PAGE_2M), Some(at) if at.raw() == FLOOR + PAGE_2M)); + // Two leaves handed out; the second is room in its own right, and a + // mid-span address off a leaf boundary is not. + assert!(one.handed_out(Iova::translated(FLOOR + PAGE_2M), PAGE_2M)); + assert!(!one.handed_out(Iova::translated(FLOOR + 1), PAGE_2M)); + assert!(one.reserve(1).is_none()); +}; + +/// [`ceiling`] over the windows the T14's firmware declares, unsorted as it +/// declares them, and over `virt`'s: a 39-bit unit's window ends where the +/// first root-bridge window above its floor begins, one reaching over the +/// floor leaves it nothing, and windows all below the floor leave the whole +/// input. +const _: () = { + const FLOOR: u64 = first_address(39); + const T14: [(u64, u64); 7] = [ + (0xA200_0000, 0xBD00_0000), + (0x40_0000_0000, 0x60_3DC0_0000), + (0xA080_0000, 0xA200_0000), + (0xBD00_0000, 0xC000_0000), + (0xFF00_0000, 0xFFB8_0000), + (0xFFD3_A070, 0x1_0000_0000), + (0x60_3DC0_0000, 0x80_0000_0000), + ]; + assert!(FLOOR == 0x20_0000_0000); + assert!(ceiling(39, FLOOR, &T14) == 0x40_0000_0000); + assert!(ceiling(39, FLOOR, &[]) == 1 << 39); + assert!(ceiling(39, FLOOR, &[(0x10_0000_0000, FLOOR + 1)]) < FLOOR); + const VIRT: [(u64, u64); 2] = [(0x1000_0000, 0x3f00_0000), (0x80_0000_0000, 0x100_0000_0000)]; + assert!(ceiling(48, first_address(48), &VIRT) == 1 << 48); +}; diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 8c0339617d9..d5138775c97 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -727,8 +727,8 @@ pub enum Profile { /// `HVC`, as under HVF. `virt_el1_smp`'s machine while a boot's last word /// can miss the console under HVF. VirtTcg, - /// [`Profile::Virt`] with its SMMUv3 and QEMU's `iommu-testdev`, a function - /// that writes where it is told to through the unit. + /// [`Profile::Virt`] with its SMMUv3 and two of QEMU's `iommu-testdev`, a + /// function that writes where it is told to through the unit. VirtSmmu, } @@ -870,9 +870,9 @@ struct Shape { rng: bool, } -/// Whether `virt` has its SMMUv3. With it comes QEMU's `iommu-testdev`, whose -/// writes are the only DMA a guest of this suite can aim at an address of its -/// choosing: a unit no function writes through is a unit no test reads. +/// Whether `virt` has its SMMUv3. With it come two of QEMU's `iommu-testdev`, +/// whose writes are the only DMA a guest of this suite can aim at an address +/// of its choosing: a unit no function writes through is a unit no test reads. #[derive(Clone, Copy, PartialEq, Eq)] enum Smmu { Absent, @@ -2212,6 +2212,9 @@ fn qemu_command( qemu.arg("-device").arg("virtio-rng-pci"); } if shape.smmu == Smmu::WithTestdev { + // Two, the first enumerated below the second: the kernel's selftest + // routes nothing for the first, whose entry the table holds. + qemu.arg("-device").arg("iommu-testdev"); qemu.arg("-device").arg("iommu-testdev"); } diff --git a/tests/toyos.rs b/tests/toyos.rs index d9684a50e5c..142cf4c3f10 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2021,13 +2021,15 @@ fn virt_no_seed_refused(profile: qemu::Profile, test_config: &Path) -> Result<() Ok(()) } -/// The SMMUv3 armed from the IORT, judged by what QEMU's `iommu-testdev` can -/// and cannot write through it (`kernel/src/arch/aarch64/smmu/selftest.rs`): -/// `GBPA` read back aborting, every stream's entry aborting, a `CMD_SYNC` -/// consumed; the device's write refused on the entry it starts with, landing -/// where its own domain maps it, and refused where the domain maps nothing — -/// that one recorded, its event taken on the wired SPI and named, and the -/// machine halted for a function no process drives. +/// The SMMUv3 armed from the IORT, judged by what two of QEMU's +/// `iommu-testdev` can and cannot write through it +/// (`kernel/src/arch/aarch64/smmu/selftest.rs`): `GBPA` read back aborting, +/// a `CMD_SYNC` consumed; one device's write refused on the entry its stream +/// starts with and nothing recorded, landing where its own domain maps it, +/// and refused once the domain takes that address back; the other's, under a +/// StreamID no function is routed from inside the stream table, refused. +/// Those two are recorded, the unit's first and second events, taken on the +/// wired SPI, named, and the machine halted for functions no process drives. fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { const FAULT: &str = "iommu: DMA FAULT owner=kernel"; let options = BootOptions { @@ -2043,26 +2045,46 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { } } let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[], options); - let rest = qemu.drain_until(Duration::from_secs(30), |l| l.contains(FAULT)); + let rest = qemu.drain_until(Duration::from_secs(30), |l| l.contains(FAULT) && l.contains(" unitfaults=2 ")); let serial = format!("{}\n{rest}", qemu.boot_log()); let line = |want: &str| serial.lines().find(|l| l.contains(want)).map(str::to_string); + let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); let Some(armed) = line("armed, CR0ACK") else { return Err(format!("the SMMUv3 was never armed\nserial:\n{serial}")); }; - for want in ["every entry aborting", "a CMD_SYNC consumed", "events on SPI 106"] { + for want in ["every other entry invalid", "a CMD_SYNC consumed", "events on SPI 106"] { if !armed.contains(want) { return Err(format!("{want:?} not in {armed:?}")); } } eprintln!(" [virt] {armed}"); + let table = armed.split_once("in a table of ").and_then(|(_, rest)| rest.split(',').next()?.parse::().ok()); + let Some(table) = table else { + return Err(format!("{armed:?} names no table size")); + }; // `SMMU_GBPA.ABORT`, bit 20, as the unit reads it back. let gbpa = line(", every transaction aborts while SMMUEN is clear") - .and_then(|l| l.split_once(": GBPA 0x").and_then(|(_, v)| u32::from_str_radix(v.split(',').next()?, 16).ok())); + .and_then(|l| l.split_once(": GBPA ").and_then(|(_, v)| hex(v.split(',').next()?))); match gbpa { Some(gbpa) if gbpa & 1 << 20 != 0 => eprintln!(" [virt] GBPA {gbpa:#x}: ABORT"), _ => return Err(format!("GBPA does not read back aborting: {gbpa:?}\nserial:\n{serial}")), } - for (said, verdict) in [("on the entry every stream starts with", "refused"), ("mapped to", "landed there")] { + let Some(unrouted) = line("is given no route") else { + return Err(format!("the selftest left no function unrouted\nserial:\n{serial}")); + }; + eprintln!(" [virt] {unrouted}"); + let Some(stream) = unrouted.split_once(", StreamID ").and_then(|(_, rest)| hex(rest.split(',').next()?)) else { + return Err(format!("{unrouted:?} names no StreamID")); + }; + if stream >= table { + return Err(format!("StreamID {stream:#x} is past a table of {table}: {unrouted:?}")); + } + for (said, verdict) in [ + ("on the entry its stream starts with", "refused"), + ("mapped to", "landed there"), + ("which its domain no longer maps", "refused"), + ("under no route", "refused"), + ] { let Some(found) = line(said) else { return Err(format!("the selftest never said {said:?}\nserial:\n{serial}")); }; @@ -2071,22 +2093,31 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { return Err(format!("{found}\nserial:\n{serial}")); } } - let Some(aimed) = line("which its domain does not map") else { - return Err(format!("the selftest never aimed a write past its domain\nserial:\n{serial}")); - }; - let aimed_at = aimed.split_once("smmu-selftest: ").and_then(|(_, said)| said.split_once(" writes at 0x")); - let Some((function, Ok(at))) = aimed_at.map(|(function, rest)| { - (function, u64::from_str_radix(rest.split(',').next().unwrap_or_default(), 16)) - }) else { - return Err(format!("{aimed:?} names no function and address")); + let Some(again) = line("which its domain no longer maps") else { + return Err(format!("the selftest never wrote where its domain took a mapping back\nserial:\n{serial}")); }; - let Some(fault) = line(FAULT) else { - return Err(format!("no event reached the handler\nserial:\n{serial}")); + let aimed = again.split_once("smmu-selftest: ").and_then(|(_, said)| said.split_once("'s write at ")); + let Some((function, Some(at))) = aimed.map(|(function, rest)| (function, rest.split(' ').next().and_then(hex))) + else { + return Err(format!("{again:?} names no function and address")); }; - eprintln!(" [virt] {fault}"); - let named = format!("stream={function} addr={at:#018x} access=write"); - if !fault.contains(&named) || !fault.ends_with("F_TRANSLATION") { - return Err(format!("the event does not name {named:?} and F_TRANSLATION: {fault}")); + let faults: Vec<&str> = serial.lines().filter(|l| l.contains(FAULT)).collect(); + for fault in &faults { + eprintln!(" [virt] {fault}"); + } + // The unit's first and second events, so the write on the entry its + // stream starts with was not recorded. + let wants = [ + (format!("stream={function} addr={at:#018x} access=write"), " unitfaults=1 ", "F_TRANSLATION"), + (format!("stream={stream:#x} "), " unitfaults=2 ", "C_BAD_STE"), + ]; + if faults.len() != wants.len() { + return Err(format!("{} events reached the handler, not {}\nserial:\n{serial}", faults.len(), wants.len())); + } + for (fault, (named, count, name)) in faults.iter().zip(&wants) { + if !fault.contains(named.as_str()) || !fault.contains(count) || !fault.ends_with(name) { + return Err(format!("the event does not name {named:?},{count}and {name}: {fault}")); + } } Ok(()) } diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs index 63106680031..c29c8c971da 100644 --- a/toyos-smmu/src/queue.rs +++ b/toyos-smmu/src/queue.rs @@ -237,6 +237,8 @@ pub struct Attempt { pub struct Event { /// `StreamID`, bits [63:32]. pub stream: u32, + /// The event's number, bits [7:0], which `code` names. + pub number: u8, pub code: Code, pub attempt: Option, } @@ -270,5 +272,5 @@ pub const fn event(record: [u64; 4]) -> Event { } _ => None, }; - Event { stream: (record[0] >> 32) as u32, code, attempt } + Event { stream: (record[0] >> 32) as u32, number: record[0] as u8, code, attempt } } diff --git a/toyos-smmu/src/unit.rs b/toyos-smmu/src/unit.rs index b706371d68d..f6ffe05782b 100644 --- a/toyos-smmu/src/unit.rs +++ b/toyos-smmu/src/unit.rs @@ -20,17 +20,24 @@ pub const IRQ_CTRL: usize = 0x050; pub const IRQ_CTRLACK: usize = 0x054; pub const GERROR: usize = 0x060; pub const GERRORN: usize = 0x064; +pub const GERROR_IRQ_CFG0: usize = 0x068; pub const STRTAB_BASE: usize = 0x080; pub const STRTAB_BASE_CFG: usize = 0x088; pub const CMDQ_BASE: usize = 0x090; pub const CMDQ_PROD: usize = 0x098; pub const CMDQ_CONS: usize = 0x09C; pub const EVENTQ_BASE: usize = 0x0A0; +pub const EVENTQ_IRQ_CFG0: usize = 0x0B0; +/// Present only where [`IDR0_PRI`] is set (§6.3.34). +pub const PRIQ_IRQ_CFG0: usize = 0x0D0; /// The event queue's indexes are in register page 1, 64 KiB above page 0 /// (§6.1, §6.2.2): `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS` at 0xA8 and 0xAC of it. pub const EVENTQ_PROD: usize = 0x1_00A8; pub const EVENTQ_CONS: usize = 0x1_00AC; +/// `SMMU_IDR0.PRI` [16] (§6.3.1): the unit has a PRI queue, and its registers. +pub const IDR0_PRI: u32 = 1 << 16; + /// `SMMU_CR0` (§6.3.9), and `SMMU_CR0ACK`, which reads each bit back once /// the unit has acted on it. pub const CR0_SMMUEN: u32 = 1 << 0; @@ -44,9 +51,10 @@ pub const CR0_CMDQEN: u32 = 1 << 3; pub const CR1_WRITE_BACK: u32 = 0b11 << 10 | 0b01 << 8 | 0b01 << 6 | 0b11 << 4 | 0b01 << 2 | 0b01; /// `SMMU_CR2` (§6.3.12): `RECINVSID` [1], so a transaction under a StreamID -/// past the stream table is recorded as well as aborted, and `PTM` [2], so -/// no CPU's broadcast TLB invalidation reaches the unit's entries: its ASIDs -/// are its own. `E2H` [0] stays clear. +/// past the stream table is recorded as well as aborted, as one under an +/// invalid entry in it is, and `PTM` [2], so no CPU's broadcast TLB +/// invalidation reaches the unit's entries: its ASIDs are its own. `E2H` [0] +/// stays clear. pub const CR2_RECORD_PRIVATE: u32 = 1 << 1 | 1 << 2; /// `SMMU_GBPA` (§6.3.15): what happens to every transaction while `SMMUEN` diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs index ff58b28810b..2a29c88f115 100644 --- a/toyos-smmu/tests/encodings.rs +++ b/toyos-smmu/tests/encodings.rs @@ -215,7 +215,7 @@ fn a_context_descriptors_ips_is_the_units_output_size_and_no_more_than_48_bits() /// (§6.3.130, §6.3.131). #[test] fn the_registers_sit_where_the_register_overview_has_them() { - let page0: [(usize, usize); 18] = [ + let page0: [(usize, usize); 21] = [ (0x0000, unit::IDR0), (0x0004, unit::IDR1), (0x0014, unit::IDR5), @@ -228,12 +228,15 @@ fn the_registers_sit_where_the_register_overview_has_them() { (0x0054, unit::IRQ_CTRLACK), (0x0060, unit::GERROR), (0x0064, unit::GERRORN), + (0x0068, unit::GERROR_IRQ_CFG0), (0x0080, unit::STRTAB_BASE), (0x0088, unit::STRTAB_BASE_CFG), (0x0090, unit::CMDQ_BASE), (0x0098, unit::CMDQ_PROD), (0x009C, unit::CMDQ_CONS), (0x00A0, unit::EVENTQ_BASE), + (0x00B0, unit::EVENTQ_IRQ_CFG0), + (0x00D0, unit::PRIQ_IRQ_CFG0), ]; for (specified, declared) in page0 { assert_eq!(declared, specified); @@ -254,6 +257,7 @@ fn the_control_values_hold_each_field_where_its_register_has_it() { ); // E2H clear, RECINVSID, PTM. assert_eq!(u64::from(unit::CR2_RECORD_PRIVATE), word(&[(0, 0, 0), (1, 1, 1), (2, 2, 1)])); + assert_eq!(u64::from(unit::IDR0_PRI), word(&[(16, 16, 1)])); assert_eq!(u64::from(unit::GBPA_ABORT | unit::GBPA_UPDATE), word(&[(20, 20, 1), (31, 31, 1)])); assert_eq!(u64::from(unit::IRQ_GERROR | unit::IRQ_EVENTQ), word(&[(0, 0, 1), (2, 2, 1)])); assert_eq!( @@ -489,6 +493,7 @@ fn a_translation_fault_names_its_stream_its_address_and_whether_it_wrote() { event(record(number, 0x0001_0008, 0xffff_8000_dead_b000, read)), Event { stream: 0x0001_0008, + number, code, attempt: Some(Attempt { address: 0xffff_8000_dead_b000, write: !read }) } @@ -524,7 +529,7 @@ fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { let decoded = event(record(number, 7, 0x1000, false)); assert_eq!(decoded.code.name() == "unnamed", !named.iter().any(|(n, _)| *n == number), "event {number:#x}"); let code = named.iter().find(|(n, _)| *n == number).map_or(Code::Other(number), |(_, code)| *code); - assert_eq!((decoded.stream, decoded.code), (7, code), "event {number:#x}"); + assert_eq!((decoded.stream, decoded.number, decoded.code), (7, number, code), "event {number:#x}"); assert_eq!(decoded.attempt.is_some(), (0x10..=0x13).contains(&number), "event {number:#x}"); } } From 93c4107c88f873a8bb9a2b331d166b9dca2455f6 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 11:28:28 +0200 Subject: [PATCH 3/4] A DMA fault naming no enumerated function is refused and the machine goes on; the SMMUv3 reads at most a queue's worth of events per interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared fault policy halted on every record with no user owner, and that included a record naming no enumerated function: a requester this kernel never took on, whose write the unit had already refused. A hot-plugged function, a VF a claimed PF enabled, or any device writing under a StreamID no route names could then halt the machine at will. Fail-fast answers a defect of this kernel; a refused device input is not one. `report` now answers "halt" only for an enumerated function this kernel drives, and a record naming none is logged under `owner=none`, which the harness reads as a record (never clean) and not as a death. The VT-d line is unchanged but for that word, and an unenumerated requester on VT-d no longer halts. Nothing can clear `BME` on a requester nobody enumerated, so the handler's work per interrupt is bounded by itself: VT-d's already reads at most `CAP.NFR` records, and the SMMUv3's drain now reads at most the queue's entries per interrupt and, if records remain, pends its SPI again through `GICD_ISPENDR`, because the unit raises no edge for a record left behind (IHI 0070 H.a §3.18.2). `smmu-selftest` now has the unrouted testdev write twice, both read in one drain, and says the machine went on; the kernel-driven function's write on its taken-back address comes last and is the halting one. `virt_smmu` reads on to `panic_reboot::arm`'s line, which the halt path writes once every other CPU is stopped, and reds on a `FAIL` line or any death but the record: before, it returned at the record and never read the selftest's "the machine went on" panic. `toyos-smmu`'s per-mnemonic name test is deleted: it was a second copy of `Code::name`'s match, and the two names a gate reads are held by `virt_smmu`. A function the SMMUv3 does not route still panics at `attach` rather than being refused at `create`; filed with an owner and an exit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...-smmuv3-does-not-route-panics-at-attach.md | 24 +++++++ kernel/src/arch/aarch64/irqchip.rs | 10 +++ kernel/src/arch/aarch64/smmu/fault.rs | 17 +++-- kernel/src/arch/aarch64/smmu/mod.rs | 4 +- kernel/src/arch/aarch64/smmu/selftest.rs | 58 +++++++++------ kernel/src/arch/x86_64/vtd/fault.rs | 8 +-- kernel/src/iommu/fault.rs | 67 ++++++++++------- tests/checks/serial.rs | 17 +++++ tests/common/serial.rs | 14 ++-- tests/toyos.rs | 72 ++++++++++++++----- toyos-smmu/src/queue.rs | 5 ++ toyos-smmu/tests/encodings.rs | 28 -------- 12 files changed, 217 insertions(+), 107 deletions(-) create mode 100644 issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md diff --git a/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md b/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md new file mode 100644 index 00000000000..f7a60b0fde3 --- /dev/null +++ b/issues/a-function-the-smmuv3-does-not-route-panics-at-attach.md @@ -0,0 +1,24 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# A function the SMMUv3 does not route panics at attach + +`DeviceSpace::create` and `OwnSpace::create` take no function, so on the +SMMUv3 (`kernel/src/arch/aarch64/smmu/domain.rs`) a domain is made for any +caller. `attach` then looks the function up in the unit's routes, and +`Live::stream` panics where it finds none: an enumerated function whose IORT +route names another unit, names none, or was refused +(`kernel/src/arch/aarch64/smmu/mod.rs`, `init`'s route loop). A gap in a +firmware table becomes a kernel panic at `attach`, where it is a refusal at +`create`. Nothing on QEMU's `virt` reaches it: every function there is routed +through its one unit. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6, whose claim through an +SMMUv3 domain is the first caller to hand `create` a function a process names. + +**Exit.** `create` takes the function and refuses, by name, one the unit does +not route, before an id is spent; `attach` has no lookup that can fail; and a +guest test, or a host test of the refusal, reds with the refusal deleted. diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 04ec59f72e6..fbff0cf92c5 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -72,6 +72,7 @@ const GICD_TYPER: u64 = 0x0004; const GICD_IGROUPR: u64 = 0x0080; const GICD_ISENABLER: u64 = 0x0100; const GICD_ICENABLER: u64 = 0x0180; +const GICD_ISPENDR: u64 = 0x0200; const GICD_ICPENDR: u64 = 0x0280; const GICD_IPRIORITYR: u64 = 0x0400; const GICD_ICFGR: u64 = 0x0C00; @@ -325,6 +326,15 @@ pub(super) fn route_iommu_events(intid: u32) -> Result<(), u32> { Ok(()) } +/// Pend the IOMMU's event SPI again, for records its handler left behind: the +/// unit raises none for them. Taken once the handler's `end` deactivates it. +pub(super) fn pend_iommu_events() { + let intid = IOMMU_EVENTS.load(Relaxed); + assert!(intid != 0, "GIC: the IOMMU's event SPI is pended before it is routed"); + let gicd = Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME); + gicd.write_u32(GICD_ISPENDR + u64::from(intid / 32) * 4, 1 << (intid % 32)); +} + /// The IOMMU's event SPI, or zero where [`route_iommu_events`] routed none. pub(super) fn iommu_events() -> u32 { IOMMU_EVENTS.load(Relaxed) diff --git a/kernel/src/arch/aarch64/smmu/fault.rs b/kernel/src/arch/aarch64/smmu/fault.rs index 8c273b10cdc..995602fd0f9 100644 --- a/kernel/src/arch/aarch64/smmu/fault.rs +++ b/kernel/src/arch/aarch64/smmu/fault.rs @@ -6,9 +6,12 @@ //! is atomics. What a record ends is `crate::iommu::fault`'s, the policy every //! backend applies; this reads the queue into it. //! -//! The unit pulses its interrupt only as the queue goes from empty to -//! non-empty (IHI 0070 H.a §3.18.2), so a drain ends only on a queue it read -//! empty: a record left behind raises nothing. +//! One interrupt reads at most a queue's worth of records, so a device that +//! never stops writing, which nothing here can stop when no enumerated +//! function is behind its stream, cannot hold the CPU. The unit pulses its +//! interrupt only as the queue goes from empty to non-empty (IHI 0070 H.a +//! §3.18.2), so a record left behind raises nothing: a drain that stops short +//! of empty pends the interrupt again itself. use alloc::boxed::Box; use core::sync::atomic::{AtomicPtr, AtomicU32, Ordering}; @@ -64,7 +67,10 @@ pub fn service() { let mut kernel_owned = 0usize; let mut cons = regs.read(reg::EVENTQ_CONS); let mut prod = regs.read(reg::EVENTQ_PROD); - while !events.is_empty(prod, cons) { + for _ in 0..events.entries() { + if events.is_empty(prod, cons) { + break; + } if events.overflowed(prod, cons) { log!("iommu: the SMMUv3's event queue overflowed: earlier events are lost"); } @@ -86,6 +92,9 @@ pub fn service() { regs.write(reg::EVENTQ_CONS, cons); prod = regs.read(reg::EVENTQ_PROD); } + if !events.is_empty(prod, cons) { + super::super::irqchip::pend_iommu_events(); + } let errors = reg::active_errors(regs.read(reg::GERROR), regs.read(reg::GERRORN)); if errors & reg::GERROR_EVENTQ_ABORT != 0 { log!("iommu: the SMMUv3 could not write its event queue: events are lost (GERROR {errors:#x})"); diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index 6df76be26e4..379fa132a55 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -11,8 +11,8 @@ //! mastering the bus until its driver resets it. **Every stream no //! enumerated function is routed from has one answer**, inside the stream //! table or past it: it aborts and is recorded — `C_BAD_STE` on an entry left -//! invalid, `C_BAD_STREAMID` under `CR2.RECINVSID` — and, nobody driving it, -//! halts the machine, as an unenumerated requester does on VT-d. A unit this +//! invalid, `C_BAD_STREAMID` under `CR2.RECINVSID` — and the machine goes on, +//! as it does for an unenumerated requester on VT-d. A unit this //! kernel cannot program, and every unit of a machine whose IORT names more //! than one, is left aborting every transaction, and no domain is given. //! diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs index d6cab2e2cfd..72f7d47a0bd 100644 --- a/kernel/src/arch/aarch64/smmu/selftest.rs +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -6,17 +6,20 @@ //! with nothing recorded; //! 2. on a domain of its own, it writes at an address the domain maps, and //! lands at the memory mapped there and nowhere else; -//! 3. once the domain takes that address back, it writes there again, and is -//! refused and recorded; -//! 4. the other, which [`super::init`] gives no route, as it gives none to a -//! function it never enumerated, writes under a StreamID inside the stream -//! table, and is refused and recorded. +//! 3. the other, which [`super::init`] gives no route, as it gives none to a +//! function it never enumerated, writes twice under a StreamID inside the +//! stream table, and is refused and recorded twice, and the machine goes +//! on; +//! 4. once the domain takes its address back, the first writes there again, +//! and is refused and recorded, and the machine halts: the function is +//! one this kernel drives. //! -//! The last two are made with this CPU's interrupts masked, so one drain reads -//! both: the event interrupt reaches the handler, which names each and, their -//! functions being no process's, halts the machine. Each step says what it -//! saw on its own line; the harness judges them, and the records by the -//! handler's, whose count says the first step recorded nothing. +//! The writes of 3, and the write of 4, are each made with this CPU's +//! interrupts masked, so one drain reads both of 3's and 4's verdict is on +//! the console before its event is taken, on this CPU, which the event SPI is +//! routed to. Each step says what it saw on its own line; the harness judges +//! them, and the records by the handler's, whose count says the first step +//! recorded nothing. use crate::drivers::pci::PciDevice; use crate::iommu::OwnSpace; @@ -46,7 +49,8 @@ const BAR_BYTES: u64 = 0x1000; const LANDED: u32 = 0; const REFUSED: u32 = 0xdead_0002; -/// How long the two events are given to reach this CPU once it unmasks. +/// How long a refused write's events are given to reach this CPU once it +/// unmasks. const EVENT_TIMEOUT: Tripwire = Tripwire::absurd( Duration::from_secs(1), "the unit records each event and pulses its interrupt while the refused write is still being made", @@ -118,27 +122,41 @@ pub(super) fn run(devices: &[PciDevice]) { space.unmap(at, PAGE_2M).expect("smmu-selftest: the leaf it placed"); let before = super::fault::recorded(); + let masked = crate::arch::IrqGuard::close(); + let stray_wrote = [write(stray_regs, phys, phys), write(stray_regs, phys, phys)]; + for (i, answer) in stray_wrote.into_iter().enumerate() { + log!( + "smmu-selftest: {unrouted}'s write {} at {phys:#x}, under no route, answered {answer:#x}: {}", + i + 1, + verdict(answer, REFUSED) + ); + } + drop(masked); + events_reach_this_cpu(before + 2, "the unrouted function's two refused writes"); + log!("smmu-selftest: the unrouted function's two events were read, and the machine goes on"); + let masked = crate::arch::IrqGuard::close(); let taken_back = write(regs, at + 0x40, phys + 0x40); - let stray_wrote = write(stray_regs, phys, phys); log!( "smmu-selftest: {function}'s write at {:#x} again, which its domain no longer maps, answered {taken_back:#x}: {}", at + 0x40, verdict(taken_back, REFUSED) ); - log!( - "smmu-selftest: {unrouted}'s write at {phys:#x}, under no route, answered {stray_wrote:#x}: {}", - verdict(stray_wrote, REFUSED) - ); drop(masked); + events_reach_this_cpu(before + 3, "the kernel-driven function's refused write"); + panic!("smmu-selftest: FAIL: the handler read the event of a function this kernel drives and the machine went on"); +} + +/// Spin until the handler has read `want` events in all, which `what` made; +/// `FAIL` past [`EVENT_TIMEOUT`]. +fn events_reach_this_cpu(want: u32, what: &str) { let deadline = crate::clock::nanos_since_boot() + EVENT_TIMEOUT.nanos(); - while super::fault::recorded() < before + 2 { + while super::fault::recorded() < want { assert!( crate::clock::nanos_since_boot() < deadline, - "smmu-selftest: FAIL: {} of the two refused writes' events reached this CPU within {EVENT_TIMEOUT}", - super::fault::recorded() - before + "smmu-selftest: FAIL: the handler had read {} events, not {want}, {EVENT_TIMEOUT} after {what}", + super::fault::recorded() ); core::hint::spin_loop(); } - panic!("smmu-selftest: FAIL: the handler read the events of functions no process drives and the machine went on"); } diff --git a/kernel/src/arch/x86_64/vtd/fault.rs b/kernel/src/arch/x86_64/vtd/fault.rs index 729aa99099e..6fdae956d32 100644 --- a/kernel/src/arch/x86_64/vtd/fault.rs +++ b/kernel/src/arch/x86_64/vtd/fault.rs @@ -119,8 +119,8 @@ pub fn service() { crate::arch::apic::eoi(); } -/// Records drained, of which the answer counts only the ones on a stream this -/// kernel drives — the ones with nobody to hand the fault to. +/// Records drained, of which the answer counts only the ones on an enumerated +/// function this kernel drives — the ones the machine halts on. fn drain(index: usize, regs: Mmio, records: u64, count: u32) -> usize { let status = regs.read_u32(FSTS_REG); if status & FSTS_OVERFLOW != 0 { @@ -151,9 +151,9 @@ fn drain(index: usize, regs: Mmio, records: u64, count: u32) -> usize { reason, name: reason_name(reason), }; - let unowned = policy::report(index, &UNITS[index].faults, fault); + let halts = policy::report(index, &UNITS[index].faults, fault); clear_record(regs, record); - if unowned { + if halts { seen += 1; } } diff --git a/kernel/src/iommu/fault.rs b/kernel/src/iommu/fault.rs index 979c8102ea6..8c6eb26e686 100644 --- a/kernel/src/iommu/fault.rs +++ b/kernel/src/iommu/fault.rs @@ -9,13 +9,17 @@ //! ceiling on a storm, since a function that cannot master the bus cannot //! raise a second fault (PCI 3.0 §6.2.2, bit 2 of `COMMAND`). //! -//! **What differs is who the fault is handed to.** A stream every driver of -//! which is in this kernel has nobody, so the terminal action is a halt — the -//! last thing that happens rather than the first. A stream a process drives has -//! an owner: `pcidev` is told, that claim refuses every later call, and the +//! **What differs is who the fault is handed to.** An enumerated function +//! every driver of which is in this kernel has nobody, and its fault is a +//! defect of this kernel, so the terminal action is a halt — the last thing +//! that happens rather than the first. A function a process drives has an +//! owner: `pcidev` is told, that claim refuses every later call, and the //! machine goes on, because one process's bug taking the machine down is the -//! thing moving a driver out of the kernel was for. - +//! thing moving a driver out of the kernel was for. A record that names no +//! enumerated function is a device's input this kernel never took on: the +//! unit has already refused it, so it is counted and logged and the machine +//! goes on. Nothing here can clear such a requester's `BME`, so a backend +//! bounds what it reads per interrupt by itself, never by the device. use alloc::boxed::Box; use alloc::vec::Vec; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; @@ -171,8 +175,8 @@ pub struct Fault { /// `fault`, read off unit `unit`, whose count of reported faults is `count`: /// its function stopped, it is handed to whoever drives that function, and -/// its line written. `true` where that is nobody, which the drain ends on -/// with [`conclude`]. +/// its line written. `true` only for an enumerated function this kernel +/// drives, which the drain ends on with [`conclude`]. pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { let function = find(fault.who); // First, before anything that can be slow or say no: a function that @@ -185,23 +189,24 @@ pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { let key = fault.who.key(); let _ = FIRST.compare_exchange(u64::MAX, key, Ordering::AcqRel, Ordering::Relaxed); // Before the line, so `owner=` in it is what was actually told. - let owner = function.and_then(|f| match f.user_slot.load(Ordering::Acquire) { - NO_SLOT => None, - slot => Some(slot as usize), - }); - if let Some(slot) = owner { + let owner = match function.map(|f| f.user_slot.load(Ordering::Acquire)) { + None => Owner::Nobody, + Some(NO_SLOT) => Owner::Kernel, + Some(slot) => Owner::Slot(slot as usize), + }; + if let Owner::Slot(slot) = owner { crate::pcidev::note_fault(slot); } let count = count.fetch_add(1, Ordering::Relaxed) + 1; log!( // `owner=` first, because it is the only field that decides whether // this machine is still running: `tests/common/serial.rs` reads - // `iommu: DMA FAULT owner=kernel` as a death and the other form as - // a record. The reason's name is the last word: every gate takes it + // `iommu: DMA FAULT owner=kernel` as a death and the other forms as + // records. The reason's name is the last word: every gate takes it // from there. "iommu: DMA FAULT owner={} unit{unit} stream={} addr={:#018x} access={} reason={:#04x} \ domain={} bme={} unitfaults={count} streamfaults={seen_here} first={} {}", - Owner(owner), + owner, fault.who, fault.address, fault.access, @@ -211,11 +216,11 @@ pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { if FIRST.load(Ordering::Relaxed) == key { 'y' } else { 'n' }, fault.name, ); - owner.is_none() + matches!(owner, Owner::Kernel) } -/// The end of a drain that read `kernel_owned` faults nobody owns: any at all -/// halts the machine. +/// The end of a drain that read `kernel_owned` faults on functions this kernel +/// drives: any at all halts the machine. /// /// The halt is the whole response and there is no recovery missing from it: a /// faulting device reached an address this kernel never gave it, and nothing @@ -223,10 +228,11 @@ pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { /// — a report of one carries the fault record and `panic_reboot`'s arm line, /// and no `panicked at` line; `capture` puts the fault on the panel first. /// -/// **Only for a stream this kernel drives.** A function a process drives has +/// **Only for a function this kernel drives.** A function a process drives has /// an owner to refuse: its bus mastering is already gone by the time this is /// reached, its claim answers every later call `Io`, and the machine — whose -/// other drivers are untouched — goes on. +/// other drivers are untouched — goes on. A requester nobody enumerated has +/// no driver here to be wrong, and the unit already refused what it sent. pub fn conclude(kernel_owned: usize) { if kernel_owned > 0 { crate::drivers::panic_console::capture(); @@ -243,14 +249,23 @@ fn config_window(phys: u64) -> Mmio { } /// Who a fault was handed to, in the line. A word rather than a number, because -/// which of the two it is decides whether this machine is still running. -struct Owner(Option); +/// which of them it is decides whether this machine is still running. +#[derive(Clone, Copy)] +enum Owner { + /// An enumerated function only this kernel drives: the halt. + Kernel, + /// An enumerated function the process on this `pcidev` slot drives. + Slot(usize), + /// No enumerated function: a requester this kernel never took on. + Nobody, +} impl core::fmt::Display for Owner { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self.0 { - None => write!(f, "kernel"), - Some(slot) => write!(f, "slot{slot}"), + match self { + Self::Kernel => write!(f, "kernel"), + Self::Slot(slot) => write!(f, "slot{slot}"), + Self::Nobody => write!(f, "none"), } } } diff --git a/tests/checks/serial.rs b/tests/checks/serial.rs index cba7757dcbc..098bd6de213 100644 --- a/tests/checks/serial.rs +++ b/tests/checks/serial.rs @@ -72,6 +72,15 @@ pub fn self_check() -> Result<(), String> { ) .must_be_clean() }), + ("must_be_clean on a DMA fault naming no enumerated function", false, &|| { + Serial::named( + "test capture", + "[ 0.001 cpu0 kernel] NVMe: found\n[ 4.100 cpu0 kernel] iommu: DMA FAULT \ + owner=none unit0 stream=0x18 addr=0x0000000000000000 access=none reason=0x04 \ + domain=unknown bme=unknown-function unitfaults=1 streamfaults=0 first=y C_BAD_STE\n", + ) + .must_be_clean() + }), ("must_be_clean on an empty capture", false, &|| dead.must_be_clean()), ]; @@ -168,6 +177,14 @@ pub fn self_check() -> Result<(), String> { addr=0x1000 access=read reason=0x06 read-permission", None, ), + // And on a requester no enumerated function is: the unit refused it, + // and the machine goes on. + ( + "[ 4.100 cpu0 kernel] iommu: DMA FAULT owner=none unit0 stream=00:1f.7 \ + addr=0x1000 access=write reason=0x02 domain=unknown bme=unknown-function \ + context-entry-not-present", + None, + ), ("[ 0.001 cpu0 kernel] EARLY PANIC: nothing is up yet", Some(Died::Kernel)), ( "[ 2.000 cpu1 kernel] DOUBLE PANIC: the cpu was already in Fatal; first: invalid \ diff --git a/tests/common/serial.rs b/tests/common/serial.rs index cd587c8bce1..a726f90ce09 100644 --- a/tests/common/serial.rs +++ b/tests/common/serial.rs @@ -96,10 +96,11 @@ pub(crate) const DEATHS: &[(&str, Died, Died)] = &[ // `machine_check_handler`, the one exception a Ring 3 frame does not make // the process's fault. Also `-> !`. ("MACHINE CHECK", Died::Kernel, Died::Kernel), - // kernel/src/arch/x86_64/vtd/fault.rs — a fault on a stream this kernel drives - // has nobody to hand it to, so the handler halts. One a *process* drives - // says `owner=slot` and the machine goes on, which is why the needle is - // the owner rather than the fault. + // kernel/src/iommu/fault.rs — a fault on a function this kernel drives has + // nobody to hand it to, so the handler halts. One a *process* drives says + // `owner=slot`, and one naming no enumerated function `owner=none`, and + // the machine goes on, which is why the needle is the owner rather than + // the fault. ("iommu: DMA FAULT owner=kernel", Died::Kernel, Died::Kernel), // kernel/src/main.rs — a panic that landed on a CPU already inside a fault // or a report. The rest of the line is `panic::last_words`: which of the @@ -348,12 +349,15 @@ impl Serial { /// all — so the only capture allowed to hold one is the capture of the test /// that staged it, and every other boot in the estate reds. const NEVER_CLEAN: &[&str] = &[ - // kernel/src/arch/x86_64/vtd/fault.rs — a function a *process* drives reached an + // kernel/src/iommu/fault.rs — a function a *process* drives reached an // address its own domain does not map. The machine goes on and the claim // refuses every later call, so this is not a death; it is a driver whose // descriptors are wrong, and a netstack that did it on every boot would // otherwise pass everywhere. "iommu: DMA FAULT owner=slot", + // The same file — a requester no enumerated function is: refused by the + // unit, and on a boot that staged none, a device nothing here accounts for. + "iommu: DMA FAULT owner=none", // kernel/src/log/console.rs — the stop waited out its whole budget for // `klogd` to let the console's wire go. super::power::WIRE_LATE, diff --git a/tests/toyos.rs b/tests/toyos.rs index b474a80aa45..cb923dd75a7 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2041,13 +2041,19 @@ fn virt_no_seed_refused(profile: qemu::Profile, test_config: &Path) -> Result<() /// `iommu-testdev` can and cannot write through it /// (`kernel/src/arch/aarch64/smmu/selftest.rs`): `GBPA` read back aborting, /// a `CMD_SYNC` consumed; one device's write refused on the entry its stream -/// starts with and nothing recorded, landing where its own domain maps it, -/// and refused once the domain takes that address back; the other's, under a -/// StreamID no function is routed from inside the stream table, refused. -/// Those two are recorded, the unit's first and second events, taken on the -/// wired SPI, named, and the machine halted for functions no process drives. +/// starts with and nothing recorded, and landing where its own domain maps +/// it; the other's two, under a StreamID no function is routed from inside +/// the stream table, refused, recorded as no function's, and the machine +/// going on; and the first's again once its domain takes that address back, +/// refused, recorded as the kernel's, and the machine halted on it. Each +/// record taken on the wired SPI and named. Read to the halt path's own line +/// past the stop, so nothing the guest could still say is missed. fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { - const FAULT: &str = "iommu: DMA FAULT owner=kernel"; + const FAULT: &str = "iommu: DMA FAULT owner="; + const WENT_ON: &str = "smmu-selftest: the unrouted function's two events were read, and the machine goes on"; + // `panic_reboot::arm`'s line, which `halt_all_cpus` writes once every + // other CPU is stopped: one of its two heads. + let halted = |l: &str| l.contains("panic: rebooting") || l.contains("panic: holding this panel"); let options = BootOptions { profile, kernel_params: &["smmu-selftest"], @@ -2061,10 +2067,18 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { } } let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[], options); - let rest = qemu.drain_until(Duration::from_secs(30), |l| l.contains(FAULT) && l.contains(" unitfaults=2 ")); + let rest = qemu.drain_until(Duration::from_secs(30), halted); let serial = format!("{}\n{rest}", qemu.boot_log()); - let line = |want: &str| serial.lines().find(|l| l.contains(want)).map(str::to_string); + let lines: Vec<&str> = serial.lines().collect(); + let at = |want: &str| lines.iter().position(|l| l.contains(want)); + let line = |want: &str| at(want).map(|i| lines[i].to_string()); let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); + if let Some(failed) = line("smmu-selftest: FAIL") { + return Err(format!("{failed}\nserial:\n{serial}")); + } + let Some(stop) = lines.iter().position(|l| halted(l)) else { + return Err(format!("the machine never reached the halt path's line past the stop\nserial:\n{serial}")); + }; let Some(armed) = line("armed, CR0ACK") else { return Err(format!("the SMMUv3 was never armed\nserial:\n{serial}")); }; @@ -2098,8 +2112,9 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { for (said, verdict) in [ ("on the entry its stream starts with", "refused"), ("mapped to", "landed there"), + ("'s write 1 at ", "refused"), + ("'s write 2 at ", "refused"), ("which its domain no longer maps", "refused"), - ("under no route", "refused"), ] { let Some(found) = line(said) else { return Err(format!("the selftest never said {said:?}\nserial:\n{serial}")); @@ -2113,28 +2128,49 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { return Err(format!("the selftest never wrote where its domain took a mapping back\nserial:\n{serial}")); }; let aimed = again.split_once("smmu-selftest: ").and_then(|(_, said)| said.split_once("'s write at ")); - let Some((function, Some(at))) = aimed.map(|(function, rest)| (function, rest.split(' ').next().and_then(hex))) + let Some((function, Some(address))) = + aimed.map(|(function, rest)| (function, rest.split(' ').next().and_then(hex))) else { return Err(format!("{again:?} names no function and address")); }; - let faults: Vec<&str> = serial.lines().filter(|l| l.contains(FAULT)).collect(); - for fault in &faults { - eprintln!(" [virt] {fault}"); + let faults: Vec = (0..lines.len()).filter(|&i| lines[i].contains(FAULT)).collect(); + for &fault in &faults { + eprintln!(" [virt] {}", lines[fault]); } - // The unit's first and second events, so the write on the entry its - // stream starts with was not recorded. + // The unit's first three events, so the write on the entry its stream + // starts with was not recorded. let wants = [ - (format!("stream={function} addr={at:#018x} access=write"), " unitfaults=1 ", "F_TRANSLATION"), - (format!("stream={stream:#x} "), " unitfaults=2 ", "C_BAD_STE"), + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=1 ", "C_BAD_STE"), + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=2 ", "C_BAD_STE"), + ( + format!("owner=kernel unit0 stream={function} addr={address:#018x} access=write"), + " unitfaults=3 ", + "F_TRANSLATION", + ), ]; if faults.len() != wants.len() { return Err(format!("{} events reached the handler, not {}\nserial:\n{serial}", faults.len(), wants.len())); } - for (fault, (named, count, name)) in faults.iter().zip(&wants) { + for (&fault, (named, count, name)) in faults.iter().zip(&wants) { + let fault = lines[fault]; if !fault.contains(named.as_str()) || !fault.contains(count) || !fault.ends_with(name) { return Err(format!("the event does not name {named:?},{count}and {name}: {fault}")); } } + // The machine went on past the unrouted records and stopped at the + // kernel's: the selftest's line between them, and the stop after the last. + let went_on = at(WENT_ON).ok_or_else(|| format!("the selftest never said {WENT_ON:?}\nserial:\n{serial}"))?; + if !(faults[1] < went_on && went_on < faults[2] && faults[2] < stop) { + return Err(format!( + "the unrouted records, the selftest going on, the kernel's record and the stop are out of order: \ + lines {}, {went_on}, {} and {stop}\nserial:\n{serial}", + faults[1], faults[2] + )); + } + // A halt, which writes the record and the stop's line and no panic. + if let Some(death) = lines.iter().find(|l| serial::died(l).is_some() && !l.contains("owner=kernel")) { + return Err(format!("the machine died of something other than the record: {death}\nserial:\n{serial}")); + } Ok(()) } diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs index c29c8c971da..122892e9f68 100644 --- a/toyos-smmu/src/queue.rs +++ b/toyos-smmu/src/queue.rs @@ -74,6 +74,11 @@ const OVERFLOW: u32 = 1 << 31; pub struct Events(pub(crate) Indexes); impl Events { + /// The records a full queue holds. + pub const fn entries(self) -> u32 { + 1 << self.0.log2size + } + /// The entry the next record is read from. pub const fn slot(self, cons: u32) -> usize { self.0.slot(cons) diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs index 2a29c88f115..75677363add 100644 --- a/toyos-smmu/tests/encodings.rs +++ b/toyos-smmu/tests/encodings.rs @@ -534,34 +534,6 @@ fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { } } -#[test] -fn each_event_is_named_by_its_own_mnemonic() { - let mnemonics = [ - (0x01, "F_UUT"), - (0x02, "C_BAD_STREAMID"), - (0x03, "F_STE_FETCH"), - (0x04, "C_BAD_STE"), - (0x05, "F_BAD_ATS_TREQ"), - (0x06, "F_STREAM_DISABLED"), - (0x07, "F_TRANSL_FORBIDDEN"), - (0x08, "C_BAD_SUBSTREAMID"), - (0x09, "F_CD_FETCH"), - (0x0A, "C_BAD_CD"), - (0x0B, "F_WALK_EABT"), - (0x10, "F_TRANSLATION"), - (0x11, "F_ADDR_SIZE"), - (0x12, "F_ACCESS"), - (0x13, "F_PERMISSION"), - (0x20, "F_TLB_CONFLICT"), - (0x21, "F_CFG_CONFLICT"), - (0x24, "E_PAGE_REQUEST"), - (0x25, "F_VMS_FETCH"), - ]; - for (number, mnemonic) in mnemonics { - assert_eq!(event(record(number, 7, 0x1000, false)).code.name(), mnemonic, "event {number:#x}"); - } -} - // --- the stage 1 tables (DDI 0487 M.d §D8.3.1) ----------------------------- /// A device address whose four table indexes are 0x1a5, 0x0f0, 0x00f and 0x155. From 3fc00867c70fd4e1ec9586b819ba85e48e9ecc38 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 12:02:40 +0200 Subject: [PATCH 4/4] A stray's DMA faults are written at powers of two of its own count; the re-pend cites IHI 0069D A requester no enumerated function is cannot be stopped from here: its BME is nobody's to clear, so a device writing without end under an unrouted stream wrote one owner=none line per record, up to a queue's worth per interrupt, and could evict every other record from the shard ring before logkeeper read it. Each stray is now counted apart (the first eight by key, every later one in a shared ninth count), the count goes in streamfaults=, and only a record whose count is a power of two is written: at most 32 lines per count for the boot, and each written line says how many were read since the last. unitfaults= still counts every record, so a later line shows the gap. The selftest's unrouted function now writes three times: records 1 and 2 are written, 3 is counted and not, and the kernel-owned record reads unitfaults=4, which is how virt_smmu sees a record read and not written. The comment at pend_iommu_events cites the GICv3 sections the re-pend rests on: GICD_ISPENDR makes an active SPI active and pending (8.9.16), which is never signalled (4.1.2), and deactivation leaves it pending (4.1.1, with EOImode 0 as init writes ICC_CTLR_EL1). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- kernel/src/arch/aarch64/irqchip.rs | 4 ++- kernel/src/arch/aarch64/smmu/selftest.rs | 16 ++++----- kernel/src/iommu/fault.rs | 43 +++++++++++++++++++++--- tests/checks/serial.rs | 2 +- tests/toyos.rs | 20 ++++++----- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index fbff0cf92c5..e58f914e583 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -327,7 +327,9 @@ pub(super) fn route_iommu_events(intid: u32) -> Result<(), u32> { } /// Pend the IOMMU's event SPI again, for records its handler left behind: the -/// unit raises none for them. Taken once the handler's `end` deactivates it. +/// unit raises none for them. Taken once the handler's `end` deactivates it: +/// the write makes an active SPI active and pending, which is never signalled, +/// and deactivation leaves it pending (IHI 0069D §8.9.16, §4.1.2, §4.1.1). pub(super) fn pend_iommu_events() { let intid = IOMMU_EVENTS.load(Relaxed); assert!(intid != 0, "GIC: the IOMMU's event SPI is pended before it is routed"); diff --git a/kernel/src/arch/aarch64/smmu/selftest.rs b/kernel/src/arch/aarch64/smmu/selftest.rs index 72f7d47a0bd..8d34b7774b0 100644 --- a/kernel/src/arch/aarch64/smmu/selftest.rs +++ b/kernel/src/arch/aarch64/smmu/selftest.rs @@ -7,15 +7,15 @@ //! 2. on a domain of its own, it writes at an address the domain maps, and //! lands at the memory mapped there and nowhere else; //! 3. the other, which [`super::init`] gives no route, as it gives none to a -//! function it never enumerated, writes twice under a StreamID inside the -//! stream table, and is refused and recorded twice, and the machine goes -//! on; +//! function it never enumerated, writes three times under a StreamID +//! inside the stream table, and is refused and counted three times, the +//! third record unwritten, and the machine goes on; //! 4. once the domain takes its address back, the first writes there again, //! and is refused and recorded, and the machine halts: the function is //! one this kernel drives. //! //! The writes of 3, and the write of 4, are each made with this CPU's -//! interrupts masked, so one drain reads both of 3's and 4's verdict is on +//! interrupts masked, so one drain reads all of 3's and 4's verdict is on //! the console before its event is taken, on this CPU, which the event SPI is //! routed to. Each step says what it saw on its own line; the harness judges //! them, and the records by the handler's, whose count says the first step @@ -123,7 +123,7 @@ pub(super) fn run(devices: &[PciDevice]) { let before = super::fault::recorded(); let masked = crate::arch::IrqGuard::close(); - let stray_wrote = [write(stray_regs, phys, phys), write(stray_regs, phys, phys)]; + let stray_wrote = [(); 3].map(|()| write(stray_regs, phys, phys)); for (i, answer) in stray_wrote.into_iter().enumerate() { log!( "smmu-selftest: {unrouted}'s write {} at {phys:#x}, under no route, answered {answer:#x}: {}", @@ -132,8 +132,8 @@ pub(super) fn run(devices: &[PciDevice]) { ); } drop(masked); - events_reach_this_cpu(before + 2, "the unrouted function's two refused writes"); - log!("smmu-selftest: the unrouted function's two events were read, and the machine goes on"); + events_reach_this_cpu(before + 3, "the unrouted function's three refused writes"); + log!("smmu-selftest: the unrouted function's three events were read, and the machine goes on"); let masked = crate::arch::IrqGuard::close(); let taken_back = write(regs, at + 0x40, phys + 0x40); @@ -143,7 +143,7 @@ pub(super) fn run(devices: &[PciDevice]) { verdict(taken_back, REFUSED) ); drop(masked); - events_reach_this_cpu(before + 3, "the kernel-driven function's refused write"); + events_reach_this_cpu(before + 4, "the kernel-driven function's refused write"); panic!("smmu-selftest: FAIL: the handler read the event of a function this kernel drives and the machine went on"); } diff --git a/kernel/src/iommu/fault.rs b/kernel/src/iommu/fault.rs index 8c6eb26e686..f5504386374 100644 --- a/kernel/src/iommu/fault.rs +++ b/kernel/src/iommu/fault.rs @@ -17,9 +17,11 @@ //! machine goes on, because one process's bug taking the machine down is the //! thing moving a driver out of the kernel was for. A record that names no //! enumerated function is a device's input this kernel never took on: the -//! unit has already refused it, so it is counted and logged and the machine -//! goes on. Nothing here can clear such a requester's `BME`, so a backend -//! bounds what it reads per interrupt by itself, never by the device. +//! unit has already refused it, so it is counted and the machine goes on. +//! Nothing here can clear such a requester's `BME`, so a backend bounds what +//! it reads per interrupt by itself, never by the device, and only a stray's +//! records whose count is a power of two are written, so the device cannot +//! fill the log either. use alloc::boxed::Box; use alloc::vec::Vec; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; @@ -62,6 +64,29 @@ fn functions() -> &'static [Function] { /// is decided by what the first one already broke. static FIRST: AtomicU64 = AtomicU64::new(u64::MAX); +/// How many strays — requesters no enumerated function is — are counted +/// apart; every one past them shares the last count. +const STRAYS: usize = 8; + +/// [`Who::key`] plus one of each stray counted apart, in the order each first +/// faulted; zero is free. +static STRAY_KEYS: [AtomicU64; STRAYS] = [const { AtomicU64::new(0) }; STRAYS]; + +/// The records read of each stray in [`STRAY_KEYS`], and of every later one. +static STRAY_FAULTS: [AtomicU32; STRAYS + 1] = [const { AtomicU32::new(0) }; STRAYS + 1]; + +/// This record's number among those of the stray `key`, from 1. +fn stray_fault(key: u64) -> u32 { + let at = STRAY_KEYS + .iter() + .position(|slot| match slot.compare_exchange(0, key + 1, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => true, + Err(held) => held == key + 1, + }) + .unwrap_or(STRAYS); + STRAY_FAULTS[at].fetch_add(1, Ordering::Relaxed) + 1 +} + /// Every function this machine enumerated, before any unit is armed: the /// handler reaches a faulting function's config space through this, with no lock. pub fn describe(devices: &[PciDevice]) { @@ -175,7 +200,7 @@ pub struct Fault { /// `fault`, read off unit `unit`, whose count of reported faults is `count`: /// its function stopped, it is handed to whoever drives that function, and -/// its line written. `true` only for an enumerated function this kernel +/// its line written, a stray's only at a power of two. `true` only for an enumerated function this kernel /// drives, which the drain ends on with [`conclude`]. pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { let function = find(fault.who); @@ -185,8 +210,11 @@ pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { if let Some(function) = function { pci::stop_bus_mastering(config_window(function.config)); } - let seen_here = function.map_or(0, |f| f.faults.fetch_add(1, Ordering::Relaxed) + 1); let key = fault.who.key(); + let seen_here = match function { + Some(function) => function.faults.fetch_add(1, Ordering::Relaxed) + 1, + None => stray_fault(key), + }; let _ = FIRST.compare_exchange(u64::MAX, key, Ordering::AcqRel, Ordering::Relaxed); // Before the line, so `owner=` in it is what was actually told. let owner = match function.map(|f| f.user_slot.load(Ordering::Acquire)) { @@ -198,6 +226,11 @@ pub fn report(unit: usize, count: &AtomicU32, fault: Fault) -> bool { crate::pcidev::note_fault(slot); } let count = count.fetch_add(1, Ordering::Relaxed) + 1; + // A stray's line is bounded by its count, never by the device: the next + // line it writes says how many were read in between. + if function.is_none() && !seen_here.is_power_of_two() { + return false; + } log!( // `owner=` first, because it is the only field that decides whether // this machine is still running: `tests/common/serial.rs` reads diff --git a/tests/checks/serial.rs b/tests/checks/serial.rs index 098bd6de213..d5d9091cacd 100644 --- a/tests/checks/serial.rs +++ b/tests/checks/serial.rs @@ -77,7 +77,7 @@ pub fn self_check() -> Result<(), String> { "test capture", "[ 0.001 cpu0 kernel] NVMe: found\n[ 4.100 cpu0 kernel] iommu: DMA FAULT \ owner=none unit0 stream=0x18 addr=0x0000000000000000 access=none reason=0x04 \ - domain=unknown bme=unknown-function unitfaults=1 streamfaults=0 first=y C_BAD_STE\n", + domain=unknown bme=unknown-function unitfaults=1 streamfaults=1 first=y C_BAD_STE\n", ) .must_be_clean() }), diff --git a/tests/toyos.rs b/tests/toyos.rs index cb923dd75a7..159a5ed1bcd 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2042,15 +2042,15 @@ fn virt_no_seed_refused(profile: qemu::Profile, test_config: &Path) -> Result<() /// (`kernel/src/arch/aarch64/smmu/selftest.rs`): `GBPA` read back aborting, /// a `CMD_SYNC` consumed; one device's write refused on the entry its stream /// starts with and nothing recorded, and landing where its own domain maps -/// it; the other's two, under a StreamID no function is routed from inside -/// the stream table, refused, recorded as no function's, and the machine -/// going on; and the first's again once its domain takes that address back, +/// it; the other's three, under a StreamID no function is routed from inside +/// the stream table, refused, the first two recorded as no function's and the +/// third counted and not written, and the machine going on; and the first's again once its domain takes that address back, /// refused, recorded as the kernel's, and the machine halted on it. Each /// record taken on the wired SPI and named. Read to the halt path's own line /// past the stop, so nothing the guest could still say is missed. fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { const FAULT: &str = "iommu: DMA FAULT owner="; - const WENT_ON: &str = "smmu-selftest: the unrouted function's two events were read, and the machine goes on"; + const WENT_ON: &str = "smmu-selftest: the unrouted function's three events were read, and the machine goes on"; // `panic_reboot::arm`'s line, which `halt_all_cpus` writes once every // other CPU is stopped: one of its two heads. let halted = |l: &str| l.contains("panic: rebooting") || l.contains("panic: holding this panel"); @@ -2114,6 +2114,7 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { ("mapped to", "landed there"), ("'s write 1 at ", "refused"), ("'s write 2 at ", "refused"), + ("'s write 3 at ", "refused"), ("which its domain no longer maps", "refused"), ] { let Some(found) = line(said) else { @@ -2137,14 +2138,15 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { for &fault in &faults { eprintln!(" [virt] {}", lines[fault]); } - // The unit's first three events, so the write on the entry its stream - // starts with was not recorded. + // The unit's first four events, so the write on the entry its stream + // starts with was not recorded; the stray's third is counted and, not + // being a power of two, not written. let wants = [ - (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=1 ", "C_BAD_STE"), - (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=2 ", "C_BAD_STE"), + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=1 streamfaults=1 ", "C_BAD_STE"), + (format!("owner=none unit0 stream={stream:#x} "), " unitfaults=2 streamfaults=2 ", "C_BAD_STE"), ( format!("owner=kernel unit0 stream={function} addr={address:#018x} access=write"), - " unitfaults=3 ", + " unitfaults=4 ", "F_TRANSLATION", ), ];