From 2c744d434cbb7e4b238c638bac9b5a0751e403a6 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 15:26:24 +0200 Subject: [PATCH 1/3] A claimed function's message is its claim slot's LPI through the GICv3 ITS, and no generic signature carries an x86 vector AArch64 gains the ITS (kernel/src/arch/aarch64/irqchip/its.rs), brought up from the MADT on the boot CPU's redistributor with toyos-its's encodings: the LPI configuration and pending tables, a flat device table of 2^16 DeviceIDs, the collection table, one command queue, and collection 0 on the boot CPU. A claim maps its function's DeviceID, which the IORT gives behind the SMMUv3, to a translation table of its own whose one event is its slot's LPI (MAPD, MAPTI, SYNC); a release discards and unmaps it. A DeviceID another claim holds, one past the table, or a function the IORT gives no DeviceID behind the unit is refused by name. A driver in this kernel is given no LPI. QEMU's SMMUv3 translates a function's MSI as any write it makes, measured: without a mapping, edu's first message is a DMA FAULT at 0x8090040, F_TRANSLATION, and the claim answers Io. So every domain maps the doorbell's 4 KiB page at its own address, as toyos-smmu's Leaf::Doorbell plans it. irq() holds the preempt count across every device arm, as x86-64's device_irq_entry does: the claim's arm posts a watch, whose release at depth zero would run a scheduler pass inside the handler before its end of interrupt. pcidev::isr asserts the count is raised. The seam: iommu::claim_msi takes no vector and may refuse; the kernel drivers' arming names an arch::DriverIrq, and the architecture makes its message (x86-64: the compatibility format with no unit, a remapping entry with one). VECTORS moves into idt/user_dev.rs, MSG_DEST into apic.rs, the pin's remapping into ioapic.rs, and each architecture states its own refusals. arch::msi_message, iommu::Delivery, PinRedirect, remap_pin and the SMMU's interrupt stub are deleted. virt_claim_lpi boots virt under HVF with its=on,kernel-irqchip=off (HVF's own GIC refuses an ITS), the SMMUv3 and QEMU's edu, on one CPU, and the suite's claim_lpi job reads 16 messages as 16 interrupts of its claim. Closes issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- Cargo.lock | 1 + ...-one-refusal-fewer-than-its-writer-made.md | 30 ++ ...-routing-take-an-x86-vector-and-apic-id.md | 22 - issues/toyos-runs-on-arm64.md | 28 +- kernel/Cargo.toml | 1 + kernel/src/arch/aarch64/irqchip.rs | 52 +- kernel/src/arch/aarch64/irqchip/its.rs | 446 ++++++++++++++++++ kernel/src/arch/aarch64/mod.rs | 13 +- kernel/src/arch/aarch64/smmu/domain.rs | 37 +- kernel/src/arch/aarch64/smmu/mod.rs | 36 -- kernel/src/arch/aarch64/trap.rs | 69 ++- kernel/src/arch/x86_64/apic.rs | 12 +- kernel/src/arch/x86_64/idt/mod.rs | 26 +- kernel/src/arch/x86_64/idt/user_dev.rs | 16 + kernel/src/arch/x86_64/ioapic.rs | 28 +- kernel/src/arch/x86_64/mod.rs | 5 +- kernel/src/arch/x86_64/pio.rs | 2 +- kernel/src/arch/x86_64/vtd/interrupt.rs | 52 +- kernel/src/drivers/pci.rs | 52 +- kernel/src/drivers/virtio_sound.rs | 2 +- kernel/src/drivers/xhci/mod.rs | 2 - kernel/src/drivers/xhci/wait/boot.rs | 5 +- kernel/src/iommu/mod.rs | 113 ++--- kernel/src/pcidev/mod.rs | 52 +- src/build.rs | 1 + tests/common/qemu.rs | 14 +- tests/toyos-rust-tests/src/bin/claim_lpi.rs | 60 +++ tests/toyos.rs | 93 ++++ tests/virtclaimcase/system.toml | 15 + 29 files changed, 974 insertions(+), 311 deletions(-) create mode 100644 issues/a-lane-reader-counted-one-refusal-fewer-than-its-writer-made.md delete mode 100644 issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md create mode 100644 kernel/src/arch/aarch64/irqchip/its.rs create mode 100644 tests/toyos-rust-tests/src/bin/claim_lpi.rs create mode 100644 tests/virtclaimcase/system.toml diff --git a/Cargo.lock b/Cargo.lock index 4df80a99005..46a4fa90694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3013,6 +3013,7 @@ dependencies = [ "toyos-gicv3", "toyos-gpt", "toyos-hda", + "toyos-its", "toyos-pci", "toyos-phys", "toyos-ps2", diff --git a/issues/a-lane-reader-counted-one-refusal-fewer-than-its-writer-made.md b/issues/a-lane-reader-counted-one-refusal-fewer-than-its-writer-made.md new file mode 100644 index 00000000000..622688162a0 --- /dev/null +++ b/issues/a-lane-reader-counted-one-refusal-fewer-than-its-writer-made.md @@ -0,0 +1,30 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# A lane's reader counted one refusal fewer than its writer made + +`toyos/src/log/proof.rs`'s +`a_lane_gives_its_reader_every_record_in_order_or_counts_it_refused` went red +once in `cargo run -- --ci host`'s `the toyos SDK` step, on the branch +that brings up the GICv3 ITS, which touches nothing under `toyos/`: + +``` +panicked at toyos/src/log/proof.rs:161:5: +assertion `left == right` failed + left: 49999 + right: 50000 +``` + +The reader's `Reader::refused` total was one short of the refusals the +writer thread counted from `push_lane`. The host's load average was 44.31 +on 14 cores when it was read; the same step was green an hour earlier on +a tree whose `toyos/` was the same. Either the lane loses a refusal between +the writer's count and the reader's last read after `writer.is_finished()`, +or the test reads the count before the last refusal is published: which one +is not determined. + +**Exit**: the cause is named, and the test reds on it without load, or the +lane's refusal count is fixed and the test holds under the load above. diff --git a/issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md b/issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md deleted file mode 100644 index 8c04720f4d6..00000000000 --- a/issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-26 ---- - -# MSI and pin routing take an x86 vector and APIC id in generic code - -`kernel/src/drivers/pci.rs`'s `enable_msix` and `enable_msi` take -`vector: u8`, and `kernel/src/iommu/mod.rs`'s `remap_pin` takes -`apic_id: u8` beside it. Both are x86's interrupt addressing: an 8-bit IDT -vector delivered to an xAPIC id. A GICv3 message names an LPI through the ITS -(a 32-bit event id, translated to an INTID above 8191) and a redistributor, -and neither fits a `u8`. - -Owned by stage 6 of `issues/toyos-runs-on-arm64.md`, which brings up -the GICv3 ITS for the claimed functions its SMMUv3 translates. - -**Exit condition**: a PCI function's interrupt is programmed from an -arch-provided message (address and data, as `arch::msi` already provides the -doorbell), the routing names its target by the arch's own type, and no -generic signature carries `vector: u8` or `apic_id`. diff --git a/issues/toyos-runs-on-arm64.md b/issues/toyos-runs-on-arm64.md index 405332a519e..14d90d92463 100644 --- a/issues/toyos-runs-on-arm64.md +++ b/issues/toyos-runs-on-arm64.md @@ -230,7 +230,6 @@ before any aarch64 file exists, with x86 as its only user: Each is its own issue, owned by the stage that removes it: -- `issues/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md` (stage 6) - `issues/the-crash-evidence-records-x86-fault-registers.md` (stage 5) - `issues/the-aarch64-kernel-builds-with-dead-code-allowed.md` (stage 7) @@ -327,9 +326,9 @@ Each stage names its exit; "measured" means a number from a run. function is its only consumer the small-kernel track leaves, and it needs that stage's SMMUv3 first. Stubbed on AArch64, each owned by the small-kernel track, which moves the driver out of the kernel: - - `arch::msi_message` refuses, so the kernel's xHCI (`virt`'s boot stick), - HDA, virtio-sound, virtio-console and virtio-gpu drivers each - refuse their function by name. + - a driver in this kernel is given no message (`irqchip::its::msi`), so + the kernel's xHCI (`virt`'s boot stick), HDA, virtio-sound, + virtio-console and virtio-gpu drivers each refuse their function by name. - `drivers::gop` refuses a scanout that is not whole 2 MiB pages of its own, which a `ramfb` scanout carved out of RAM need not be. @@ -368,16 +367,17 @@ Each stage names its exit; "measured" means a number from a run. before its body) get a test here that reds with `put` written back as one `self.buf.write(off, trb)`; x86's TSO hides all three from every guest test until then. - **The claim's handler is the first arm of `irq()` - (`kernel/src/arch/aarch64/trap.rs:135`) that posts a watch or lets go of a - `Lock`**, and either runs `preempt::enable`, whose pass at depth zero - (`kernel/src/preempt.rs:67`) reads nothing of `DAIF`; `do_preempt`'s - `assert_baseline(BASELINE_IRQ_EXIT)` (`kernel/src/scheduler.rs:358`) passes - at depth zero, so that pass would run inside the handler, before - `irqchip::end`. Owed before that arm lands: `irq()` holds the preempt count - across every device arm, as x86-64's `device_irq_entry` does, or - `preempt::enable` refuses a pass with interrupts masked, which `IrqOff`'s - SAFETY (`kernel/src/sched/driver.rs:49`) already assumes. + A claimed function's message is its claim slot's LPI through the GICv3 + ITS (`kernel/src/arch/aarch64/irqchip/its.rs`), on QEMU's emulated GIC: + HVF's own refuses an ITS, so a `virt` with one is + `its=on,kernel-irqchip=off`. QEMU's SMMUv3 translates the message as a + write of the function's, so every domain maps the doorbell's page + (`virt_claim_lpi`, red without it on a `DMA FAULT` at `GITS_TRANSLATER`). + `irq()` holds the preempt count across every device arm, as x86-64's + `device_irq_entry` does, and `pcidev::isr` asserts it. + Not yet tested: that a release's `DISCARD` and `MAPD` with `V` clear + leave the function's DeviceID translating nothing, which no function + QEMU offers can show, since a released function's MSI is disabled. 7. **Userland boots.** `init`, `logd`, the compositor, netd, soundd and sshd, built for `aarch64-unknown-toyos`. Every program of `system.toml` builds diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 38083419d45..d3a3b8adfdd 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -489,6 +489,7 @@ toyos-elf = { path = "../toyos-elf" } toyos-gicv3 = { path = "gicv3" } toyos-gpt = { path = "../toyos-gpt" } toyos-hda = { path = "../toyos-hda" } +toyos-its = { path = "../toyos-its" } toyos-pci = { path = "pci" } toyos-phys = { path = "../toyos-phys" } toyos-smmu = { path = "../toyos-smmu" } diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 08efbb0def0..9fcf1627f81 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -1,22 +1,21 @@ //! The interrupt controller and the timer: a GICv3 — the distributor the -//! MADT names, this CPU's redistributor, and the system-register CPU -//! interface (GIC architecture specification IHI 0069H) — and the generic -//! timer's EL1 virtual timer (Arm ARM K.a, chapter D12), whose PPI the GTDT -//! names. +//! MADT names, this CPU's redistributor, the system-register CPU interface +//! and the ITS ([`its`]) (GIC architecture specification IHI 0069H) — and the +//! generic timer's EL1 virtual timer (Arm ARM K.a, chapter D12), whose PPI the +//! GTDT names. //! -//! **What this kernel takes is SGIs, the timer's PPI and the IOMMU's event -//! SPI ([`route_iommu_events`]), and nothing else.** No other SPI is routed -//! and no LPI exists: a device's interrupt is a message the -//! GICv3 ITS translates, and every device that would take one is either a -//! driver the small-kernel track moves out of the kernel or a claimed function, -//! which the SMMUv3 of the port's stage 6 must translate first -//! (`super::msi_message`). +//! **What this kernel takes is SGIs, the timer's PPI, the IOMMU's event SPI +//! ([`route_iommu_events`]) and one LPI per claim slot ([`its`]), and nothing +//! else.** No other SPI is routed: a device's interrupt is a message the ITS +//! translates, and only a claimed function's is. //! //! **The virtual timer, not the physical**: it is the one EL1 owns outright — //! under a hypervisor the physical one traps — and with `CNTVOFF_EL2` written //! zero by the entry from EL2 the two count alike. It is level-triggered, so a //! handler that neither re-arms nor stops it takes it again at once. +pub mod its; + use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed}; use alloc::vec::Vec; @@ -31,9 +30,7 @@ use crate::mm::policy::MmioPolicy; use crate::mm::{DirectMap, Mmio}; use crate::hw::MIN_ONE_SHOT; -/// Every INTID this kernel names, in one space: the SGIs it raises, then the -/// identities the generic drivers program for a message-signalled interrupt, -/// which `super::msi_message` refuses on this machine. +/// The SGIs this kernel raises. #[repr(u8)] pub(super) enum Intid { /// Asks a CPU for a scheduler pass, or for its counters: x86-64's kick. @@ -44,8 +41,6 @@ pub(super) enum Intid { Off, /// What `irq-storm` floods this CPU with. Storm, - Hda, - VirtioSound, } pub(super) const SGI_KICK: u32 = Intid::Kick as u32; @@ -165,17 +160,20 @@ impl Gic { pub fn init(rsdp_addr: u64) -> Gic { let madt = toyos_acpi::find_table(direct_phys(), rsdp_addr, b"APIC", toyos_acpi::MADT_ENTRIES) .unwrap_or_else(|e| panic!("GIC: the MADT is unusable: {e:?}")); - let (mut gicd, mut ranges, mut cpus) = (None, Vec::new(), Vec::new()); + let (mut gicd, mut ranges, mut cpus, mut translators) = (None, Vec::new(), Vec::new(), Vec::new()); for entry in toyos_acpi::madt_entries(&madt) { + let entry = entry.unwrap_or_else(|halt| { + panic!("GIC: a MADT entry at +{} declares {} bytes of a {}-byte list", halt.at, halt.declared, halt.list_len) + }); match entry { - Ok(MadtEntry::Gicd { base, .. }) => gicd = Some(base), - Ok(MadtEntry::Gicr { base, length }) => ranges.push(( + MadtEntry::Gicd { base, .. } => gicd = Some(base), + MadtEntry::Gicr { base, length } => ranges.push(( base, crate::mm::paging::map_mmio(base, u64::from(length), MmioPolicy::Uncacheable), )), - Ok(MadtEntry::Gicc(gicc)) if gicc.enabled => cpus.push(gicc), - Ok(_) => {} - Err(halt) => panic!("GIC: a MADT entry at +{} declares {} bytes of a {}-byte list", halt.at, halt.declared, halt.list_len), + MadtEntry::Gicc(gicc) if gicc.enabled => cpus.push(gicc), + MadtEntry::Its { id, base } => translators.push((id, base)), + _ => {} } } let gicd = gicd.expect("GIC: the MADT names no distributor"); @@ -205,10 +203,18 @@ pub fn init(rsdp_addr: u64) -> Gic { ); let gic = Gic { cpus, ranges }; - init_cpu(gic.redistributor(cpu::hardware_id())); + let frame = gic.redistributor(cpu::hardware_id()); + init_cpu(frame); + its::init(&translators, frame, rsdp_addr); gic } +/// `GICD_TYPER`, which says whether the distributor takes LPIs and how wide +/// an INTID is. +fn distributor_typer() -> u32 { + Mmio::new(DirectMap::from_phys(DISTRIBUTOR.load(Relaxed)), FRAME).read_u32(GICD_TYPER) +} + /// Bring this CPU's redistributor — the frame at `frame`, which /// [`Gic::redistributor`] mapped — its CPU interface and its timer up: every /// SGI and the timer's PPI enabled at [`PRIORITY`], the timer stopped. diff --git a/kernel/src/arch/aarch64/irqchip/its.rs b/kernel/src/arch/aarch64/irqchip/its.rs new file mode 100644 index 00000000000..e1047ca294f --- /dev/null +++ b/kernel/src/arch/aarch64/irqchip/its.rs @@ -0,0 +1,446 @@ +//! The GICv3 Interrupt Translation Service (IHI 0069H.b chapter 5), driven +//! with `toyos_its`'s encodings: what turns a claimed function's message into +//! that claim's own LPI, and into no other. +//! +//! **A message is translated by the DeviceID of the function that wrote it**, +//! which the bus says and the function cannot (§5.1, §5.2.1): a claim's +//! function is mapped to a translation table of its own (`MAPD`) whose one +//! event is its slot's LPI (`MAPTI`), so whatever a function writes to the +//! doorbell, it raises its own slot's LPI or nothing. A DeviceID no claim +//! holds is mapped to no table, and its writes are dropped (§5.3.10). +//! +//! **Every LPI is collection 0's, on the boot CPU's redistributor**, as every +//! claimed function's interrupt on x86-64 is cpu0's. +//! +//! **What the ITS and the redistributor read is handed over before it is +//! named:** each table is written whole through [`Mmio`], whose write orders +//! it before the register write or command that names it, and the +//! configuration of every claim slot's LPI is written, enabled, before the +//! redistributor's `EnableLPIs`, so no `INV` is owed (§5.1.1). +//! +//! An ITS this kernel cannot drive is left disabled, and every claim is +//! refused on its account ([`is_armed`]); a command the ITS refuses, or does +//! not consume within [`TIMEOUT_PER_SECOND`]'s bound, is a kernel defect, and +//! panics. + +use alloc::vec::Vec; + +use toyos_acpi::Route; +use toyos_its::command::Command; +use toyos_its::lpi::{self, Layout, Lpi}; +use toyos_its::{Collection, CommandQueue, EventBits, Its, Table, Target}; +use toyos_phys::Phys; + +use crate::drivers::acpi::direct_phys; +use crate::iommu::StreamId; +use crate::log; +use crate::mm::pmm::{self, PhysPage}; +use crate::mm::policy::MmioPolicy; +use crate::mm::{DirectMap, Mmio, PAGE_2M}; +use crate::pcidev::MAX_FUNCTIONS; +use crate::sync::Lock; + +/// A command is consumed within a second, and a register settles in one: +/// `settles` takes the bound as a rate. +const TIMEOUT_PER_SECOND: u64 = 1; + +/// The ITS's control frame; the translation frame is the next 64 KiB. +const FRAME: u64 = 0x1_0000; + +/// `GITS_CTLR.Quiescent` [31] (§12.19.4): the ITS has finished with every +/// table and may be reprogrammed. +const CTLR_QUIESCENT: u32 = 1 << 31; + +/// Offsets in a redistributor's `RD_base` frame (§12.10). +const GICR_CTLR: u64 = 0x0000; +const GICR_TYPER: u64 = 0x0008; + +/// INTID bits the LPI tables are laid out for: the least that names an LPI +/// (§5.1.1), 8192 of them, which no claim slot count comes near. +const ID_BITS: u8 = 14; + +/// The widest DeviceID a claim is given: a PCI requester ID's, which is what +/// the IORT maps each function from. The device table holds no more. +const DEVICE_BITS: u8 = 16; + +/// One event per claim, so the least table there is: two events. +const EVENT_BITS: u8 = 1; + +/// A claim's one event: what its message carries as data. +const EVENT: u32 = 0; + +/// Every LPI a claim slot raises runs at the one priority every SGI and PPI +/// does (`super::PRIORITY`). +const PRIORITY: u8 = super::PRIORITY; + +/// `GITS_BASER`'s `Type` [58:56] and `Entry_Size` [52:48], which the ITS +/// states and a write leaves as they are (§12.19.1). +const BASER_READ_ONLY: u64 = 0b111 << 56 | 0x1F << 48; + +/// The translation table entries one claim's function writes are read +/// through: `MAPD`'s `ITT_addr` is 256-byte aligned (§5.3.10). +const ITT_ALIGN: u64 = 256; + +static ITS: Lock> = Lock::new(None); + +/// The ITS this kernel drives, armed. +struct Live { + regs: Mmio, + /// The MADT's id for it, which the IORT names it by. + id: u32, + /// `GITS_TRANSLATER`'s physical address: every claim's message address. + doorbell: u32, + queue: Mmio, + commands: CommandQueue, + /// `GITS_CWRITER` as last written. + cwriter: u64, + /// The boot CPU's redistributor, as this ITS names it. + target: Target, + collection: Collection, + events: EventBits, + /// Each claim slot's translation table, and the DeviceID mapped to it + /// while the slot is claimed. + itts: [Phys<8>; MAX_FUNCTIONS], + held: [Option; MAX_FUNCTIONS], + /// Each slot's LPI. + lpis: [Lpi; MAX_FUNCTIONS], + /// Where the IORT is read from at each claim. + rsdp: u64, + /// The pages every table is in: never given back, since the ITS and the + /// redistributor may read any of them for the machine's life. + _pages: Vec, +} + +impl Live { + /// `commands`, then a `SYNC` to the boot CPU's redistributor, consumed: + /// every one has taken effect when this returns (§5.3.15). + fn issue(&mut self, commands: &[Command]) { + let (regs, queue) = (self.regs, self.commands); + for command in commands.iter().copied().chain([Command::Sync(self.target)]) { + let cwriter = self.cwriter; + super::settles(TIMEOUT_PER_SECOND, "ITS: room in its command queue", || { + !queue.is_full(cwriter, regs.read_u64(toyos_its::GITS_CREADR as u64)) + }); + let at = u64::from(queue.offset(cwriter)); + for (i, word) in command.words().iter().enumerate() { + self.queue.write_u64(at + 8 * i as u64, *word); + } + self.cwriter = queue.after(cwriter); + } + let cwriter = self.cwriter; + regs.write_u64(toyos_its::GITS_CWRITER as u64, cwriter); + super::settles(TIMEOUT_PER_SECOND, "ITS: its commands consumed", || { + let creadr = regs.read_u64(toyos_its::GITS_CREADR as u64); + assert!( + creadr & toyos_its::CREADR_STALLED == 0, + "ITS: stalled on the command at {:#x} of {commands:?}", + queue.offset(creadr) + ); + queue.is_empty(cwriter, creadr) + }); + } +} + +/// Bump-allocated zeroed memory, aligned as asked, out of 2 MiB pages. +struct Memory { + pages: Vec, + used: u64, +} + +impl Memory { + fn alloc(&mut self, bytes: u64, align: u64) -> u64 { + assert!(align.is_power_of_two() && bytes <= PAGE_2M, "ITS: {bytes:#x} bytes aligned to {align:#x}"); + let mut at = self.used.next_multiple_of(align); + if self.pages.is_empty() || at + bytes > PAGE_2M { + // Zeroed by the allocator. + self.pages.push(pmm::alloc_page().expect("ITS: no memory for its tables")); + at = 0; + } + self.used = at + bytes; + self.pages.last().expect("a page was just pushed").direct_map().phys() + at + } +} + +/// `bytes` of [`Memory`] at `phys`. +fn window(phys: u64, bytes: u64) -> Mmio { + // SAFETY: every caller names memory `Memory::alloc` handed out, which + // `Live` keeps for the machine's life and the direct map covers. + unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } +} + +/// What claim slot `slot`'s message raises, for the record. +pub(crate) fn slot_interrupt(slot: usize) -> impl core::fmt::Display { + let intid = lpi::FIRST + slot as u32; + alloc::format!("LPI {intid}") +} + +/// The claim slot LPI `intid` is raised for, for the interrupt's handler: +/// `None` for any INTID no slot owns. +pub(in crate::arch::aarch64) fn slot_of(intid: u32) -> Option { + let slot = usize::try_from(intid.checked_sub(lpi::FIRST)?).ok()?; + (slot < MAX_FUNCTIONS).then_some(slot) +} + +/// Bring the one ITS the MADT names up, on the boot CPU's redistributor at +/// `frame`, once that CPU's GIC is, from the `(id, base)` of every ITS the +/// MADT names: every refusal is logged and leaves no claim a message. +pub(super) fn init(named: &[(u32, u64)], frame: u64, rsdp: u64) { + let (id, base) = match *named { + [] => { + log!("ITS: the MADT names none, so no claimed function is given an interrupt"); + return; + } + [one] => one, + _ => { + log!("ITS: the MADT names {}, and this kernel drives one: no claimed function is given an interrupt", named.len()); + return; + } + }; + match bring_up(id, base, frame, rsdp) { + Ok(live) => { + log!( + "ITS: {id} at {base:#x} armed: doorbell {:#x}, collection 0 on the boot CPU's redistributor at {frame:#x}, \ + LPIs {:?} for the claim slots", + live.doorbell, + live.lpis.map(|lpi| lpi.intid()), + ); + *ITS.lock() = Some(live); + } + Err(why) => log!("ITS: {id} at {base:#x} is left disabled, so no claimed function is given an interrupt: {why}"), + } +} + +fn bring_up(id: u32, base: u64, frame: u64, rsdp: u64) -> Result { + use alloc::format; + let doorbell = base + FRAME + toyos_its::GITS_TRANSLATER; + let doorbell = u32::try_from(doorbell) + .map_err(|_| format!("GITS_TRANSLATER at {doorbell:#x} is past the 32 bits a message address this kernel writes holds"))?; + let regs = crate::mm::paging::map_mmio(base, FRAME, MmioPolicy::Uncacheable); + let typer = regs.read_u64(toyos_its::GITS_TYPER as u64); + let its: Its = toyos_its::probe(typer).map_err(|lacks| format!("it lacks {lacks:?} (GITS_TYPER {typer:#x})"))?; + let events = its.events(EVENT_BITS).ok_or_else(|| format!("it has no EventID (GITS_TYPER {typer:#x})"))?; + if its.device_bits < DEVICE_BITS { + return Err(format!("its DeviceIDs are {} bits, short of a requester ID's {DEVICE_BITS}", its.device_bits)); + } + + // Disabled and quiescent before any table is named (§12.19.4). + let ctlr = regs.read_u32(toyos_its::GITS_CTLR as u64); + if ctlr & toyos_its::CTLR_ENABLED != 0 { + regs.write_u32(toyos_its::GITS_CTLR as u64, ctlr & !toyos_its::CTLR_ENABLED); + } + super::settles(TIMEOUT_PER_SECOND, "ITS: GITS_CTLR.Quiescent", || { + regs.read_u32(toyos_its::GITS_CTLR as u64) & CTLR_QUIESCENT != 0 + }); + + // The redistributor first: an ITS whose LPIs no redistributor takes + // would translate into nothing. + let rd = Mmio::new(DirectMap::from_phys(frame), FRAME); + let rd_typer = rd.read_u64(GICR_TYPER); + if rd_typer & lpi::TYPER_PLPIS == 0 { + return Err(format!("the boot CPU's redistributor takes no physical LPI (GICR_TYPER {rd_typer:#x})")); + } + if rd.read_u32(GICR_CTLR) & lpi::CTLR_ENABLE_LPIS != 0 { + return Err("the boot CPU's redistributor was handed over with its LPIs enabled, whose tables nothing may move".into()); + } + let gicd_typer = super::distributor_typer(); + let layout = Layout::new(gicd_typer, ID_BITS) + .ok_or_else(|| format!("the distributor takes no LPI of {ID_BITS} bits (GICD_TYPER {gicd_typer:#x})"))?; + + let mut memory = Memory { pages: Vec::new(), used: 0 }; + let configuration = memory.alloc(layout.configuration_bytes(), 1 << 12); + let pending = memory.alloc(layout.pending_bytes(), 1 << 16); + let queue_at = memory.alloc(4096, 1 << 16); + let commands = CommandQueue::new(1).expect("one page is a queue"); + + // Each `GITS_BASER` that asks for a table this kernel needs is given + // one; the others back nothing (§12.19.1). + let mut collections_in_table = 0; + for n in 0..8 { + let at = toyos_its::gits_baser(n).expect("eight registers") as u64; + let backing = toyos_its::table(regs.read_u64(at)); + let entries = match backing.table { + Table::Unimplemented | Table::Other(_) => continue, + Table::Devices => 1u64 << DEVICE_BITS, + // One collection: the boot CPU's. + Table::Collections => 1, + }; + let pages = backing + .pages(entries) + .ok_or_else(|| format!("GITS_BASER{n} cannot hold {entries} entries in a flat table"))?; + let bytes = u64::from(pages) * backing.page.bytes(); + if bytes > PAGE_2M { + return Err(format!("GITS_BASER{n}'s table of {entries} entries is {bytes:#x} bytes, past a page")); + } + let table = Phys::<16>::new(memory.alloc(bytes, 1 << 16)).expect("a 64 KiB-aligned page below 2^48"); + let value = backing.baser(table, pages).expect("pages inside the field"); + regs.write_u64(at, value); + let read = regs.read_u64(at); + if read & !BASER_READ_ONLY != value { + return Err(format!("GITS_BASER{n} reads {read:#x} after {value:#x} was written")); + } + if backing.table == Table::Collections { + collections_in_table = backing.entries(pages); + } + } + let collection = its + .collections(collections_in_table) + .collection(0) + .ok_or_else(|| format!("it holds no collection, and no GITS_BASER backs a table of them (GITS_TYPER {typer:#x})"))?; + + let queue_phys = Phys::<16>::new(queue_at).expect("a 64 KiB-aligned page below 2^48"); + regs.write_u64(toyos_its::GITS_CBASER as u64, commands.cbaser(queue_phys)); + regs.write_u64(toyos_its::GITS_CWRITER as u64, 0); + + let propbaser = layout.propbaser(Phys::new(configuration).expect("a 4 KiB-aligned page below 2^48")); + rd.write_u64(lpi::GICR_PROPBASER as u64, propbaser); + let read = rd.read_u64(lpi::GICR_PROPBASER as u64); + let space = layout.space(read).ok_or_else(|| format!("GICR_PROPBASER reads {read:#x} after {propbaser:#x} was written"))?; + // Every slot's LPI enabled, before the redistributor reads the table. + let lpis: [Lpi; MAX_FUNCTIONS] = core::array::from_fn(|slot| { + space.lpi(lpi::FIRST + slot as u32).expect("a claim slot's LPI is among the 8192 the layout holds") + }); + let config = window(configuration, layout.configuration_bytes()); + for lpi in lpis { + config.write_u8(lpi.configuration_index() as u64, lpi::configuration(PRIORITY, true)); + } + rd.write_u64(lpi::GICR_PENDBASER as u64, lpi::pendbaser(Phys::new(pending).expect("a 64 KiB-aligned page below 2^48"))); + rd.write_u32(GICR_CTLR, rd.read_u32(GICR_CTLR) | lpi::CTLR_ENABLE_LPIS); + if rd.read_u32(GICR_CTLR) & lpi::CTLR_ENABLE_LPIS == 0 { + return Err("the boot CPU's redistributor reads EnableLPIs clear after it was set".into()); + } + + regs.write_u32(toyos_its::GITS_CTLR as u64, toyos_its::CTLR_ENABLED); + if regs.read_u32(toyos_its::GITS_CTLR as u64) & toyos_its::CTLR_ENABLED == 0 { + return Err("GITS_CTLR reads Enabled clear after it was set".into()); + } + + let target = its.target(lpi::processor_number(rd_typer), Phys::new(frame).expect("a redistributor frame is 64 KiB aligned")); + let itt_bytes = its.itt_bytes(events).next_multiple_of(ITT_ALIGN); + let itts = core::array::from_fn(|_| Phys::<8>::new(memory.alloc(itt_bytes, ITT_ALIGN)).expect("256-byte aligned below 2^48")); + let mut live = Live { + regs, + id, + doorbell, + queue: window(queue_at, 4096), + commands, + cwriter: 0, + target, + collection, + events, + itts, + held: [None; MAX_FUNCTIONS], + lpis, + rsdp, + _pages: memory.pages, + }; + live.issue(&[Command::MapCollection { collection, target }]); + Ok(live) +} + +/// The 4 KiB page `GITS_TRANSLATER` is in, where an ITS is armed: what every +/// claimed function's domain maps for its messages to reach it. +pub(in crate::arch::aarch64) fn doorbell_page() -> Option { + ITS.lock().as_ref().map(|live| u64::from(live.doorbell) & !(crate::mm::PAGE_SIZE - 1)) +} + +/// Whether an ITS is armed to give a claimed function its own LPI. +pub fn is_armed() -> bool { + ITS.lock().is_some() +} + +pub struct Msi { + pub address: u32, + pub data: u32, +} + +/// Why a source could not be given a message. +#[derive(Clone, Copy)] +pub enum Refused { + /// It is a driver in this kernel's. + KernelDriver, + /// The IORT gives it no DeviceID at this ITS behind the SMMUv3. + NoDeviceId, + /// Its DeviceID is past the device table's. + DeviceIdTooWide(u32), + /// Its DeviceID is another claim's: one table translates both, so each + /// could raise the other's interrupt. + DeviceIdHeld(u32), +} + +impl core::fmt::Display for Refused { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::KernelDriver => write!( + f, + "a driver in this kernel is given no LPI: the ITS maps only a claimed function's \ + DeviceID, and such a driver has no address space for its DMA" + ), + Self::NoDeviceId => write!( + f, + "the IORT gives it no DeviceID at this kernel's ITS behind the SMMUv3, so no \ + table of its own would translate its message" + ), + Self::DeviceIdTooWide(id) => write!(f, "its DeviceID {id:#x} is past the ITS's device table"), + Self::DeviceIdHeld(id) => write!( + f, + "the IORT gives it DeviceID {id:#x}, which another claimed function's messages \ + are already translated by" + ), + } + } +} + +/// A driver in this kernel is given no LPI: only a claim slot's function is +/// mapped, and a driver in this kernel on this machine has no address space +/// to do DMA in (`super::super::iommu_unit`). +pub fn msi(_source: StreamId, _irq: crate::arch::DriverIrq) -> Result { + Err(Refused::KernelDriver) +} + +/// The DeviceID the IORT gives `source` at the ITS behind the SMMUv3. +fn device_of(live: &Live, source: StreamId) -> Result { + let iort = toyos_acpi::iort(direct_phys(), live.rsdp).map_err(|_| Refused::NoDeviceId)?; + let segment = u32::from(crate::pcidev::segment()); + match iort.route(segment, source.requester()) { + Ok(Route::Translated { its: Some(device), .. }) if device.its == live.id => { + if device.device >> DEVICE_BITS == 0 { + Ok(device.device) + } else { + Err(Refused::DeviceIdTooWide(device.device)) + } + } + _ => Err(Refused::NoDeviceId), + } +} + +/// Map claim slot `slot`'s translation table to `source`'s DeviceID, its one +/// event to the slot's LPI, and answer the message that raises it. +pub fn claim(slot: usize, source: StreamId) -> Result { + let mut held = ITS.lock(); + let live = held.as_mut().unwrap_or_else(|| panic!("ITS: claim slot {slot} mapped with no ITS armed")); + assert!(live.held[slot].is_none(), "ITS: claim slot {slot} is still mapped"); + let device = device_of(live, source)?; + if live.held.contains(&Some(device)) { + return Err(Refused::DeviceIdHeld(device)); + } + let event = live.events.event(EVENT).expect("event 0 is in every table"); + let (table, lpi, collection, events) = (live.itts[slot], live.lpis[slot], live.collection, live.events); + live.issue(&[ + Command::MapDevice { device, table, events }, + Command::MapEvent { device, event, lpi, collection }, + ]); + live.held[slot] = Some(device); + log!("ITS: {source}, DeviceID {device:#x}, event {EVENT} is slot {slot}'s LPI {}", lpi.intid()); + Ok(Msi { address: live.doorbell, data: EVENT }) +} + +/// Claim slot `slot`'s DeviceID unmapped, and anything it left pending +/// dropped, before this returns. +pub fn release(slot: usize, source: StreamId) { + let mut held = ITS.lock(); + let live = held.as_mut().unwrap_or_else(|| panic!("ITS: claim slot {slot} released with no ITS armed")); + let device = live.held[slot].take().unwrap_or_else(|| panic!("ITS: claim slot {slot} released unmapped")); + let event = live.events.event(EVENT).expect("event 0 is in every table"); + live.issue(&[Command::Discard { device, event }, Command::UnmapDevice { device }]); + log!("ITS: {source}, DeviceID {device:#x}, unmapped from slot {slot}"); +} diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index a3c29dfc62d..6d9c87c2cce 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -39,21 +39,14 @@ pub mod tlb; pub mod trap; pub mod watchdog; +pub use irqchip::its as message_unit; +pub(crate) use irqchip::its::slot_interrupt; pub use smmu as iommu_unit; +pub use trap::DriverIrq; /// The machine every program image this kernel loads must be built for. pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::Aarch64; -/// A message-signalled interrupt's address and data for `vector` on CPU -/// `dest`, which this machine does not give: the doorbell is an ITS's -/// `GITS_TRANSLATER` and the data an event the ITS maps, and nothing here -/// drives an ITS. Every function that would take one is a driver the -/// small-kernel track moves out of the kernel, or a claimed function the -/// SMMUv3 of the port's stage 6 must translate first; each is refused by name. -pub fn msi_message(_dest: u32, _vector: u8) -> Result<(u32, u32), &'static str> { - Err("AArch64 delivers no message-signalled interrupt to this kernel: the GICv3 ITS is unported") -} - /// Interrupts masked on this CPU for as long as the guard lives, and then put /// back as they were — restored, not enabled — so a guard nests inside a region /// that is already masked. `DAIF`'s `I` and `F` are what it closes; `D` and `A` diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs index 4b61ff63cb7..f40e4c64132 100644 --- a/kernel/src/arch/aarch64/smmu/domain.rs +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -1,8 +1,14 @@ //! A device's address space: one context descriptor, tagged with an ASID of -//! its own, over stage 1 tables that map 2 MiB blocks and nothing else +//! its own, over stage 1 tables that map 2 MiB blocks //! (`toyos_smmu::table`), and the device addresses it hands out, over the //! unit's 48-bit input (`crate::iommu::window`). //! +//! **And the ITS's doorbell, the one 4 KiB page at its own address**: the unit +//! translates a function's message as it does any write it makes, so a domain +//! without that page is one whose function's messages it refuses. The page +//! holds `GITS_TRANSLATER` and nothing else a device may write, and below +//! memory it is no address a domain hands out. +//! //! A new leaf needs no invalidation — the unit caches no translation that //! faulted — and a removed one is invalidated by the domain's ASID, behind a //! `CMD_SYNC`, before [`unmap`] returns. @@ -19,6 +25,11 @@ use crate::iommu::{DomainId, IommuError, Iova, StreamId}; use crate::log; use crate::mm::{PAGE_2M, PAGE_SIZE}; +/// One 2 MiB block of memory at `phys`, which a device reads and writes. +fn block(phys: u64) -> Leaf { + Leaf::Memory(Phys::<21>::new(phys).expect("SMMU: a 2 MiB-aligned page of memory"), Access::ReadWrite) +} + /// Domain ids start at 1: `DomainId` is never 0. const FIRST: u16 = 1; @@ -34,10 +45,9 @@ impl Live { &mut self.domains[usize::from(id.raw() - FIRST)] } - /// One 2 MiB block of `phys` at `at`, its tables grown on the way. - fn put(&mut self, root: Phys<12>, at: u64, phys: u64) { - let block = Phys::<21>::new(phys).expect("SMMU: a 2 MiB-aligned page of memory"); - let path = plan(at, Leaf::Memory(block, Access::ReadWrite), &self.unit).expect("SMMU: a handed-out address"); + /// `leaf` at `at`, its tables grown on the way. + fn put(&mut self, root: Phys<12>, at: u64, leaf: Leaf) { + let path = plan(at, leaf, &self.unit).expect("SMMU: a handed-out address"); let mut table = root; let (leaf, walk) = path.indices().split_last().expect("a path names at least its leaf's table"); for (level, index) in walk.iter().enumerate() { @@ -83,6 +93,7 @@ impl Live { /// A new domain, with `room` bytes of it handed out first: refused, with no id /// spent, where it would have less than that. pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { + let doorbell = super::super::irqchip::its::doorbell_page(); let mut held = UNIT.lock(); let live = held.as_mut().ok_or(IommuError::NoUnit)?; let domains = if live.unit.asid(0x100).is_some() { 1 << 16 } else { 1 << 8 }; @@ -98,14 +109,22 @@ pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { descriptor.write_u64(8 * i as u64, *word); } descriptor.write_u64(0, words[0]); + if let Some(page) = doorbell { + assert!(page < addresses.floor(), "SMMU: the doorbell at {page:#x} is inside the addresses a domain hands out"); + live.put(root, page, Leaf::Doorbell(Phys::new(page).expect("SMMU: a 4 KiB page below 2^48"))); + } let domain = Domain { asid, root, context, addresses }; log!( - "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {:#x} to {:#x}", + "iommu: domain{id} root={:#x} context={:#x} asid={} addresses from {:#x} to {:#x}, the doorbell {}", root.get(), context.get(), asid.get(), addresses.floor(), - addresses.ceiling() + addresses.ceiling(), + match doorbell { + Some(page) => alloc::format!("page at {page:#x}"), + None => "page nowhere: no ITS is armed".into(), + }, ); live.domains.push(domain); Ok((DomainId::new(id), first)) @@ -121,7 +140,7 @@ pub fn map(id: DomainId, phys: u64, bytes: u64) -> Result { let at = domain.addresses.reserve(bytes).ok_or(IommuError::AddressesExhausted(domain.addresses.ceiling()))?; let root = domain.root; for offset in (0..bytes).step_by(PAGE_2M as usize) { - live.put(root, at.raw() + offset, phys + offset); + live.put(root, at.raw() + offset, block(phys + offset)); } log!("iommu: domain{} maps {phys:#x}..{:#x} at {:#x}", id.raw(), phys + bytes.next_multiple_of(PAGE_2M), at.raw()); Ok(at) @@ -148,7 +167,7 @@ pub fn place(id: DomainId, at: Iova, phys: u64, bytes: u64) -> Result bool { - false - } - - pub fn msi(_source: StreamId, _vector: u8, _dest: u32) -> Result { - unreachable!("no interrupt remapping on this machine, and `is_armed` said so") - } - - pub fn claim(_slot: usize, _source: StreamId, _vector: u8) -> Msi { - unreachable!("no interrupt remapping on this machine, and `is_armed` said so") - } - - pub fn release(_slot: usize, _source: StreamId) { - unreachable!("no interrupt remapping on this machine, and `is_armed` said so") - } - - pub fn pin(_apic_id: u8, _vector: u8, _dest: u32, _level: bool) -> Result { - unreachable!("no interrupt remapping on this machine, and `is_armed` said so") - } -} diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 41881d73319..4266f67d346 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -160,6 +160,11 @@ fn exception(frame: &Frame, entry: u64) -> ! { /// from EL0 a tick or a kick does, once the handler is done, since the context /// holds nothing; from EL1 it only asks for the pass the context will run when /// it may. +/// +/// **Every device arm runs with the preempt count raised**, as x86-64's +/// `device_irq_entry` brackets its handler: an arm that posts a watch or lets +/// go of a `Lock` reaches `preempt::enable`, whose pass at depth zero would +/// otherwise run inside the handler, before its `end`. fn irq(frame: &Frame, from_el0: bool) -> bool { let Some(intid) = irqchip::acknowledge() else { percpu::irq_took(Source::Spurious); @@ -202,37 +207,47 @@ fn irq(frame: &Frame, from_el0: bool) -> bool { // own, so the interface signals this CPU nothing and the halt stays. irqchip::SGI_HALT => cpu::halt(), irqchip::SGI_OFF => super::power::cpu_off(), + _ => {} + } + percpu::preempt_count_up(); + let resched = match intid { irqchip::SGI_KICK => { percpu::irq_took(Source::Kick); - // Raised as an interrupt entry raises it, so the answer's post runs no pass here. - percpu::preempt_count_up(); crate::counters::serve_here(); - percpu::preempt_count_down(); - irqchip::end(intid); - if from_el0 { - return true; - } - crate::preempt::set_need_resched(); + true } #[cfg(feature = "boot-actuators")] irqchip::SGI_STORM => { storm::sgi(); - irqchip::end(intid); + false } // Zero until routed, which the kick's arm above takes first. intid if intid == irqchip::iommu_events() => { percpu::irq_took(Source::DmaFault); - percpu::preempt_count_up(); crate::iommu::fault_interrupt(); - percpu::preempt_count_down(); - irqchip::end(intid); + false } - _ => { - percpu::irq_took(Source::Unclaimed); - UNCLAIMED.fetch_add(1, Relaxed); - LAST_UNCLAIMED.store(intid, Relaxed); - irqchip::end(intid); + _ => match irqchip::its::slot_of(intid) { + Some(slot) => { + percpu::irq_took(Source::UserDev); + crate::pcidev::isr(slot); + true + } + None => { + percpu::irq_took(Source::Unclaimed); + UNCLAIMED.fetch_add(1, Relaxed); + LAST_UNCLAIMED.store(intid, Relaxed); + false + } + }, + }; + percpu::preempt_count_down(); + irqchip::end(intid); + if resched { + if from_el0 { + return true; } + crate::preempt::set_need_resched(); } false } @@ -497,8 +512,24 @@ pub fn install() { } } -pub const HDA_VECTOR: u8 = irqchip::Intid::Hda as u8; -pub const VIRTIO_SOUND_VECTOR: u8 = irqchip::Intid::VirtioSound as u8; +/// What a driver in this kernel names its function's interrupt by, which +/// it is refused a message for (`irqchip::its::msi`). +#[derive(Clone, Copy, Debug)] +pub enum DriverIrq { + Xhci, + Hda, + VirtioSound, +} + +impl core::fmt::Display for DriverIrq { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{self:?}'s") + } +} + +pub const XHCI_VECTOR: DriverIrq = DriverIrq::Xhci; +pub const HDA_VECTOR: DriverIrq = DriverIrq::Hda; +pub const VIRTIO_SOUND_VECTOR: DriverIrq = DriverIrq::VirtioSound; /// The crash report for a panic, from the frame pointer the panic handler /// stood on: the backtrace, which CPU is on which stack, and what the diff --git a/kernel/src/arch/x86_64/apic.rs b/kernel/src/arch/x86_64/apic.rs index 03079ebe015..79b2583a14d 100644 --- a/kernel/src/arch/x86_64/apic.rs +++ b/kernel/src/arch/x86_64/apic.rs @@ -47,10 +47,14 @@ pub const TIMER_VECTOR: u8 = 0x20; /// event all start here. pub const MSI_DOORBELL: u32 = 0xFEE0_0000; -/// The compatibility-format message that raises `vector` on the CPU whose APIC -/// ID is `dest`: the destination in address bits 19:12, the vector in the data. -pub fn msi_message(dest: u32, vector: u8) -> Result<(u32, u32), &'static str> { - Ok((MSI_DOORBELL | (dest << 12), vector as u32)) +/// Every device interrupt in this kernel targets cpu0, named as a destination +/// for a message and for a remapping entry. +pub(crate) const MSG_DEST: u32 = 0; + +/// The compatibility-format message that raises `vector` on [`MSG_DEST`]: the +/// destination in address bits 19:12, the vector in the data. +pub(super) fn msi_message(vector: u8) -> (u32, u32) { + (MSI_DOORBELL | (MSG_DEST << 12), vector as u32) } /// Calibrated LAPIC timer ticks per 10ms (computed on BSP, reused by APs), diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index 4194826b6ab..b6d5f9b0768 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -14,6 +14,8 @@ mod user_dev; mod virtio_sound; mod xhci; +pub(crate) use user_dev::{claim_vector, slot_interrupt}; + use core::arch::naked_asm; use super::cpu; @@ -43,11 +45,31 @@ pub const ISA_VECTORS: [u8; crate::isa::MAX_ROWS] = [Vector::Isa0 as u8, Vector: /// The vector an IOMMU writes into its own `FEDATA`. pub const DMA_FAULT_VECTOR: u8 = Vector::DmaFault as u8; +/// The vector a driver in this kernel arms its function's message with: one +/// of the three below, each its own gate. +#[derive(Clone, Copy)] +pub struct DriverIrq(Vector); + +impl DriverIrq { + pub(crate) const fn vector(self) -> u8 { + self.0 as u8 + } +} + +impl core::fmt::Display for DriverIrq { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "vector {:#x}", self.vector()) + } +} + +/// The vector the xHCI controller's message-signalled interrupt carries. +pub const XHCI_VECTOR: DriverIrq = DriverIrq(Vector::Xhci); + /// The vector the HDA controller's message-signalled interrupt carries. -pub const HDA_VECTOR: u8 = Vector::Hda as u8; +pub const HDA_VECTOR: DriverIrq = DriverIrq(Vector::Hda); /// The vector the virtio-sound device's MSI-X entry carries. -pub const VIRTIO_SOUND_VECTOR: u8 = Vector::VirtioSound as u8; +pub const VIRTIO_SOUND_VECTOR: DriverIrq = DriverIrq(Vector::VirtioSound); const PF_PRESENT: u64 = 1 << 0; const PF_WRITE: u64 = 1 << 1; diff --git a/kernel/src/arch/x86_64/idt/user_dev.rs b/kernel/src/arch/x86_64/idt/user_dev.rs index 9ad2dccbc32..f26206dfcb3 100644 --- a/kernel/src/arch/x86_64/idt/user_dev.rs +++ b/kernel/src/arch/x86_64/idt/user_dev.rs @@ -10,6 +10,22 @@ //! the claim's watch is posted from the handler. use super::device_irq::device_irq_entry; +use super::Vector; + +/// The vector each claim slot's message carries, by slot: the vector is how +/// the kernel knows which claim a message belongs to. +const VECTORS: [Vector; crate::pcidev::MAX_FUNCTIONS] = + [Vector::UserDev0, Vector::UserDev1, Vector::UserDev2, Vector::UserDev3]; + +/// The vector claim slot `slot`'s remapping entry delivers. +pub(crate) fn claim_vector(slot: usize) -> u8 { + VECTORS[slot] as u8 +} + +/// What claim slot `slot`'s message raises, for the record. +pub(crate) fn slot_interrupt(slot: usize) -> impl core::fmt::Display { + super::DriverIrq(VECTORS[slot]) +} fn took(slot: usize) { crate::arch::percpu::irq_took!(UserDev); diff --git a/kernel/src/arch/x86_64/ioapic.rs b/kernel/src/arch/x86_64/ioapic.rs index a46b441292e..577e6012498 100644 --- a/kernel/src/arch/x86_64/ioapic.rs +++ b/kernel/src/arch/x86_64/ioapic.rs @@ -21,7 +21,6 @@ pub use toyos_acpi::{Polarity, Trigger}; use crate::arch::IrqGuard; use crate::drivers::acpi::MadtInfo; -use crate::iommu::Delivery; use crate::log; use crate::mm::policy::MmioPolicy; use crate::mm::Mmio; @@ -245,7 +244,7 @@ fn locate(gsi: Gsi) -> Result<(&'static Unit, u32), RouteError> { /// /// Under remapping the entry names a table slot and the destination lives in /// that slot — but the id must still fit whatever names it, so `DestTooWide` -/// moves rather than disappearing and arrives as [`Delivery::Refused`]. +/// moves rather than disappearing and arrives as [`RouteError::NotRemappable`]. pub fn route( gsi: Gsi, vector: u8, @@ -255,17 +254,20 @@ pub fn route( ) -> Result<(), RouteError> { let (unit, n) = locate(gsi)?; let level = trigger == Trigger::Level; - let (index, high) = - match crate::iommu::remap_pin(unit.id, vector, dest_apic_id, level) { - Delivery::Direct => { - if dest_apic_id >= 0xFF { - return Err(RouteError::DestTooWide(dest_apic_id)); - } - (0, dest_apic_id << 24) - } - Delivery::Remapped(pin) => (pin.low, pin.high), - Delivery::Refused(why) => return Err(RouteError::NotRemappable(gsi, why)), - }; + // Under no unit that remaps, the entry names its CPU itself; under one, a + // pin with no entry of its own is refused rather than written in a form + // the unit blocks. + let (index, high) = if !super::vtd::interrupt::is_armed() { + if dest_apic_id >= 0xFF { + return Err(RouteError::DestTooWide(dest_apic_id)); + } + (0, dest_apic_id << 24) + } else { + match super::vtd::interrupt::pin(unit.id, vector, dest_apic_id, level) { + Ok(pin) => (pin.low, pin.high), + Err(why) => return Err(RouteError::NotRemappable(gsi, why)), + } + }; let low = vector as u32 | index | if polarity == Polarity::Low { RTE_POLARITY_LOW } else { 0 } diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index de6dfb94d6b..c1fe5528e4c 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -46,8 +46,11 @@ pub use apic as irqchip; pub use i8042 as keyboard_controller; pub use idt as trap; pub use vtd as iommu_unit; +pub use vtd::interrupt as message_unit; -pub use apic::{msi_message, MSI_DOORBELL}; +pub use apic::MSI_DOORBELL; +pub use idt::DriverIrq; +pub(crate) use idt::slot_interrupt; /// The machine every program image this kernel loads must be built for. pub const ELF_MACHINE: toyos_elf::Machine = toyos_elf::Machine::X86_64; diff --git a/kernel/src/arch/x86_64/pio.rs b/kernel/src/arch/x86_64/pio.rs index 05b014183fc..c27e1d7615b 100644 --- a/kernel/src/arch/x86_64/pio.rs +++ b/kernel/src/arch/x86_64/pio.rs @@ -194,7 +194,7 @@ pub fn route(row: usize, wire: Wire) -> Result { ioapic::route( wire.gsi, super::idt::ISA_VECTORS[row], - crate::drivers::pci::MSG_DEST, + super::apic::MSG_DEST, wire.trigger, wire.polarity, ) diff --git a/kernel/src/arch/x86_64/vtd/interrupt.rs b/kernel/src/arch/x86_64/vtd/interrupt.rs index a4d8b254607..c6596c24eb2 100644 --- a/kernel/src/arch/x86_64/vtd/interrupt.rs +++ b/kernel/src/arch/x86_64/vtd/interrupt.rs @@ -35,8 +35,9 @@ use crate::log; use alloc::vec::Vec; -use crate::drivers::pci::MSG_DEST; -use crate::iommu::{Refused, StreamId}; +use super::super::apic::MSG_DEST; +use super::super::idt::DriverIrq; +use crate::iommu::StreamId; use crate::sync::Lock; use super::table::Table; @@ -73,6 +74,32 @@ pub struct Msi { pub data: u32, } +/// Why a source could not be given an entry. Carried rather than collapsed: +/// one message for all three sends whoever reads it looking in the wrong place. +#[derive(Clone, Copy)] +pub enum Refused { + /// Wider than the destination an entry holds without extended interrupt mode. + DestinationTooWide(u32), + /// Every entry in the table is already spoken for. + TableFull, + /// Firmware's device scopes named no requester id for this interrupt controller. + ControllerUnnamed(u8), +} + +impl core::fmt::Display for Refused { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::DestinationTooWide(id) => { + write!(f, "apic id {id:#x} does not fit a remapping entry's destination") + } + Self::TableFull => write!(f, "the interrupt remapping table is full"), + Self::ControllerUnnamed(id) => { + write!(f, "firmware named no requester id for interrupt controller {id}") + } + } + } +} + pub struct Pin { pub low: u32, pub high: u32, @@ -211,13 +238,22 @@ fn message(index: u16) -> Msi { } } -pub fn msi(source: StreamId, vector: u8, dest: u32) -> Result { - Ok(message(allocate(source, vector, dest, false)?)) +/// The message a kernel driver's `source` raises `irq` with: an entry of its +/// own where the table is armed, and the compatibility format where no unit +/// remaps, which is what that message has always said. +pub fn msi(source: StreamId, irq: DriverIrq) -> Result { + if !is_armed() { + let (address, data) = super::super::apic::msi_message(irq.vector()); + return Ok(Msi { address, data }); + } + Ok(message(allocate(source, irq.vector(), MSG_DEST, false)?)) } -/// Claim slot `slot`'s entry, written for `source` at `vector` and published -/// to every unit before this returns, and the message that reaches it. -pub fn claim(slot: usize, source: StreamId, vector: u8) -> Msi { +/// Claim slot `slot`'s entry, written for `source` at the slot's vector and +/// published to every unit before this returns, and the message that reaches +/// it. +pub fn claim(slot: usize, source: StreamId) -> Result { + let vector = super::super::idt::claim_vector(slot); let (index, read) = { let remap = REMAP.lock(); let Some(table) = remap.table else { @@ -239,7 +275,7 @@ pub fn claim(slot: usize, source: StreamId, vector: u8) -> Msi { (index, table.read_pair(slot)) }; report(index, source, MSG_DEST, read); - message(index) + Ok(message(index)) } /// Claim slot `slot`'s entry not present again, and gone from every unit's diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index adfa2773c63..78f795f6742 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -25,10 +25,6 @@ const INVALID_VENDOR: u16 = 0xFFFF; /// The one MSI-X table entry this kernel programs; a device's queues must point at it too. pub const MSIX_ENTRY: u16 = 0; -// Every device interrupt in this kernel targets cpu0, named as a destination for -// the message and for the unit to put in an entry. -pub(crate) const MSG_DEST: u32 = 0; - /// Why a walk of a function's capability list answered no capability. pub enum NoCapability { /// The walk reached the list's terminator and nothing on it carried the id. @@ -278,20 +274,20 @@ impl PciDevice { self.mmio.read_u16(COMMAND); } - /// Point this function's [`MSIX_ENTRY`] at `vector` and enable it: a - /// kernel driver's arming, whose message is compatibility format on a - /// machine that remaps nothing. A claimed function is armed by + /// Point this function's [`MSIX_ENTRY`] at `irq` and enable it: a kernel + /// driver's arming, whose message the architecture makes + /// ([`crate::iommu::driver_msi`]). A claimed function is armed by /// [`arm_claimed_msix`] instead. /// /// Answers the entry's own window, which stays this kernel's: masking is a /// write to it, and a claimant that could reach it could aim the device's /// message at any address the LAPIC decodes. /// - /// `None` where it has no MSI-X entry this kernel can write, or the unit - /// refuses its message. - pub(in crate::drivers) fn enable_msix(&self, vector: u8) -> Option { + /// `None` where it has no MSI-X entry this kernel can write, or it is + /// refused a message. + pub(in crate::drivers) fn enable_msix(&self, irq: crate::arch::DriverIrq) -> Option { let entry = self.msix_entry().ok()?; - let (address, data) = self.message(vector)?; + let (address, data) = self.message(irq)?; Some(self.write_msix(entry, address, data)) } @@ -358,31 +354,21 @@ impl PciDevice { self.bus, self.dev, self.func); } - /// The address and data this function's interrupt registers take for `vector`. + /// The address and data this function's interrupt registers take for `irq`. /// - /// `None` refuses the device: the machine remaps interrupts and this - /// function has no entry, so the message it would otherwise write is one the - /// unit blocks. - fn message(&self, vector: u8) -> Option<(u32, u32)> { - let dest = MSG_DEST; - match crate::iommu::remap_msi(self.bus, self.dev, self.func, vector, dest) { - // The same CPU the remappable one would name. - crate::iommu::Delivery::Direct => match crate::arch::msi_message(dest, vector) { - Ok(message) => Some(message), - Err(why) => { - log!("PCI {:02x}:{:02x}.{}: not armed — {why}", self.bus, self.dev, self.func); - None - } - }, - crate::iommu::Delivery::Remapped(m) => Some((m.address, m.data)), - crate::iommu::Delivery::Refused(why) => { + /// `None` refuses the device: the message it would otherwise write is one + /// nothing delivers. + fn message(&self, irq: crate::arch::DriverIrq) -> Option<(u32, u32)> { + match crate::iommu::driver_msi(self.bus, self.dev, self.func, irq) { + Ok(message) => Some((message.address, message.data)), + Err(why) => { log!("PCI {:02x}:{:02x}.{}: not armed — {why}", self.bus, self.dev, self.func); None } } } - /// Point this function's single MSI message at `vector` and enable it: a + /// Point this function's single MSI message at `irq` and enable it: a /// kernel driver's arming, as [`Self::enable_msix`] is. /// /// **A driver in this kernel may arm this however the MSI-X walk failed**, a @@ -390,11 +376,11 @@ impl PciDevice { /// holder, so an MSI-X table past that link is one nobody but this kernel /// could reach. A hand-over is the caller that has to tell the two apart, /// and `crate::pcidev`'s header says why. - pub(in crate::drivers) fn enable_msi(&self, vector: u8) -> bool { + pub(in crate::drivers) fn enable_msi(&self, irq: crate::arch::DriverIrq) -> bool { let Ok(cap) = self.capability(msi::CAP_ID) else { return false; }; - let Some((address, data)) = self.message(vector) else { + let Some((address, data)) = self.message(irq) else { return false; }; self.write_msi(cap, address, data); @@ -495,8 +481,8 @@ impl PciDevice { } } -/// Point a claimed function's [`MSIX_ENTRY`] at its slot's own remapping -/// entry and enable it. Takes the message and nothing else: the function is +/// Point a claimed function's [`MSIX_ENTRY`] at its slot's own message and +/// enable it. Takes the message and nothing else: the function is /// the one the entry was written for, and no other message reaches here. pub fn arm_claimed_msix(message: &crate::iommu::Remapped) -> Result { let pci = message.function(); diff --git a/kernel/src/drivers/virtio_sound.rs b/kernel/src/drivers/virtio_sound.rs index e39ff6c5b76..910b4a26161 100644 --- a/kernel/src/drivers/virtio_sound.rs +++ b/kernel/src/drivers/virtio_sound.rs @@ -449,6 +449,6 @@ fn arm_interrupt(pci: &PciDevice, device: &VirtioDevice) -> bool { ); return false; } - log!("virtio-sound: MSI-X vector {vector:#x} on table entry {MSIX_ENTRY}"); + log!("virtio-sound: MSI-X {vector} on table entry {MSIX_ENTRY}"); true } diff --git a/kernel/src/drivers/xhci/mod.rs b/kernel/src/drivers/xhci/mod.rs index fd76926adf9..4977346b789 100644 --- a/kernel/src/drivers/xhci/mod.rs +++ b/kernel/src/drivers/xhci/mod.rs @@ -67,8 +67,6 @@ const IR0_ERSTSZ: u64 = 0x28; const IR0_ERSTBA: u64 = 0x30; // 64-bit const IR0_ERDP: u64 = 0x38; // 64-bit -const XHCI_VECTOR: u8 = 0x21; - #[repr(C)] #[derive(Clone, Copy)] struct Trb { diff --git a/kernel/src/drivers/xhci/wait/boot.rs b/kernel/src/drivers/xhci/wait/boot.rs index 92915c6f221..4f1843f7582 100644 --- a/kernel/src/drivers/xhci/wait/boot.rs +++ b/kernel/src/drivers/xhci/wait/boot.rs @@ -15,7 +15,8 @@ use device::{begin, reset_done, reset_port}; use super::super::{ErstEntry, Layout, MscBlock, PortMask, Portsc, Protocols, TrbRing}; use super::super::{XhciController, PAGE, RING_SIZE, USB_TIMEOUT_NS}; use super::super::{CAP_CAPLENGTH, CAP_DBOFF, CAP_HCCPARAMS1, CAP_HCSPARAMS1, CAP_HCSPARAMS2}; -use super::super::{CAP_RTSOFF, HCC_PPC, XHCI_VECTOR}; +use super::super::{CAP_RTSOFF, HCC_PPC}; +use crate::arch::trap::XHCI_VECTOR; use super::super::{IR0_ERDP, IR0_ERSTBA, IR0_ERSTSZ, IR0_IMAN, IR0_IMOD}; use super::super::{OFF_CMD_RING, OFF_DCBAA, OFF_ERST, OFF_EVT_RING}; use super::super::{OP_CONFIG, OP_CRCR, OP_DCBAAP, OP_PAGESIZE, OP_PORT_BASE, OP_USBCMD, OP_USBSTS}; @@ -252,7 +253,7 @@ fn init_one(pci_dev: &PciDevice) -> Option { ); return None; }; - log!("xHCI: {irq} enabled (vector {XHCI_VECTOR:#x})"); + log!("xHCI: {irq} enabled ({XHCI_VECTOR})"); let bar = crate::mm::paging::map_mmio(bar_addr, 0x10000, MmioPolicy::Uncacheable); diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 003e4cd1423..667d2e086c0 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -2,7 +2,7 @@ //! //! Inventories the machine's IOMMU units, turns translation on, and hands a driver an address space of its own to put its DMA in; an unusable unit is logged and left off rather than halting boot. Names above the backend stay backend-neutral, and what every backend decides alike is declared here once: a domain's addresses ([`window`]) and what a fault record ends ([`fault`]). //! -//! The refusal is deliberately not yet built for a driver in this kernel: landing it before any userspace driver exists would cost every machine and protect nothing. A function a *process* drives is the other case and is refused ([`OwnSpace`], [`remapping`]), because a descriptor it writes a physical address into is an arbitrary read and write over all of memory, and a message it sends unremapped is any vector at any CPU; its message is its claim slot's own entry ([`Remapped`]). +//! The refusal is deliberately not yet built for a driver in this kernel: landing it before any userspace driver exists would cost every machine and protect nothing. A function a *process* drives is the other case and is refused ([`OwnSpace`], [`remapping`]), because a descriptor it writes a physical address into is an arbitrary read and write over all of memory, and a message nothing translates for it alone raises any claim's interrupt; its message is its claim slot's own, translated for it alone ([`Remapped`]). //! //! `trait Iommu` is deliberately not added: with one backend it would have a single implementor. @@ -10,6 +10,7 @@ #![warn(clippy::undocumented_unsafe_blocks)] use crate::arch::iommu_unit as unit; +use crate::arch::message_unit as messages; pub(crate) mod fault; pub(crate) mod window; @@ -275,81 +276,40 @@ pub fn init( unit::init(rsdp_addr, devices, windows); } -/// How a kernel driver's source must address its interrupt. Not a yes/no: a -/// caller that folded the third answer into [`Delivery::Direct`] would write a -/// message the unit blocks and lose the device in silence. -pub enum Delivery { - /// No unit remaps interrupts on this machine; write what has always been written. - Direct, - /// Write this instead — the interrupt now reaches its destination through the unit. - Remapped(T), - /// The unit remaps and this source has no entry; the caller refuses the device. - Refused(Refused), -} - -/// Why a source could not be given an entry. Carried rather than collapsed: -/// one message for all three sends whoever reads it looking in the wrong place. -#[derive(Clone, Copy)] -pub enum Refused { - /// Wider than the destination an entry holds without extended interrupt mode. - DestinationTooWide(u32), - /// Every entry in the table is already spoken for. - TableFull, - /// Firmware's device scopes named no requester id for this interrupt controller. - ControllerUnnamed(u8), -} - -impl core::fmt::Display for Refused { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::DestinationTooWide(id) => { - write!(f, "apic id {id:#x} does not fit a remapping entry's destination") - } - Self::TableFull => write!(f, "the interrupt remapping table is full"), - Self::ControllerUnnamed(id) => { - write!(f, "firmware named no requester id for interrupt controller {id}") - } - } - } -} +/// Why a source could not be given a message: the architecture's own reasons. +pub use messages::Refused; pub struct MsiMessage { pub address: u32, pub data: u32, } -pub struct PinRedirect { - pub low: u32, - pub high: u32, -} - -/// Every unit on this machine remaps interrupts, so a claimed function can be -/// given a message only its own entry delivers. Only [`remapping`] makes one. +/// Every claimed function's message is translated to its claim slot's own +/// interrupt, and no other's. Only [`remapping`] makes one. #[derive(Clone, Copy)] pub struct Remapping(()); -/// No unit remaps this machine's interrupts, so a function a process drives -/// would carry a compatibility-format message, which names any vector at any -/// CPU. +/// Nothing on this machine translates a claimed function's message to its +/// claim's own interrupt, so the message it would carry names any. pub struct NotRemapped; /// Whether a claimed function's message can be remapped: a fact of the /// machine, fixed before the first driver arms anything. pub fn remapping() -> Result { - if unit::interrupt::is_armed() { + if messages::is_armed() { Ok(Remapping(())) } else { Err(NotRemapped) } } -/// A claimed function's message: claim slot `slot`'s own remapping entry, -/// written for [`Self::function`] alone. +/// A claimed function's message: claim slot `slot`'s own, translated for +/// [`Self::function`] alone. /// /// The only message a function a process drives is armed with, since -/// [`claim_msi`] is the only thing that makes one; and dropping it puts the -/// entry back to not present, so no refusal after it is written leaves the -/// function an entry it can reach. +/// [`claim_msi`] is the only thing that makes one; and dropping it takes the +/// translation back, so no refusal after it is written leaves the function a +/// message that reaches anything. pub struct Remapped { slot: usize, function: crate::drivers::pci::PciDevice, @@ -377,49 +337,32 @@ impl Remapped { impl Drop for Remapped { fn drop(&mut self) { - unit::interrupt::release(self.slot, self.stream()); + messages::release(self.slot, self.stream()); } } -/// Write claim slot `slot`'s entry for `function` at `vector`. +/// Claim slot `slot`'s message, translated for `function` alone. pub fn claim_msi( _: Remapping, slot: usize, function: &crate::drivers::pci::PciDevice, - vector: u8, -) -> Remapped { +) -> Result { let stream = StreamId::pci(function.bus, function.dev, function.func); - let msi = unit::interrupt::claim(slot, stream, vector); - Remapped { slot, function: *function, address: msi.address, data: msi.data } + let msi = messages::claim(slot, stream)?; + Ok(Remapped { slot, function: *function, address: msi.address, data: msi.data }) } -/// Where a kernel driver's `bus:device.function`'s message-signalled interrupt -/// must point. Takes the triple, not a [`StreamId`]: what a requester id is -/// stays in this module. -pub fn remap_msi( +/// The message a kernel driver's `bus:device.function` raises `irq` with. +/// Takes the triple, not a [`StreamId`]: what a requester id is stays in this +/// module. +pub fn driver_msi( bus: u8, device: u8, function: u8, - vector: u8, - dest: u32, -) -> Delivery { - if !unit::interrupt::is_armed() { - return Delivery::Direct; - } - match unit::interrupt::msi(StreamId::pci(bus, device, function), vector, dest) { - Ok(msi) => Delivery::Remapped(MsiMessage { address: msi.address, data: msi.data }), - Err(why) => Delivery::Refused(why), - } -} - -pub fn remap_pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Delivery { - if !unit::interrupt::is_armed() { - return Delivery::Direct; - } - match unit::interrupt::pin(apic_id, vector, dest, level) { - Ok(pin) => Delivery::Remapped(PinRedirect { low: pin.low, high: pin.high }), - Err(why) => Delivery::Refused(why), - } + irq: crate::arch::DriverIrq, +) -> Result { + messages::msi(StreamId::pci(bus, device, function), irq) + .map(|msi| MsiMessage { address: msi.address, data: msi.data }) } /// Record that `bus:device.function` is driven by a process on `slot`, or is no @@ -429,7 +372,7 @@ pub fn remap_pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Delivery) { fault::user_owned(StreamId::pci(bus, device, function), slot); } diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index 0b5487fa7e2..9cc224fad37 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -3,9 +3,9 @@ //! The line through the device is **who can name an address**. This module //! keeps config space — there is no write path to it from userland — puts the //! function in an address space of its own at the unit *before* it enables bus -//! mastering, programs the interrupt vector into whichever of the function's two -//! message mechanisms it has, through its slot's own remapping entry and no -//! other message, and hands out every device address a descriptor +//! mastering, programs its slot's own message into whichever of the function's +//! two message mechanisms it has, translated for that function alone and no +//! other, and hands out every device address a descriptor //! may carry. Nothing the holder writes into a descriptor can make the device //! touch memory the kernel did not grant it: the domain maps the grants — the //! claim's own, and the regions of ordinary memory its holder lends it @@ -101,8 +101,8 @@ //! by the time it was used. The binding is lent by the claim under the lock //! its release takes it with (`object::Held`): a call runs wholly before the //! release or finds no binding. Outside a claim's own mint and release, only -//! the two handlers name a slot, by the vector and the requester the hardware -//! gave them. +//! the two handlers name a slot, by the interrupt and the requester the +//! hardware gave them. //! //! **A poll is on a slot's watch only while its claim holds the slot.** It is //! registered with the binding ([`add_poll`]), so before the release, whose @@ -131,7 +131,7 @@ use toyos_pci::{af, aperture, bar, express, msix, placement, pm, probe}; use crate::device::{Claim, ClaimError}; use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, NoCapability, NoEntry, PciDevice}; -use crate::iommu::{IommuError, OwnSpace, Remapped}; +use crate::iommu::{IommuError, OwnSpace, Refused, Remapped}; use crate::mm::policy::{CachePolicy, MmioPolicy}; use crate::mm::{align_2m, DirectMap, Mmio, PAGE_2M}; use crate::object::shm::{Region, SharedMemObject}; @@ -141,15 +141,10 @@ use crate::watch::IrqWatch; /// How many functions this machine can hand out at once. /// -/// A small fixed number because each one costs an interrupt vector, and a -/// vector in `arch::idt` is declared rather than allocated. +/// A small fixed number because each one costs an interrupt of its own, which +/// the architecture declares per slot rather than allocates. pub const MAX_FUNCTIONS: usize = 4; -/// The vectors those functions' messages carry, one per slot. Declared here and -/// in `arch::idt`'s table; `VECTORS.len() == MAX_FUNCTIONS` is what keeps the -/// two from disagreeing. -pub const VECTORS: [u8; MAX_FUNCTIONS] = [0x28, 0x29, 0x2A, 0x2B]; - /// The most one grant may be. A driver's rings and buffers are kilobytes to a /// megabyte; this is room for a queue depth nothing in reach uses, and a bound /// so a refused-but-plausible request is a refusal rather than the machine's @@ -234,9 +229,9 @@ struct Aimed { static RESIDUE: [Lock>; MAX_FUNCTIONS] = [const { Lock::new(Vec::new()) }; MAX_FUNCTIONS]; -/// How a claimed function was made to speak, and the slot's remapping entry it -/// speaks through. Both deliver [`VECTORS`]`[slot]` into the same -/// [`Interrupt`] and the claim answers the same handle either way. +/// How a claimed function was made to speak, and the slot's message it speaks +/// with. Both deliver the slot's own interrupt into the same [`Interrupt`] and +/// the claim answers the same handle either way. enum Armed { /// This function's one MSI-X table entry, mapped for the kernel alone. Msix(Mmio, Remapped), @@ -599,9 +594,11 @@ fn account_for(firmware: &[RootBridgeWindow], decoded: &[(u16, u64, u64)]) { /// Why a function could not be handed over. Carried rather than collapsed: one /// message for all of them sends whoever reads the log looking in the wrong /// place. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy)] enum Refusal { NotRemapped, + /// The machine translates messages, and none could be made for this function. + NoMessage(Refused), NoInterrupt, MsixUnusable, CapsTruncated, @@ -639,6 +636,7 @@ impl core::fmt::Display for Refusal { "its interrupts would not be remapped on this machine, and a message that is not \ remapped can raise any vector on any CPU" ), + Self::NoMessage(why) => write!(f, "no message of its own could be made for it — {why}"), Self::NoInterrupt => write!( f, "neither its MSI-X nor its MSI could be armed, and a claim with no interrupt \ @@ -769,14 +767,13 @@ pub fn claim(id: PciId) -> Result<(PciFunctionInfo, Claim), ClaimError> { IRQ[slot].clear(); crate::iommu::note_user_owned(pci.bus, pci.dev, pci.func, Some(slot)); log!( - "pcidev: PCI {:02x}:{:02x}.{} [{:04x}:{:04x}] handed over on slot {slot}, \ - vector {:#x}", + "pcidev: PCI {:02x}:{:02x}.{} [{:04x}:{:04x}] handed over on slot {slot}, {}", pci.bus, pci.dev, pci.func, id.vendor, id.device, - VECTORS[slot], + crate::arch::slot_interrupt(slot), ); Ok((info, Claim::pci(binding))) } @@ -826,7 +823,7 @@ fn bring_up(pci: PciDevice, id: PciId, slot: usize) -> Result { // mechanism can be armed on is one no holder could ever be told anything // about. Every refusal from here drops `message`, which puts the slot's // entry back to not present. - let message = crate::iommu::claim_msi(remapping, slot, &pci, VECTORS[slot]); + let message = crate::iommu::claim_msi(remapping, slot, &pci).map_err(Refusal::NoMessage)?; let armed = match arm_claimed_msix(&message) { Ok(entry) => Armed::Msix(entry, message), Err(NoEntry::Unusable) => return Err(Refusal::MsixUnusable), @@ -1737,7 +1734,7 @@ pub fn take_record(binding: &Binding) -> Result, Syscall } let taken = IRQ[slot].take(); if taken.is_some() && IRQ[slot].take_unannounced() { - log!("pcidev: slot {slot} took its first message on vector {:#x}", VECTORS[slot]); + log!("pcidev: slot {slot} took its first message on {}", crate::arch::slot_interrupt(slot)); } Ok(taken.map(|count| DeviceIrqRecord { count })) } @@ -1748,10 +1745,15 @@ pub fn has_irq(binding: &Binding) -> bool { IRQ[binding.slot].armed() || IRQ[binding.slot].faulted() } -/// Records one message and posts the claim's watch. Called from the vector's -/// handler, so it allocates nothing; `record.rs` owns the counting, and -/// `kernel-loom` models it against a concurrent reader. +/// Records one message and posts the claim's watch. Called from the +/// interrupt's handler, so it allocates nothing; `record.rs` owns the +/// counting, and `kernel-loom` models it against a concurrent reader. +/// +/// **The handler holds the preempt count raised across it**: the post lets go +/// of a count of its own, and at depth zero that would run a scheduler pass +/// inside the handler, before its end of interrupt. pub fn isr(slot: usize) { + assert!(crate::preempt::count() > 0, "pcidev: slot {slot}'s interrupt arrived with the preempt count at zero"); IRQ[slot].took(); WATCHES[slot].post_in_place(); } diff --git a/src/build.rs b/src/build.rs index 63a8eefa7c9..cb37eb243dd 100644 --- a/src/build.rs +++ b/src/build.rs @@ -2932,6 +2932,7 @@ mod tests { "tests/proctreecase/system.toml", "tests/slotscase/system.toml", "tests/testcases/system.toml", + "tests/virtclaimcase/system.toml", "tests/virtjobcase/system.toml", "tests/virtpaniccase/system.toml", "tests/virtrebootcase/system.toml", diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 448a000089d..978b6c61e9c 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -736,13 +736,16 @@ pub enum Profile { /// [`Profile::Virt`] with its SMMUv3 and two of QEMU's `iommu-testdev`, a /// function that writes where it is told to through the unit. VirtSmmu, + /// [`Profile::Virt`] with its SMMUv3, QEMU's emulated GICv3 and its ITS, + /// and QEMU's `edu`, a function that sends an MSI when told to. + VirtIts, } impl Profile { /// The architecture this machine is. pub fn arch(self) -> Arch { match self { - Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg | Self::VirtSmmu => { + Self::Virt | Self::VirtNoRng | Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg | Self::VirtSmmu | Self::VirtIts => { Arch::Aarch64 } Self::Headless @@ -895,6 +898,10 @@ struct Shape { enum Smmu { Absent, WithTestdev, + /// With the ITS behind it, which needs QEMU's own GIC: HVF's refuses one. + /// With it comes one `edu`, whose message is the only one a guest of + /// this suite can raise on demand. + WithIts, } /// Where a machine's image and its DATA are. A size is stated because a @@ -942,6 +949,7 @@ impl Profile { Self::VirtEl2 | Self::VirtEl2NoVhe | Self::VirtTcg => Self::Virt.shape(), Self::VirtNoRng => Shape { rng: false, ..Self::Virt.shape() }, Self::VirtSmmu => Shape { smmu: Smmu::WithTestdev, ..Self::Virt.shape() }, + Self::VirtIts => Shape { smmu: Smmu::WithIts, ..Self::Virt.shape() }, Self::Virt => Shape { vga: "std", panel: None, @@ -2143,6 +2151,7 @@ fn qemu_command( match shape.smmu { Smmu::Absent => machine, Smmu::WithTestdev => format!("{machine},iommu=smmuv3"), + Smmu::WithIts => format!("{machine},iommu=smmuv3,its=on,kernel-irqchip=off"), } } }; @@ -2276,6 +2285,9 @@ fn qemu_command( qemu.arg("-device").arg("iommu-testdev"); qemu.arg("-device").arg("iommu-testdev"); } + if shape.smmu == Smmu::WithIts { + qemu.arg("-device").arg("edu"); + } // The NIC before the virtio block, so a profile that has one and not the // other still creates it after the unit and before everything else. diff --git a/tests/toyos-rust-tests/src/bin/claim_lpi.rs b/tests/toyos-rust-tests/src/bin/claim_lpi.rs new file mode 100644 index 00000000000..0c3a305b14d --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/claim_lpi.rs @@ -0,0 +1,60 @@ +//! QEMU's `edu` claimed, and told to send its MSI again and again: each +//! message is its claim's one interrupt, read off the claim once the poll on +//! it answers, one record of one interrupt each time. +//! +//! The function masters the bus only once it has a grant, and QEMU sends an +//! MSI as a write the function makes, so one grant comes first. + +use std::time::Duration; + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::poller::{Poller, READABLE}; +use toyos::syscap::SysCap; +use toyos::volatile::Window; +use toyos::PciDev; +use toyos_abi::syscall::{PciId, SyscallError}; + +/// `hw/misc/edu.c`: QEMU's educational device. +const EDU: PciId = PciId { vendor: 0x1234, device: 0x11e8 }; + +/// Its BAR 0: the identification register, and the two that raise and +/// acknowledge its interrupt (`docs/specs/edu.rst`). +const IDENTIFICATION: usize = 0x00; +const RAISE: usize = 0x60; +const ACKNOWLEDGE: usize = 0x64; +/// The identification register's low byte. +const EDU_ID: u32 = 0xed; + +/// How many messages the job asks for. +const MESSAGES: u64 = 16; + +/// How long one message has to come back as the claim's interrupt: a +/// liveness ceiling. +const BOUND: Duration = Duration::from_secs(10); + +fn main() { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability"); + let dev: PciDev = cap.claim_pci(EDU).unwrap_or_else(|why| panic!("claim_lpi: the claim on edu was refused: {why:?}")); + let info = dev.describe().expect("claim_lpi: the claim's description"); + let bar = dev.map_bar(0, info.bar_bytes[0]).expect("claim_lpi: BAR 0"); + // SAFETY: the mapping is the BAR's length and lives as long as `bar`. + let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; + let id = regs.read::(IDENTIFICATION); + assert_eq!(id & 0xff, EDU_ID, "claim_lpi: BAR 0 answers {id:#010x}, which is not edu's identification"); + let _grant = dev.dma_alloc(4096).expect("claim_lpi: a grant"); + assert_eq!(dev.irq().map(|r| r.count), Err(SyscallError::WouldBlock), "claim_lpi: an interrupt before any was raised"); + + let poller = Poller::new(1); + for sent in 1..=MESSAGES { + regs.write::(RAISE, 1); + poller.watch(&dev, READABLE, 0); + poller.wait(1, BOUND.as_nanos() as u64, |_| {}); + match dev.irq() { + Ok(record) => assert_eq!(record.count, 1, "claim_lpi: message {sent} read as {} interrupts", record.count), + Err(why) => panic!("claim_lpi: message {sent} was not read as the claim's interrupt within {BOUND:?}: {why:?}"), + } + regs.write::(ACKNOWLEDGE, 1); + } + assert_eq!(dev.irq().map(|r| r.count), Err(SyscallError::WouldBlock), "claim_lpi: an interrupt no message raised"); + println!("claim_lpi: {MESSAGES} messages edu sent were read as {MESSAGES} interrupts of its claim, one each"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 5dc34455148..fc8d725a002 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -114,6 +114,8 @@ const RUST_SKIP: &[&str] = &[ "trace_flood", // It takes the machine down; `virt_fatal_halts_the_others_first` runs it. "panic_halts_first", + // It claims QEMU's `edu`, which only `virt_claim_lpi`'s machine has. + "claim_lpi", // Needs a launcher and a declared `cat` and shell, which `tests/testcases` // does not give: the `process_tree` metal row runs it on tests/proctreecase. "process_tree", @@ -297,6 +299,7 @@ const SCREEN_TESTS: &[(&str, qemu::Profile)] = &[ ("virt_jobs_at_el2", qemu::Profile::VirtEl2), ("virt_random_differs", qemu::Profile::Virt), ("virt_smmu", qemu::Profile::VirtSmmu), + ("virt_claim_lpi", qemu::Profile::VirtIts), ("virt_no_seed_refused", qemu::Profile::VirtNoRng), ]; @@ -2206,6 +2209,95 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { Ok(()) } +/// `tests/toyos-rust-tests`' binary that `tests/virtclaimcase` runs as its job +/// `test_rs_claim_lpi`. +const VIRT_CLAIM_LPI: &str = "claim_lpi"; + +/// A claimed function's message, translated by the ITS into its claim slot's +/// LPI: `tests/virtclaimcase`'s one job claims QEMU's `edu` and reads every +/// message it asks it for as one interrupt of its claim +/// (`tests/toyos-rust-tests/src/bin/claim_lpi.rs`). Judged beside it on the +/// kernel's lines: the ITS armed from the MADT; the function handed over on +/// a slot, its DeviceID the requester ID QEMU's IORT maps it to, its slot's +/// LPI raised; and at the claim's end the DeviceID unmapped again, and no +/// access of the function's refused at the SMMUv3. +fn virt_claim_lpi(profile: qemu::Profile) -> Result<(), String> { + const JOB: &str = "test_rs_claim_lpi"; + const SAID: &str = "claim_lpi: 16 messages edu sent were read as 16 interrupts of its claim, one each"; + const EDU: &str = "[1234:11e8]"; + let config = compile::repo_root().join("tests/virtclaimcase/system.toml"); + let case = config.parent().expect("system.toml has a directory"); + // One CPU, so each message is taken where the job wrote the register + // that sends it: at EL0, which holds the preempt count at zero. + let options = BootOptions { + profile, + smp: 1, + ready_marker: "control registers: SCTLR_EL1=", + extra_root_files: vec![suite_bin(profile.arch(), VIRT_CLAIM_LPI)], + ..Default::default() + }; + let argv = qemu::profile_argv(&options); + for want in ["its=on", "kernel-irqchip=off", "iommu=smmuv3", "edu"] { + if !argv.iter().any(|a| a.contains(want)) { + return Err(format!("the machine has no {want}: {argv:?}")); + } + } + let mut qemu = QemuInstance::boot_with_options(case, &[], &[], options); + let mut serial = virt_console(&qemu); + judge_virt_job(&mut qemu, &mut serial, JOB, SAID)?; + await_marker(&mut qemu, &mut serial, "unmapped from slot ", "the claim's DeviceID unmapped at its end")?; + let line = |want: &str| serial.lines().find(|l| l.contains(want)).map(str::to_string); + let Some(armed) = line("ITS: 0 at 0x8080000 armed") else { + return Err(format!("the MADT's ITS was never armed +serial: +{serial}")); + }; + eprintln!(" [virt] {armed}"); + let Some(handed) = line(&format!("{EDU} handed over on slot ")) else { + return Err(format!("edu was never handed over +serial: +{serial}")); + }; + eprintln!(" [virt] {handed}"); + // `pcidev: PCI bb:dd.f [1234:11e8] handed over on slot N, LPI M`. + let parsed = handed.split_once("pcidev: PCI ").and_then(|(_, rest)| { + let (at, rest) = rest.split_once(' ')?; + let (slot, lpi) = rest.split_once("on slot ")?.1.split_once(", LPI ")?; + Some((at.to_string(), slot.parse::().ok()?, lpi.trim().parse::().ok()?)) + }); + let Some((at, slot, lpi)) = parsed else { + return Err(format!("{handed:?} names no function, slot and LPI")); + }; + if lpi != 8192 + slot { + return Err(format!("slot {slot}'s LPI is {lpi}, not the {} its slot names", 8192 + slot)); + } + // QEMU's IORT maps a function's requester ID to the SMMUv3's StreamID and + // that to the ITS's DeviceID unchanged. + let (bus, rest) = at.split_once(':').ok_or_else(|| format!("{at} is no bus:device.function"))?; + let (dev, func) = rest.split_once('.').ok_or_else(|| format!("{at} is no bus:device.function"))?; + let hex = |text: &str| u32::from_str_radix(text, 16).map_err(|e| format!("{text}: {e}")); + let rid = hex(bus)? << 8 | hex(dev)? << 3 | hex(func)?; + for want in [ + format!("ITS: {at}, DeviceID {rid:#x}, event 0 is slot {slot}'s LPI {lpi}"), + format!("pcidev: slot {slot} took its first message on LPI {lpi}"), + format!("pcidev: PCI {at} {EDU} released from slot {slot}"), + format!("ITS: {at}, DeviceID {rid:#x}, unmapped from slot {slot}"), + ] { + let Some(found) = line(&want) else { + return Err(format!("the kernel never said {want:?} +serial: +{serial}")); + }; + eprintln!(" [virt] {found}"); + } + if let Some(fault) = line("DMA FAULT") { + return Err(format!("the unit refused an access: {fault} +serial: +{serial}")); + } + Ok(()) +} + /// Everything the PL011 has carried on `qemu`'s boot: the capture each /// [`judge_virt_job`] after the first goes on from, since a drain reads past /// the marker it waited for. @@ -2937,6 +3029,7 @@ fn run_screen_test(name: &str, profile: qemu::Profile, test_config: &Path) -> Re "virt_random_differs" => virt_random_differs(profile), "virt_no_seed_refused" => virt_no_seed_refused(profile, test_config), "virt_smmu" => virt_smmu(profile, test_config), + "virt_claim_lpi" => virt_claim_lpi(profile), "virt_mask_windows" => virt_mask_windows(profile), "virt_irq_storm" => { // The CPU floods itself with SGIs until the timer has fired a diff --git a/tests/virtclaimcase/system.toml b/tests/virtclaimcase/system.toml new file mode 100644 index 00000000000..952e566c1e8 --- /dev/null +++ b/tests/virtclaimcase/system.toml @@ -0,0 +1,15 @@ +# One job on `virt` with its ITS: the suite's `test_rs_claim_lpi`, which the +# harness puts on ROOT, claims QEMU's `edu` and reads the interrupts it raises. +[boot] +start = ["logkeeper", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] + +# `device` because the job mints the claim, `dup` because test-runner hands a +# job its capability as a duplicate and hands none without it. +[programs.test-runner] +receives = ["power"] +args = ["--bound-ms=240000", "test_rs_claim_lpi"] +syscap = ["device", "dup"] From b26a9cddbd1815933a925badfb8e427f26a12e48 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 16:10:40 +0200 Subject: [PATCH 2/3] A device leaf is no address the kernel copies or reports through; LPI tables outlive a late refusal; a claim's DeviceID is the SMMUv3's own route Review round 1 of #846. AArch64 paging: `AddressSpace::walk`, and so `translate` and `leaf`, and `read_user_word` answer nothing for a user leaf of Device type. A claimed function's BAR is mapped into its holder that way, and the kernel reached it through the direct map for a syscall's copy and for the crash report's backtrace and memory dump. `map_window_if_absent` asks the tables for presence directly, since a translation now refuses a leaf that is present. Closes issues/an-aarch64-crash-report-reads-through-any-user-leaf.md; its rule moves to paging.rs's module header. ITS bring-up: a failure found after the redistributor's EnableLPIs panics, rather than dropping pages GICR_PROPBASER and GICR_PENDBASER still name. GITS_CBASER, GICR_PROPBASER and GICR_PENDBASER are each read back whole (PENDBASER without its write-only PTZ, now `toyos_its::lpi::PENDBASER_PTZ`) and refused on a mismatch. A claim's DeviceID comes from the SMMUv3's routes, which now carry the ITS mapping of each routed stream, instead of a second read of the IORT: a function the unit would not put on a domain is refused before `attach`. Measured on QEMU 11.1.1: a function behind a plain pcie-root-port is routed through the unit; one behind a root port of a pxb-pcie with bypass_iommu=on is `Untranslated`. `virt_claim_lpi`'s machine gains such an e1000e; `claim_unrouted` is refused it and claims edu after, and `claim_bar_fault` faults twice in edu's BAR and names it as a read's and a write's buffer. Issues: the storm issue names each architecture's slot interrupt; the ARM track says what stops a released function's doorbell write and what test is owed for the ITS half. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- .../a-claimed-function-can-storm-its-cpu.md | 19 ++- ...rash-report-reads-through-any-user-leaf.md | 42 ------ issues/toyos-runs-on-arm64.md | 13 +- kernel/src/arch/aarch64/irqchip.rs | 2 +- kernel/src/arch/aarch64/irqchip/its.rs | 90 +++++++----- kernel/src/arch/aarch64/paging.rs | 23 ++- kernel/src/arch/aarch64/smmu/fault.rs | 4 +- kernel/src/arch/aarch64/smmu/mod.rs | 29 ++-- tests/common/qemu.rs | 11 +- tests/toyos-rust-tests/src/arch/stack.rs | 22 +++ .../src/bin/claim_bar_fault.rs | 132 ++++++++++++++++++ .../src/bin/claim_unrouted.rs | 22 +++ tests/toyos.rs | 79 ++++++++--- tests/virtclaimcase/system.toml | 9 +- toyos-its/src/lpi.rs | 9 +- 15 files changed, 378 insertions(+), 128 deletions(-) delete mode 100644 issues/an-aarch64-crash-report-reads-through-any-user-leaf.md create mode 100644 tests/toyos-rust-tests/src/bin/claim_bar_fault.rs create mode 100644 tests/toyos-rust-tests/src/bin/claim_unrouted.rs diff --git a/issues/a-claimed-function-can-storm-its-cpu.md b/issues/a-claimed-function-can-storm-its-cpu.md index 1f5ba0f3068..b00177f7f24 100644 --- a/issues/a-claimed-function-can-storm-its-cpu.md +++ b/issues/a-claimed-function-can-storm-its-cpu.md @@ -6,18 +6,23 @@ opened: 2026-10-04 # A claimed function can storm its CPU -A function a process drives raises `pcidev::VECTORS[slot]` on cpu0 for every -message it sends (`kernel/src/pcidev/mod.rs`, `isr`), and nothing bounds how -often. Its holder programs the device through the BARs it maps, so a hostile -or broken holder can make the device send messages as fast as the bus carries +A function a process drives raises its claim slot's interrupt on cpu0 for +every message it sends — on x86-64 the slot's vector +(`kernel/src/arch/x86_64/idt/user_dev.rs`), on AArch64 the slot's LPI through +the ITS (`kernel/src/arch/aarch64/irqchip/its.rs`) — and `pcidev::isr` +(`kernel/src/pcidev/mod.rs`) takes each one; nothing bounds how often. Its +holder programs the device through the BARs it maps, so a hostile or broken +holder can make the device send messages as fast as the bus carries them, and each one is an interrupt cpu0 takes before anything else it runs. `issues/an-xhci-storm-starves-the-cpu-that-takes-it.md` measured what that does to a CPU for a kernel driver's source; a claim is the same source with its holder outside the kernel. -The lever is the slot's own remapping entry: making it not present stops the -function's messages at the unit, where masking through the device would need -the holder's cooperation. What rate is a storm is not decided, and a number +The lever is the slot's own translation: on x86-64 its remapping entry, which +made not present stops the function's messages at the unit; on AArch64 its +LPI's configuration byte, which cleared and followed by an `INV` disables +the LPI at the redistributor. Masking through the device would need the +holder's cooperation. What rate is a storm is not decided, and a number chosen without a measurement behind it is the silent decision this file exists to refuse. diff --git a/issues/an-aarch64-crash-report-reads-through-any-user-leaf.md b/issues/an-aarch64-crash-report-reads-through-any-user-leaf.md deleted file mode 100644 index 49e1dea5844..00000000000 --- a/issues/an-aarch64-crash-report-reads-through-any-user-leaf.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# An AArch64 crash report reads through any user leaf - -`kernel/src/arch/aarch64/paging.rs`'s `read_user_word`, which the report -of an EL0 fault calls with the faulting thread's own `x29` to walk its -frames (`trap.rs`'s `user_backtrace`), reads the frame any valid user leaf -names through the direct map. The direct map holds memory and nothing -else, so a leaf naming a device's registers — a claimed function's BAR, -once the port's stage 6 maps one into a process — names an address the -direct map does not hold, and the report's read of it is an EL1 data abort: -a user fault with `x29` pointed into its own BAR ends the machine. - -Two more readers take the same leaf to the direct map: - -- **Every syscall's user copy.** `AddressSpace::leaf` answers a Device - leaf as it answers a Normal one, for a `Write` too where the leaf is - EL0 read-write, and `kernel/src/user_ptr.rs` copies through - `DirectMap::from_phys` of what it answered: `translate_now`, `object_run` - (`copy_in`, `copy_out`) and `View::pieces` (`UserBytes::read_at`, - `UserBytesMut::write_at` and their kin). A `read(fd, bar, n)` is an EL1 - abort on an address the direct map does not hold, or, where `map_mmio` - holds it Device-nGnRE, an alignment fault on `memcpy`'s unaligned access: - a kernel panic a process chose. -- **The fault dump.** `kernel/src/process.rs`'s `dump_crash_diagnostics`, - which AArch64's `trap.rs` calls on an EL0 fault, reads the words around - the fault address and the faulting PC through `translate` and the direct - map, the same way. - -Nothing maps a BAR into an AArch64 process yet — `arch::msi_message` -refuses there, so pcidev refuses every hand-over — so this is latent until -stage 6 of `issues/toyos-runs-on-arm64.md`. - -**Exit condition**: `read_user_word`, `leaf` and the fault dump's -`translate` answer only a leaf of the memory type the direct map holds, and -guest tests see a process end and the kernel live when it faults with `x29` -inside a mapped BAR, faults at an address inside one, and names one as a -`read`'s buffer — the last refused with `BadAddress`. diff --git a/issues/toyos-runs-on-arm64.md b/issues/toyos-runs-on-arm64.md index 14d90d92463..24a0e2d5990 100644 --- a/issues/toyos-runs-on-arm64.md +++ b/issues/toyos-runs-on-arm64.md @@ -375,9 +375,16 @@ Each stage names its exit; "measured" means a number from a run. (`virt_claim_lpi`, red without it on a `DMA FAULT` at `GITS_TRANSLATER`). `irq()` holds the preempt count across every device arm, as x86-64's `device_irq_entry` does, and `pcidev::isr` asserts it. - Not yet tested: that a release's `DISCARD` and `MAPD` with `V` clear - leave the function's DeviceID translating nothing, which no function - QEMU offers can show, since a released function's MSI is disabled. + A released function's stream stays on its slot's domain, which maps + the doorbell; its bus mastering off and its message masked are what + stop its writes, and a write it makes regardless reaches the ITS, where + the release's `DISCARD` and `MAPD` with `V` clear leave its DeviceID + translating nothing. That last is read, not tested: no function QEMU + offers writes once mastering is off but `iommu-testdev`, which no claim + can tell to write. **Owed**: a `smmu-selftest` arm that puts an + `iommu-testdev` on a domain, maps its DeviceID to a slot and unmaps it as + a release does, has it write event 0 to `GITS_TRANSLATER`, and is red + when the slot's LPI is raised. 7. **Userland boots.** `init`, `logd`, the compositor, netd, soundd and sshd, built for `aarch64-unknown-toyos`. Every program of `system.toml` builds diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 9fcf1627f81..3f2e3709138 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -205,7 +205,7 @@ pub fn init(rsdp_addr: u64) -> Gic { let gic = Gic { cpus, ranges }; let frame = gic.redistributor(cpu::hardware_id()); init_cpu(frame); - its::init(&translators, frame, rsdp_addr); + its::init(&translators, frame); gic } diff --git a/kernel/src/arch/aarch64/irqchip/its.rs b/kernel/src/arch/aarch64/irqchip/its.rs index e1047ca294f..0de1b110714 100644 --- a/kernel/src/arch/aarch64/irqchip/its.rs +++ b/kernel/src/arch/aarch64/irqchip/its.rs @@ -19,19 +19,22 @@ //! redistributor's `EnableLPIs`, so no `INV` is owed (§5.1.1). //! //! An ITS this kernel cannot drive is left disabled, and every claim is -//! refused on its account ([`is_armed`]); a command the ITS refuses, or does -//! not consume within [`TIMEOUT_PER_SECOND`]'s bound, is a kernel defect, and -//! panics. +//! refused on its account ([`is_armed`]); a refusal found once the +//! redistributor's `EnableLPIs` is set, a command the ITS refuses, or one it +//! does not consume within [`TIMEOUT_PER_SECOND`]'s bound, is a kernel +//! defect, and panics. +//! +//! **A claim's DeviceID is the one the SMMUv3's own routes carry** +//! (`super::super::smmu::its_device`): a function the unit would not put on a +//! domain is refused before its claim reaches the unit. use alloc::vec::Vec; -use toyos_acpi::Route; use toyos_its::command::Command; use toyos_its::lpi::{self, Layout, Lpi}; use toyos_its::{Collection, CommandQueue, EventBits, Its, Table, Target}; use toyos_phys::Phys; -use crate::drivers::acpi::direct_phys; use crate::iommu::StreamId; use crate::log; use crate::mm::pmm::{self, PhysPage}; @@ -104,8 +107,6 @@ struct Live { held: [Option; MAX_FUNCTIONS], /// Each slot's LPI. lpis: [Lpi; MAX_FUNCTIONS], - /// Where the IORT is read from at each claim. - rsdp: u64, /// The pages every table is in: never given back, since the ITS and the /// redistributor may read any of them for the machine's life. _pages: Vec, @@ -184,7 +185,7 @@ pub(in crate::arch::aarch64) fn slot_of(intid: u32) -> Option { /// Bring the one ITS the MADT names up, on the boot CPU's redistributor at /// `frame`, once that CPU's GIC is, from the `(id, base)` of every ITS the /// MADT names: every refusal is logged and leaves no claim a message. -pub(super) fn init(named: &[(u32, u64)], frame: u64, rsdp: u64) { +pub(super) fn init(named: &[(u32, u64)], frame: u64) { let (id, base) = match *named { [] => { log!("ITS: the MADT names none, so no claimed function is given an interrupt"); @@ -196,7 +197,7 @@ pub(super) fn init(named: &[(u32, u64)], frame: u64, rsdp: u64) { return; } }; - match bring_up(id, base, frame, rsdp) { + match bring_up(id, base, frame) { Ok(live) => { log!( "ITS: {id} at {base:#x} armed: doorbell {:#x}, collection 0 on the boot CPU's redistributor at {frame:#x}, \ @@ -210,7 +211,7 @@ pub(super) fn init(named: &[(u32, u64)], frame: u64, rsdp: u64) { } } -fn bring_up(id: u32, base: u64, frame: u64, rsdp: u64) -> Result { +fn bring_up(id: u32, base: u64, frame: u64) -> Result { use alloc::format; let doorbell = base + FRAME + toyos_its::GITS_TRANSLATER; let doorbell = u32::try_from(doorbell) @@ -287,14 +288,25 @@ fn bring_up(id: u32, base: u64, frame: u64, rsdp: u64) -> Result::new(queue_at).expect("a 64 KiB-aligned page below 2^48"); - regs.write_u64(toyos_its::GITS_CBASER as u64, commands.cbaser(queue_phys)); + let cbaser = commands.cbaser(queue_phys); + regs.write_u64(toyos_its::GITS_CBASER as u64, cbaser); + let read = regs.read_u64(toyos_its::GITS_CBASER as u64); + if read != cbaser { + return Err(format!("GITS_CBASER reads {read:#x} after {cbaser:#x} was written")); + } regs.write_u64(toyos_its::GITS_CWRITER as u64, 0); let propbaser = layout.propbaser(Phys::new(configuration).expect("a 4 KiB-aligned page below 2^48")); rd.write_u64(lpi::GICR_PROPBASER as u64, propbaser); let read = rd.read_u64(lpi::GICR_PROPBASER as u64); - let space = layout.space(read).ok_or_else(|| format!("GICR_PROPBASER reads {read:#x} after {propbaser:#x} was written"))?; + let space = layout + .space(read) + .filter(|_| read == propbaser) + .ok_or_else(|| format!("GICR_PROPBASER reads {read:#x} after {propbaser:#x} was written"))?; // Every slot's LPI enabled, before the redistributor reads the table. let lpis: [Lpi; MAX_FUNCTIONS] = core::array::from_fn(|slot| { space.lpi(lpi::FIRST + slot as u32).expect("a claim slot's LPI is among the 8192 the layout holds") @@ -303,16 +315,27 @@ fn bring_up(id: u32, base: u64, frame: u64, rsdp: u64) -> Result Result write!( f, "the IORT gives it no DeviceID at this kernel's ITS behind the SMMUv3, so no \ - table of its own would translate its message" + table of its own would translate its message and no domain of its own would \ + confine its writes" ), Self::DeviceIdTooWide(id) => write!(f, "its DeviceID {id:#x} is past the ITS's device table"), Self::DeviceIdHeld(id) => write!( @@ -397,18 +420,13 @@ pub fn msi(_source: StreamId, _irq: crate::arch::DriverIrq) -> Result Result { - let iort = toyos_acpi::iort(direct_phys(), live.rsdp).map_err(|_| Refused::NoDeviceId)?; - let segment = u32::from(crate::pcidev::segment()); - match iort.route(segment, source.requester()) { - Ok(Route::Translated { its: Some(device), .. }) if device.its == live.id => { - if device.device >> DEVICE_BITS == 0 { - Ok(device.device) - } else { - Err(Refused::DeviceIdTooWide(device.device)) - } - } +/// The DeviceID `routed` names at this ITS: what the SMMUv3's own routes +/// map the function's stream on to, so a function the unit would not put on +/// a domain is refused here, before its claim reaches `attach`. +fn device_of(live: &Live, routed: Option) -> Result { + match routed { + Some(device) if device.its == live.id && device.device >> DEVICE_BITS == 0 => Ok(device.device), + Some(device) if device.its == live.id => Err(Refused::DeviceIdTooWide(device.device)), _ => Err(Refused::NoDeviceId), } } @@ -416,10 +434,12 @@ fn device_of(live: &Live, source: StreamId) -> Result { /// Map claim slot `slot`'s translation table to `source`'s DeviceID, its one /// event to the slot's LPI, and answer the message that raises it. pub fn claim(slot: usize, source: StreamId) -> Result { + // Before this lock, so it never nests with the unit's. + let routed = super::super::smmu::its_device(source); let mut held = ITS.lock(); let live = held.as_mut().unwrap_or_else(|| panic!("ITS: claim slot {slot} mapped with no ITS armed")); assert!(live.held[slot].is_none(), "ITS: claim slot {slot} is still mapped"); - let device = device_of(live, source)?; + let device = device_of(live, routed)?; if live.held.contains(&Some(device)) { return Err(Refused::DeviceIdHeld(device)); } diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 2943851f5ee..2e91a0993cf 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -14,7 +14,9 @@ //! Normal write-back; a device's registers only as [`map_mmio`] maps them; and //! the scanout. A page is never retyped: a mapping that disagrees with the one //! already there is refused by name, because one page under two memory types -//! loses coherency (D8.2.12). +//! loses coherency (D8.2.12). So the kernel reads and writes a user page +//! through the direct map only where it is memory: a user leaf of a device's +//! registers answers no translation, no syscall's copy and no crash report. //! //! **A live entry is replaced break-before-make**: written invalid, its //! translation dropped on every CPU (`super::tlb`), and only then written @@ -65,6 +67,14 @@ fn typed(cache: CachePolicy) -> u64 { } } +/// Whether the frame a user leaf maps is one the kernel may reach through the +/// direct map: not a device's registers, which the direct map holds only +/// where [`map_mmio`] mapped them, and then as a type `memcpy`'s unaligned +/// access faults on. +fn through_direct_map(leaf: u64) -> bool { + policy_of(leaf) != CachePolicy::Uncacheable +} + /// The type a leaf this file wrote names; any other index is one it never wrote. fn policy_of(leaf: u64) -> CachePolicy { match (leaf & ATTR_INDEX) >> 2 { @@ -432,7 +442,7 @@ impl AddressSpace { /// `false` leaves the mapping as found and `phys` the caller's to free: /// the check and the write are one critical section under the caller's lock. pub fn map_window_if_absent(&mut self, vaddr: UserAddr, phys: u64, prot: &WindowProt) -> bool { - if self.translate(vaddr).is_some() { + if self.tables.leaf(vaddr.raw()).is_some() { return false; } self.map_window(vaddr, phys, prot); @@ -465,7 +475,9 @@ impl AddressSpace { crate::sched::futex::revoke_range(phys, PAGE_2M); } - /// Checked here, not at the callers: only a user address names user memory. + /// Checked here, not at the callers: only a user address names user + /// memory, and a device's registers mapped there are no address the + /// direct map answers for. pub fn translate(&self, vaddr: UserAddr) -> Option { self.walk(vaddr).map(|(at, _, _)| at) } @@ -491,7 +503,7 @@ impl AddressSpace { if !toyos_userbound::is_user_addr(va) { return None; } - let (leaf, size) = self.tables.leaf(va)?; + let (leaf, size) = self.tables.leaf(va).filter(|&(leaf, _)| through_direct_map(leaf))?; let base = leaf & if size == PAGE_2M { ADDR_2M } else { ADDR }; Some((DirectMap::from_phys(base + (va & (size - 1))), leaf, size)) } @@ -859,6 +871,9 @@ pub(super) fn read_user_word(addr: u64) -> Option { } (_, false) => return None, }; + if !through_direct_map(entry) { + return None; + } let base = entry & if size == PAGE_2M { ADDR_2M } else { ADDR }; // SAFETY: a byte of a frame a valid leaf maps, reached through the direct map. return Some(unsafe { core::ptr::read_volatile(DirectMap::from_phys(base + (addr & (size - 1))).as_ptr::()) }); diff --git a/kernel/src/arch/aarch64/smmu/fault.rs b/kernel/src/arch/aarch64/smmu/fault.rs index 995602fd0f9..6e0c1d4004f 100644 --- a/kernel/src/arch/aarch64/smmu/fault.rs +++ b/kernel/src/arch/aarch64/smmu/fault.rs @@ -47,9 +47,9 @@ fn armed() -> Option<&'static Armed> { /// Everything the handler reads, before the unit's event interrupt is /// enabled: the registers, the queue and its indexes, and the routes. -pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, routes: &[(StreamId, u32)]) { +pub(super) fn arm(regs: Registers, queue: Mmio, events: Events, routes: impl Iterator) { let armed = - Box::new(Armed { regs, queue, events, routes: routes.into(), faults: AtomicU32::new(0) }); + Box::new(Armed { regs, queue, events, routes: routes.collect(), faults: AtomicU32::new(0) }); let first = ARMED.compare_exchange(core::ptr::null_mut(), Box::leak(armed), Ordering::Release, Ordering::Relaxed); assert!(first.is_ok(), "SMMU: the event handler was armed twice"); } diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index 6bffe4ff4cb..dea774b8646 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -30,7 +30,7 @@ mod selftest; use alloc::vec::Vec; -use toyos_acpi::{IortRefused, Node, Route, TableError}; +use toyos_acpi::{IortRefused, ItsDevice, Node, Route, TableError}; use toyos_phys::Phys; use toyos_smmu::config::Ste; use toyos_smmu::queue::{Command, Commands, Signal}; @@ -119,9 +119,10 @@ struct Live { prod: u32, streams: Mmio, memory: Memory, - /// Each enumerated function the IORT routes through this unit, and its - /// StreamID. - routes: Vec<(StreamId, u32)>, + /// Each enumerated function the IORT routes through this unit, its + /// StreamID, and the DeviceID the unit's own node maps that stream on to + /// at an ITS. + routes: Vec<(StreamId, u32, Option)>, domains: Vec, /// What no domain's addresses may reach, as `(start, end)`. reserved: Vec<(u64, u64)>, @@ -148,7 +149,7 @@ impl Live { /// The StreamID the IORT gives `function`'s requests through this unit. fn stream(&self, function: StreamId) -> u32 { - let route = self.routes.iter().find(|(rid, _)| *rid == function); + let route = self.routes.iter().find(|(rid, ..)| *rid == function); route.unwrap_or_else(|| panic!("SMMU: {function} is no function the IORT routes through this unit")).1 } @@ -264,7 +265,7 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::R { log!("smmu-selftest: {function}, StreamID {stream:#x}, is given no route, as a function never enumerated is not"); } - Ok(Route::Translated { smmu: by, stream, .. }) if by.base == smmu.base => routes.push((function, stream)), + Ok(Route::Translated { smmu: by, stream, its }) if by.base == smmu.base => routes.push((function, stream, its)), Ok(route) => log!("IOMMU: {function} is not routed through the SMMUv3: {route:?}"), Err(why) => log!("IOMMU: {function}'s IORT route is refused: {why:?}"), } @@ -280,6 +281,14 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::R } } +/// The DeviceID at an ITS that `function`'s messages reach it by, read off +/// the routes [`domain::attach`] puts a stream on a domain by: `None` for a +/// function this unit does not translate, whose stream is on no domain, and +/// for one whose stream the IORT maps on to no ITS. +pub fn its_device(function: StreamId) -> Option { + UNIT.lock().as_ref()?.routes.iter().find(|(rid, ..)| *rid == function)?.2 +} + /// `GBPA` aborting, the unit disabled and none of its interrupts able to /// write: from here until `SMMUEN`, and for good if it is never set, every /// transaction aborts. @@ -318,7 +327,7 @@ fn program( base: u64, coherent_override: bool, event: Option, - routes: Vec<(StreamId, u32)>, + routes: Vec<(StreamId, u32, Option)>, reserved: Vec<(u64, u64)>, ) -> Option { let refused = |why: core::fmt::Arguments<'_>| log!("IOMMU: the SMMUv3 at {base:#x} is left aborting: {why}"); @@ -342,7 +351,7 @@ fn program( refused(format_args!("the IORT gives its event queue no wired interrupt")); return None; }; - let highest = routes.iter().map(|(_, stream)| *stream).max().unwrap_or(0); + let highest = routes.iter().map(|(_, stream, _)| *stream).max().unwrap_or(0); let log2 = u32::BITS - highest.leading_zeros(); if log2 > STREAMS_LOG2_MAX || log2 > u32::from(unit.stream_bits) { refused(format_args!("StreamID {highest:#x} needs a table of 2^{log2} entries")); @@ -358,7 +367,7 @@ fn program( let streams = window(table, 64 << log2); // Every other entry stays zero, invalid: a stream no function is routed // from is recorded, as one past the table is. - for (_, stream) in &routes { + for (_, stream, _) in &routes { for (i, word) in Ste::ABORT.words().iter().enumerate() { streams.write_u64(u64::from(*stream) * 64 + 8 * i as u64, *word); } @@ -400,7 +409,7 @@ fn program( regs.write64(reg::EVENTQ_BASE, events_base); regs.write(reg::EVENTQ_PROD, 0); regs.write(reg::EVENTQ_CONS, 0); - fault::arm(regs, window(events_at, 32 << events_log2), events, &live.routes); + fault::arm(regs, window(events_at, 32 << events_log2), events, live.routes.iter().map(|&(function, stream, _)| (function, stream))); regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN, "its event queue enabled"); regs.control(reg::IRQ_CTRL, reg::IRQ_EVENTQ, "its event interrupt enabled"); regs.control(reg::CR0, reg::CR0_CMDQEN | reg::CR0_EVENTQEN | reg::CR0_SMMUEN, "SMMUEN"); diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 978b6c61e9c..ba9313d30fe 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -737,7 +737,8 @@ pub enum Profile { /// function that writes where it is told to through the unit. VirtSmmu, /// [`Profile::Virt`] with its SMMUv3, QEMU's emulated GICv3 and its ITS, - /// and QEMU's `edu`, a function that sends an MSI when told to. + /// QEMU's `edu`, a function that sends an MSI when told to, and an + /// `e1000e` the IORT routes past the SMMUv3. VirtIts, } @@ -900,7 +901,9 @@ enum Smmu { WithTestdev, /// With the ITS behind it, which needs QEMU's own GIC: HVF's refuses one. /// With it comes one `edu`, whose message is the only one a guest of - /// this suite can raise on demand. + /// this suite can raise on demand, and an `e1000e` behind a root port of + /// a `pxb-pcie` with `bypass_iommu=on`, which QEMU's IORT routes straight + /// to the ITS: a plain root port's buses it routes through the unit. WithIts, } @@ -2287,6 +2290,10 @@ fn qemu_command( } if shape.smmu == Smmu::WithIts { qemu.arg("-device").arg("edu"); + qemu.arg("-device").arg("pxb-pcie,id=bypass,bus_nr=8,bypass_iommu=on"); + qemu.arg("-device").arg("pcie-root-port,id=bypassport,bus=bypass,chassis=1"); + // No option ROM: firmware has no use for the NIC. + qemu.arg("-device").arg("e1000e,bus=bypassport,romfile="); } // The NIC before the virtio block, so a profile that has one and not the diff --git a/tests/toyos-rust-tests/src/arch/stack.rs b/tests/toyos-rust-tests/src/arch/stack.rs index dce07fb2e57..9d486da1747 100644 --- a/tests/toyos-rust-tests/src/arch/stack.rs +++ b/tests/toyos-rust-tests/src/arch/stack.rs @@ -20,3 +20,25 @@ mod x86_64 { } } } + +#[cfg(target_arch = "aarch64")] +pub use aarch64::*; + +#[cfg(target_arch = "aarch64")] +mod aarch64 { + /// An undefined instruction with `sp` and the frame pointer `x29` set + /// first, so the fault's report meets both. + pub fn undefined_with_stack_at(sp: u64, fp: u64) -> ! { + // SAFETY: none — the fault is the point, and it ends this process. + unsafe { + core::arch::asm!( + "mov sp, {s}", + "mov x29, {f}", + "udf #0", + s = in(reg) sp, + f = in(reg) fp, + options(noreturn), + ); + } + } +} diff --git a/tests/toyos-rust-tests/src/bin/claim_bar_fault.rs b/tests/toyos-rust-tests/src/bin/claim_bar_fault.rs new file mode 100644 index 00000000000..98a5a563326 --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/claim_bar_fault.rs @@ -0,0 +1,132 @@ +//! A claimed function's BAR, mapped into its holder, is no memory the kernel +//! reads or writes for that holder: a fault whose frame pointer is in it, and +//! a fault at an address in it, each end their process with the kernel's +//! crash report reading nothing there, and a syscall naming it as a buffer is +//! refused `BadAddress` with nothing read or written. +//! +//! The function is QEMU's `edu`, whose DMA source and destination registers +//! (`docs/specs/edu.rst`) are eight bytes each that keep what is written to +//! them across claims, since nothing resets it. This job writes one marker in +//! each and never reads them into a register a fault could report: the +//! harness reds on either marker anywhere on the console, which is where a +//! report that read through the BAR would put it. + +#[path = "../arch/stack.rs"] +mod stack; + +use std::os::toyos::process::CommandExt; +use std::process::Command; + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::shm::SharedMemory; +use toyos::syscap::SysCap; +use toyos::volatile::Window; +use toyos::PciDev; +use toyos_abi::syscall::{self, PciId, SyscallError}; + +const SELF_PATH: &str = "/system/bin/test_rs_claim_bar_fault"; + +/// `hw/misc/edu.c`: QEMU's educational device. +const EDU: PciId = PciId { vendor: 0x1234, device: 0x11e8 }; + +/// The DMA source register, and the destination after it: a frame pointer +/// here names a saved frame pointer of `MARK_SOURCE` and a return address of +/// `MARK_DESTINATION`. +const DMA_SOURCE: usize = 0x80; +const DMA_DESTINATION: usize = 0x88; + +/// What the two registers hold while the children fault, which +/// `tests/toyos.rs`'s `virt_claim_lpi` reds on: neither a user address nor +/// aligned, so a report that read one as a frame pointer stops there. +const MARK_SOURCE: u64 = 0xbad0_c0de_0000_5eed; +const MARK_DESTINATION: u64 = 0xbad1_c0de_0000_5eed; + +fn main() { + let role = std::env::args().nth(1); + let Some(role) = role.as_deref() else { return holder() }; + // Both held across the fault, which never returns. + let dev = claim(&endowed()); + let (bar, _) = mapped_registers(&dev); + let at = bar.as_ptr() as u64 + DMA_SOURCE as u64; + match role { + "frame" => stack::undefined_with_stack_at(at, at), + "jump" => { + // SAFETY: none — the fetch from a BAR mapped never executable is + // the fault, and it ends this process. + let into: extern "C" fn() -> ! = unsafe { core::mem::transmute(at as usize) }; + into() + } + other => panic!("claim_bar_fault: unknown role {other:?}"), + } +} + +fn endowed() -> SysCap { + Endowments::get().take(SYSCAP_LABEL).expect("claim_bar_fault: test-runner endows a device-minting capability") +} + +fn claim(cap: &SysCap) -> PciDev { + cap.claim_pci(EDU).unwrap_or_else(|why| panic!("claim_bar_fault: the claim on edu was refused: {why:?}")) +} + +/// BAR 0 mapped, and its registers. +fn mapped_registers(dev: &PciDev) -> (SharedMemory, Window) { + let info = dev.describe().expect("claim_bar_fault: the claim's description"); + let bar = dev.map_bar(0, info.bar_bytes[0]).expect("claim_bar_fault: BAR 0"); + // SAFETY: the mapping is BAR 0's length and lives as long as `bar`, which + // every caller holds as long as the window. + let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; + (bar, regs) +} + +fn holder() { + let cap = endowed(); + { + let dev = claim(&cap); + let (bar, regs) = mapped_registers(&dev); + let at = bar.as_ptr() as u64 + DMA_SOURCE as u64; + regs.write::(DMA_SOURCE, MARK_SOURCE); + regs.write::(DMA_DESTINATION, MARK_DESTINATION); + + let pipe = syscall::pipe().expect("claim_bar_fault: a pipe"); + assert_eq!(syscall::write(pipe.write, &[0x5a; 16]), Ok(16), "claim_bar_fault: the pipe took no bytes"); + // SAFETY: the sixteen bytes are the two registers, inside the live + // mapping; the slice only carries their address into the syscall. + let registers = unsafe { core::slice::from_raw_parts_mut(at as *mut u8, 16) }; + assert_eq!( + syscall::read(pipe.read, registers), + Err(SyscallError::BadAddress), + "claim_bar_fault: a read into the BAR was not refused BadAddress" + ); + assert_eq!( + syscall::write(pipe.write, registers), + Err(SyscallError::BadAddress), + "claim_bar_fault: a write from the BAR was not refused BadAddress" + ); + assert!( + regs.read::(DMA_SOURCE) == MARK_SOURCE && regs.read::(DMA_DESTINATION) == MARK_DESTINATION, + "claim_bar_fault: the refused read wrote into the BAR" + ); + syscall::close(pipe.read); + syscall::close(pipe.write); + println!("claim_bar_fault: a read into the BAR and a write from it were refused BadAddress, the registers untouched"); + } + + for role in ["frame", "jump"] { + let status = Command::new(SELF_PATH) + .arg(role) + .endow(SYSCAP_LABEL, cap.duplicate().expect("claim_bar_fault: a capability for the child").into_raw().0) + .status() + .expect("claim_bar_fault: spawn a child"); + assert_eq!(status.code(), Some(-1), "claim_bar_fault: the {role} child ended {status:?}, not by its fault"); + println!("claim_bar_fault: the {role} child's fault in the BAR ended it"); + } + + // The markers were there for both reports to read. + let dev = claim(&cap); + let (_bar, regs) = mapped_registers(&dev); + assert!( + regs.read::(DMA_SOURCE) == MARK_SOURCE && regs.read::(DMA_DESTINATION) == MARK_DESTINATION, + "claim_bar_fault: the registers lost their markers while the children faulted" + ); + println!("claim_bar_fault: two faults in the BAR ended their processes, and a syscall naming it was refused"); +} diff --git a/tests/toyos-rust-tests/src/bin/claim_unrouted.rs b/tests/toyos-rust-tests/src/bin/claim_unrouted.rs new file mode 100644 index 00000000000..768af6b1f2c --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/claim_unrouted.rs @@ -0,0 +1,22 @@ +//! A function the IORT routes past the SMMUv3 — QEMU's `e1000e`, behind a +//! root port of a `pxb-pcie` with `bypass_iommu=on` — is refused its claim, +//! and the kernel that refused it goes on: the next claim, of `edu`, is +//! handed over. + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::syscap::SysCap; +use toyos::PciDev; +use toyos_abi::syscall::{PciId, SyscallError}; + +/// QEMU's `e1000e`, an 82574L. +const E1000E: PciId = PciId { vendor: 0x8086, device: 0x10d3 }; +/// `hw/misc/edu.c`: QEMU's educational device. +const EDU: PciId = PciId { vendor: 0x1234, device: 0x11e8 }; + +fn main() { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability"); + let refused = cap.claim_pci::(E1000E).map(|_| ()); + assert_eq!(refused, Err(SyscallError::NotSupported), "claim_unrouted: the claim on e1000e was not refused as unusable"); + let _edu: PciDev = cap.claim_pci(EDU).unwrap_or_else(|why| panic!("claim_unrouted: edu, claimed after, was refused: {why:?}")); + println!("claim_unrouted: e1000e, routed past the SMMUv3, was refused, and edu was claimed after"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 7fe81c24378..5232bdc3668 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -114,8 +114,11 @@ const RUST_SKIP: &[&str] = &[ "trace_flood", // It takes the machine down; `virt_fatal_halts_the_others_first` runs it. "panic_halts_first", - // It claims QEMU's `edu`, which only `virt_claim_lpi`'s machine has. + // They claim QEMU's `edu` and `e1000e`, which only `virt_claim_lpi`'s + // machine has. "claim_lpi", + "claim_unrouted", + "claim_bar_fault", // Needs a launcher and a declared `cat` and shell, which `tests/testcases` // does not give: the `process_tree` metal row runs it on tests/proctreecase. "process_tree", @@ -2209,18 +2212,29 @@ fn virt_smmu(profile: qemu::Profile, test_config: &Path) -> Result<(), String> { Ok(()) } -/// `tests/toyos-rust-tests`' binary that `tests/virtclaimcase` runs as its job -/// `test_rs_claim_lpi`. +/// `tests/toyos-rust-tests`' binaries that `tests/virtclaimcase` runs as its +/// jobs, each `test_rs_` and its name. const VIRT_CLAIM_LPI: &str = "claim_lpi"; +const VIRT_CLAIM_UNROUTED: &str = "claim_unrouted"; +const VIRT_CLAIM_BAR_FAULT: &str = "claim_bar_fault"; + +/// What `claim_bar_fault` keeps in edu's two DMA address registers while its +/// children fault in the BAR (`MARK_SOURCE`, `MARK_DESTINATION` there): on +/// the console only if the kernel read through the BAR for a crash report. +const BAR_MARKS: [u64; 2] = [0xbad0_c0de_0000_5eed, 0xbad1_c0de_0000_5eed]; /// A claimed function's message, translated by the ITS into its claim slot's -/// LPI: `tests/virtclaimcase`'s one job claims QEMU's `edu` and reads every -/// message it asks it for as one interrupt of its claim -/// (`tests/toyos-rust-tests/src/bin/claim_lpi.rs`). Judged beside it on the -/// kernel's lines: the ITS armed from the MADT; the function handed over on -/// a slot, its DeviceID the requester ID QEMU's IORT maps it to, its slot's -/// LPI raised; and at the claim's end the DeviceID unmapped again, and no -/// access of the function's refused at the SMMUv3. +/// LPI, and the claims the kernel refuses beside it, on `tests/virtclaimcase`. +/// Its first job claims QEMU's `edu` and reads every message it asks it for as +/// one interrupt of its claim (`tests/toyos-rust-tests/src/bin/claim_lpi.rs`), +/// judged beside it on the kernel's lines: the ITS armed from the MADT; the +/// function handed over on a slot, its DeviceID the requester ID QEMU's IORT +/// maps it to, its slot's LPI raised; and at the claim's end the DeviceID +/// unmapped again. The second is refused the `e1000e` the IORT routes past +/// the SMMUv3, by the kernel's name for it, and claims edu after it. The +/// third faults twice in edu's BAR, and names it as a syscall's buffer; no +/// crash report reads through it. And no access of a function's is refused +/// at the SMMUv3. fn virt_claim_lpi(profile: qemu::Profile) -> Result<(), String> { const JOB: &str = "test_rs_claim_lpi"; const SAID: &str = "claim_lpi: 16 messages edu sent were read as 16 interrupts of its claim, one each"; @@ -2233,11 +2247,15 @@ fn virt_claim_lpi(profile: qemu::Profile) -> Result<(), String> { profile, smp: 1, ready_marker: "control registers: SCTLR_EL1=", - extra_root_files: vec![suite_bin(profile.arch(), VIRT_CLAIM_LPI)], + extra_root_files: vec![ + suite_bin(profile.arch(), VIRT_CLAIM_LPI), + suite_bin(profile.arch(), VIRT_CLAIM_UNROUTED), + suite_bin(profile.arch(), VIRT_CLAIM_BAR_FAULT), + ], ..Default::default() }; let argv = qemu::profile_argv(&options); - for want in ["its=on", "kernel-irqchip=off", "iommu=smmuv3", "edu"] { + for want in ["its=on", "kernel-irqchip=off", "iommu=smmuv3", "edu", "bypass_iommu=on", "e1000e"] { if !argv.iter().any(|a| a.contains(want)) { return Err(format!("the machine has no {want}: {argv:?}")); } @@ -2290,10 +2308,39 @@ serial: }; eprintln!(" [virt] {found}"); } - if let Some(fault) = line("DMA FAULT") { - return Err(format!("the unit refused an access: {fault} -serial: -{serial}")); + + judge_virt_job( + &mut qemu, + &mut serial, + "test_rs_claim_unrouted", + "claim_unrouted: e1000e, routed past the SMMUv3, was refused, and edu was claimed after", + )?; + // `pcidev::Refusal::NoMessage` of the ITS's `Refused::NoDeviceId`. + const UNROUTED: &str = "NOT HANDED OVER — no message of its own could be made for it — the IORT gives it no \ + DeviceID at this kernel's ITS behind the SMMUv3"; + let Some(refused) = serial.lines().find(|l| l.contains(UNROUTED)) else { + return Err(format!("the kernel never refused e1000e by {UNROUTED:?}\nserial:\n{serial}")); + }; + eprintln!(" [virt] {refused}"); + + judge_virt_job( + &mut qemu, + &mut serial, + "test_rs_claim_bar_fault", + "claim_bar_fault: two faults in the BAR ended their processes, and a syscall naming it was refused", + )?; + let faults: Vec<&str> = serial.lines().filter(|l| l.contains("FAULT pc=")).collect(); + if faults.len() != 2 { + return Err(format!("{} user faults reported, not claim_bar_fault's two: {faults:?}\nserial:\n{serial}", faults.len())); + } + let console = serial.to_lowercase(); + for mark in BAR_MARKS { + if console.contains(&format!("{mark:x}")) { + return Err(format!("{mark:#x}, in edu's BAR, is on the console: a crash report read through it\nserial:\n{serial}")); + } + } + if let Some(fault) = serial.lines().find(|l| l.contains("DMA FAULT")) { + return Err(format!("the unit refused an access: {fault}\nserial:\n{serial}")); } Ok(()) } diff --git a/tests/virtclaimcase/system.toml b/tests/virtclaimcase/system.toml index 952e566c1e8..62d54be0318 100644 --- a/tests/virtclaimcase/system.toml +++ b/tests/virtclaimcase/system.toml @@ -1,5 +1,8 @@ -# One job on `virt` with its ITS: the suite's `test_rs_claim_lpi`, which the -# harness puts on ROOT, claims QEMU's `edu` and reads the interrupts it raises. +# Three jobs on `virt` with its ITS, each one of the suite's, which the harness +# puts on ROOT: `test_rs_claim_lpi` claims QEMU's `edu` and reads the +# interrupts it raises; `test_rs_claim_unrouted` is refused the `e1000e` the +# IORT routes past the SMMUv3; `test_rs_claim_bar_fault` faults in edu's BAR +# and names it as a syscall's buffer. [boot] start = ["logkeeper", "test-runner"] @@ -11,5 +14,5 @@ syscap = ["logread"] # job its capability as a duplicate and hands none without it. [programs.test-runner] receives = ["power"] -args = ["--bound-ms=240000", "test_rs_claim_lpi"] +args = ["--bound-ms=240000", "test_rs_claim_lpi", "test_rs_claim_unrouted", "test_rs_claim_bar_fault"] syscap = ["device", "dup"] diff --git a/toyos-its/src/lpi.rs b/toyos-its/src/lpi.rs index 074c2346308..112e8be0be2 100644 --- a/toyos-its/src/lpi.rs +++ b/toyos-its/src/lpi.rs @@ -125,9 +125,12 @@ impl Space { } } +/// `GICR_PENDBASER.PTZ` [62]: the pending table is all zeroes, so the +/// redistributor need not read it. Write-only: it reads as zero (§12.11.32). +pub const PENDBASER_PTZ: u64 = 1 << 62; + /// `GICR_PENDBASER` (§12.11.32) for the pending table at `table`, which is -/// all zeroes: `Physical_Address` [51:16], and `PTZ` [62], which says so and -/// lets the redistributor not read it. +/// all zeroes: `Physical_Address` [51:16], and [`PENDBASER_PTZ`]. pub const fn pendbaser(table: Phys<16>) -> u64 { - 1 << 62 | CACHED | table.get() + PENDBASER_PTZ | CACHED | table.get() } From 8817182d57a15a3cbddad503f845c0def4a007a3 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 16:53:22 +0200 Subject: [PATCH 3/3] virt_claim_lpi waits for the claim's release line, the last it reads It waited for the ITS's "unmapped from slot" line and then read pcidev's "released from slot" line, which the kernel logs after it: a drain that ended between the two left the harness reading a line not yet there. The review's mutation at 40d4dd4e6 went red on exactly that, "the kernel never said ... released from slot 0", with the line absent from the capture. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/toyos.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index d7c359f4e13..8dbb2de596a 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -2271,7 +2271,8 @@ fn virt_claim_lpi(profile: qemu::Profile) -> Result<(), String> { let mut qemu = QemuInstance::boot_with_options(case, &[], &[], options); let mut serial = virt_console(&qemu); judge_virt_job(&mut qemu, &mut serial, JOB, SAID)?; - await_marker(&mut qemu, &mut serial, "unmapped from slot ", "the claim's DeviceID unmapped at its end")?; + // The release's own line, its last: the DeviceID's unmapping comes before it. + await_marker(&mut qemu, &mut serial, " released from slot ", "the claim's release")?; let line = |want: &str| serial.lines().find(|l| l.contains(want)).map(str::to_string); let Some(armed) = line("ITS: 0 at 0x8080000 armed") else { return Err(format!("the MADT's ITS was never armed