From 363e0e84186184cce7bf7422ecb59f0029173d02 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 10:02:28 +0200 Subject: [PATCH 1/3] x86-64 kernel leaves are global under a declared CR4.PGE, as AArch64's already are The design accepted on #873, built. A leaf is global if and only if it is the same in every root that will ever be loaded, which on x86-64 is the kernel half: every 2 MiB leaf `paging::init` and `map_mmio` write, and the 4 KiB PTEs `guard_4k` splits out of one, now carry PAGE_GLOBAL, and only `kernel_leaf` writes it. No user leaf and no upper-level entry carries it: `Prot::leaf_bits` never names it and `ensure_table` masks to TABLE_FLAGS. Every invalidation answers for a global translation: - `Owed::discharge` of a leaf whose prior entry was global is INVLPG, unconditionally: INVPCID type 0 and a CR3 write both leave a global translation, and it is held under every CR3, not only the target's. - A `Context` owed in the kernel half is `flush_tlb_all`. - `flush_tlb_all` without PCID is `control_regs::flush_global`, a PGE toggle, where it was a CR3 reload that leaves global translations. - `guard_4k` discharges each of its two writes (both priors global, so one INVLPG each) in place of its full local flush. CR4.PGE joins CR4_REQUIRED, read from CPUID.01H:EDX[13], so `self_check` asserts it on every CPU and a CPU without it is refused by name. The declaration's own write passes through PGE clear, so global translations a firmware that left PGE set may hold are dropped when the kernel takes CR4. Two departures from the design's text, both forced by the tree: - `flush_global` toggles the live CR4, not the declared one: `flush_tlb_all` runs before the BSP's declaration (`panic_console::remap`'s `map_mmio`, main.rs, before `percpu::init_bsp`), when DECLARED_CR4 is still 0. - `pat::init` already carried the same flush as a private `flush_tlb`; it now calls `flush_global`, which is then the one CPU-state write outside the declaration. The const assertion over every `Prot` the design named is not written: PAGE_GLOBAL is named in `kernel_leaf` and `Owed::discharge` alone, which a reader sees. The two claims a stale global translation would break are metal rows on the test kernel, `a_global_kernel_leafs_discharge_drops_it` and `the_machine_flush_drops_a_global_kernel_translation`, driving SYS_DEBUG action 28 (`GLOBAL_LEAF_REWRITE`): a global kernel leaf at the kernel half's last 2 MiB, read, rewritten to a second frame, invalidated as asked and read again, on one CPU with interrupts closed. Each runs its control first, the rewrite with nothing invalidated, and refuses itself if that read already sees the new frame. Metal rows, the cheaper tier, and the one that reaches every wrong flush: this host's TCG guest drops every translation on a CR3 write and has no INVPCID, so a CR3 reload or INVPCID type 3 in place of the machine flush reads fresh there. The `control_regs` metal judge and its host check name PGE as required. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- kernel/src/arch/aarch64/paging.rs | 7 + kernel/src/arch/x86_64/control_regs.rs | 43 ++++- kernel/src/arch/x86_64/cpu.rs | 2 +- kernel/src/arch/x86_64/paging.rs | 151 ++++++++++++++---- kernel/src/arch/x86_64/pat.rs | 19 +-- kernel/src/invalidation.rs | 2 +- kernel/src/syscall/dispatch.rs | 2 + sdk/abi/src/syscall.rs | 15 ++ tests/checks.rs | 13 +- .../src/bin/global_leaf_discharge.rs | 14 ++ .../rust-tests/src/bin/global_leaf_flush.rs | 14 ++ tests/guest/rust-tests/src/global_leaf.rs | 25 +++ tests/toyos.rs | 33 +++- 13 files changed, 275 insertions(+), 65 deletions(-) create mode 100644 tests/guest/rust-tests/src/bin/global_leaf_discharge.rs create mode 100644 tests/guest/rust-tests/src/bin/global_leaf_flush.rs create mode 100644 tests/guest/rust-tests/src/global_leaf.rs diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index d1676cb8690..2117551d051 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -903,3 +903,10 @@ pub fn scanout_whole_pages(phys: u64, size: u64) -> Result<(), &'static str> { } Ok(()) } + +/// x86-64's stages a global kernel leaf going stale, and this architecture's +/// kernel leaves are not judged by it: refused. +#[cfg(feature = "test-actuators")] +pub fn debug_global_leaf(_how: u64) -> u64 { + toyos_abi::syscall::SyscallError::NotSupported.to_u64() +} diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 658c87c3188..23062f3ebc5 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -1,6 +1,7 @@ //! What `CR0`, `CR4`, `IA32_EFER`, `IA32_APIC_BASE` and the performance //! request hold on every CPU in this machine. One declaration, applied by the -//! BSP and every AP and checked on each; nothing else may write any of them. +//! BSP and every AP and checked on each; nothing else may write any of them +//! but [`flush_global`], which puts back the `CR4` it read. //! Each register is written whole: `CR0` and `EFER` are constants, `CR4` is //! required bits plus whatever optional bits this CPU offers, and //! `IA32_APIC_BASE` is the local APIC at its architectural address in the one @@ -52,6 +53,7 @@ mod cr4 { pub const DE: u64 = 1 << 3; pub const PAE: u64 = 1 << 5; pub const MCE: u64 = 1 << 6; + pub const PGE: u64 = 1 << 7; pub const OSFXSR: u64 = 1 << 9; pub const OSXMMEXCPT: u64 = 1 << 10; pub const LA57: u64 = 1 << 12; @@ -126,9 +128,10 @@ const ENERGY_PERF_BIAS: u64 = 6; pub const CR0: u64 = cr0::PE | cr0::MP | cr0::ET | cr0::NE | cr0::WP | cr0::PG; /// `CR4` bits every CPU must have. `DE` is zero legacy, not need — this kernel -/// touches no debug register. `FSGSBASE` is [`CR4_FORBIDDEN`], not here. +/// touches no debug register. `PGE` makes a kernel leaf global +/// (`paging::kernel_leaf`). `FSGSBASE` is [`CR4_FORBIDDEN`], not here. const CR4_REQUIRED: u64 = - cr4::DE | cr4::PAE | cr4::MCE | cr4::OSFXSR | cr4::OSXMMEXCPT; + cr4::DE | cr4::PAE | cr4::MCE | cr4::PGE | cr4::OSFXSR | cr4::OSXMMEXCPT; /// `CR4` bits this kernel takes when the CPU offers them (checked against CPUID first: an undefined bit is `#GP`). const CR4_OPTIONAL: u64 = cr4::SMEP | cr4::SMAP | cr4::PCIDE | cr4::UMIP; @@ -224,10 +227,13 @@ pub fn init(cpu_id: u32) { if !skipped(cpu_id) { // SAFETY: `write_cr4` faults only on an undefined bit, on clearing `PAE` // in long mode, or on `PCIDE` with a nonzero PCID — `declaration` checked - // the first two and both callers use PCID 0; `wrmsr` writes [`EFER`], whose - // bits `declaration` has just confirmed this CPU defines and whose `LMA` - // is the one this CPU, in long mode, already holds. + // the first two for both values and both callers use PCID 0; `wrmsr` + // writes [`EFER`], whose bits `declaration` has just confirmed this CPU + // defines and whose `LMA` is the one this CPU, in long mode, already holds. unsafe { + // Through `PGE` clear: only a change of `PGE` drops the global + // translations a firmware that left it set may still hold. + cpu::write_cr4(declared & !cr4::PGE); cpu::write_cr4(declared); cpu::wrmsr(efer::MSR, EFER); } @@ -345,6 +351,28 @@ pub fn pcid_active() -> bool { DECLARED_CR4.load(Ordering::Acquire) & cr4::PCIDE != 0 } +/// Drops every translation this CPU holds, global ones included (SDM Vol. 3A +/// §4.10.4.1, APM Vol. 2 §5.5): `PGE` cleared and put back where it is set; +/// where it is not, nothing is global and a `CR3` reload drops the rest. For a +/// CPU whose `PCIDE` is clear, which is every CPU before its declaration. +/// +/// The live register and not the declaration, because it runs before any +/// declaration exists. No interrupt guard: a nested call leaves `CR4` as it +/// found it. +pub fn flush_global() { + let live = cpu::read_cr4(); + // SAFETY: `live` and `live` without `PGE` are both values this CPU holds or + // held, which fault nowhere; a `CR3` write of the `CR3` just read is valid. + unsafe { + if live & cr4::PGE != 0 { + cpu::write_cr4(live & !cr4::PGE); + cpu::write_cr4(live); + } else { + cpu::write_cr3(cpu::read_cr3()); + } + } +} + /// Proof that this machine's declaration carries `PCIDE`, so `INVPCID` is /// not `#UD` here. Zero-sized: an `Option` costs nothing extra. /// Never stales: `PCIDE`, once declared, is never cleared. @@ -406,10 +434,11 @@ fn declaration(cpu_id: u32) -> u64 { /// The `CR4` bits this CPU will accept, as CPUID reports them. fn supported() -> u64 { - const CPUID_1_EDX: [(u32, u64); 5] = [ + const CPUID_1_EDX: [(u32, u64); 6] = [ (2, cr4::DE), (6, cr4::PAE), (7, cr4::MCE), + (13, cr4::PGE), (24, cr4::OSFXSR), (25, cr4::OSXMMEXCPT), ]; diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index bff62883f2f..65cf3444524 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -238,7 +238,7 @@ pub fn invlpg(addr: u64) { unsafe { asm!("invlpg [{}]", in(reg) addr, options(nostack)); } } -/// Descriptor type for [`invpcid`] (SDM Vol. 3A §4.10.4.1); three variants, not four, because this kernel issues no global-page discard. +/// Descriptor type for [`invpcid`] (SDM Vol. 3A §4.10.4.1). #[derive(Clone, Copy)] #[repr(u64)] pub enum Invpcid { diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 29798570c88..5497911c22b 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -3,8 +3,9 @@ // A live-structure write's TLB invalidation is derived from the entry it // replaced, never chosen by the caller, and discharges only on this CPU — // `arch::tlb::shootdown` is the caller's job for the rest of the machine. -// No mapping here is global, which is what makes a single-address -// invalidation (INVPCID or INVLPG) complete. +// A leaf is global if and only if it is the same in every root that will ever +// be loaded, which is the kernel half and only [`kernel_leaf`] writes it; a +// mapping that differs between roots needs a constructor of its own. use crate::log; use alloc::boxed::Box; @@ -31,6 +32,9 @@ const PAGE_CACHE_DISABLE: u64 = 1 << 4; const PAGE_ACCESSED: u64 = 1 << 5; const PAGE_DIRTY: u64 = 1 << 6; const PAGE_SIZE_BIT: u64 = 1 << 7; +/// Cached across every `CR3` and every PCID, because `arch::control_regs` +/// declares `CR4.PGE`; the same bit in a 2 MiB leaf and a 4 KiB one. +const PAGE_GLOBAL: u64 = 1 << 8; /// In a 4 KiB PTE this bit is bit 7 instead, so flags can't cross a split unmoved. const PAGE_PAT_2M: u64 = 1 << 12; /// Not-executable only because `arch::control_regs` asserts `EFER.NXE` on every CPU. @@ -42,7 +46,7 @@ const ADDR_MASK_2M: u64 = 0x000F_FFFF_FFE0_0000; const TABLE_FLAGS: u64 = PAGE_PRESENT | PAGE_WRITE | PAGE_USER; impl Prot { - /// The permission bits a leaf entry carries; address and cache policy stay the caller's. + /// The permission bits a user leaf carries; address and cache policy stay the caller's. fn leaf_bits(self) -> u64 { let common = PAGE_PRESENT | PAGE_USER; match self { @@ -53,6 +57,13 @@ impl Prot { } } +/// Every bit of a 2 MiB kernel-half leaf but its address and size: the one +/// writer of [`PAGE_GLOBAL`]. Writable and executable, because the kernel's +/// image is in the direct map. +fn kernel_leaf(cache: CachePolicy) -> u64 { + PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL | cache.pde_bits() +} + /// Which PAT entry a 2 MiB mapping selects: `Normal` is entry 0 (WB), whose /// type for a range is the MTRR's (SDM Vol. 3A Table 11-7); `Uncacheable` is /// [`pat::UC_ENTRY`](crate::arch::pat::UC_ENTRY), UC under every MTRR type; @@ -144,7 +155,7 @@ impl PageTablePage { fn write_pde(&mut self, idx: usize, va: u64, value: u64) -> Owed { let prior = core::mem::replace(&mut self.0[idx], value); if prior & PAGE_PRESENT != 0 && prior & PAGE_SIZE_BIT == 0 { - Owed::Context + Owed::Context { va } } else { Owed::of(prior, va) } @@ -152,10 +163,10 @@ impl PageTablePage { /// Owes `Context` only if the widen actually changed a bit — a stale /// narrower entry can raise a spurious fault. - fn widen(&mut self, idx: usize, flags: u64) -> Owed { + fn widen(&mut self, idx: usize, va: u64, flags: u64) -> Owed { let before = self.0[idx]; self.0[idx] = before | flags; - if self.0[idx] == before { Owed::Nothing } else { Owed::Context } + if self.0[idx] == before { Owed::Nothing } else { Owed::Context { va } } } /// Not yet linked into a live structure, so nothing can be stale. @@ -175,8 +186,8 @@ enum Owed { /// One linear address, whose leaf changed under it. Address { va: u64, prior: u64 }, /// Everything under an upper-level entry that widened, or a PDE that - /// stopped naming its page table. - Context, + /// stopped naming its page table; `va` is an address under it. + Context { va: u64 }, } impl Owed { @@ -193,10 +204,14 @@ impl Owed { /// `INVPCID` names its tag directly (SDM Vol. 3A §4.10.4.1), so writing a /// child's tables invalidates only the child; without PCID, a CPU holds /// entries only for its loaded CR3, so nothing is owed unless `target` - /// is that CR3. + /// is that CR3. A global translation is the exception to both: it is held + /// under every CR3 and every PCID, and neither `INVPCID` type 0 or 1 nor + /// a CR3 write drops it. fn discharge(self, target: Cr3) { match self { Self::Nothing => {} + // `INVLPG` drops the page's global translation in every PCID. + Self::Address { va, prior } if prior & PAGE_GLOBAL != 0 => crate::arch::cpu::invlpg(va), Self::Address { va, .. } => { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::Address, target.pcid(), va); @@ -204,7 +219,9 @@ impl Owed { crate::arch::cpu::invlpg(va); } } - Self::Context => { + // Every kernel-half leaf is global. + Self::Context { va } if crate::mm::is_kernel_addr(va) => flush_tlb_all(), + Self::Context { .. } => { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::SinglePcid, target.pcid(), 0); } else if Cr3::current().phys() == target.phys() { @@ -222,7 +239,7 @@ impl Owed { Self::Address { va, prior } => { panic!("{what}: an install at {va:#x} found the present entry {prior:#x}") } - Self::Context => panic!("{what}: an install widened an upper-level entry"), + Self::Context { .. } => panic!("{what}: an install widened an upper-level entry"), } } @@ -256,18 +273,12 @@ pub fn pcid_active() -> Option { crate::arch::control_regs::PcidActive::ask() } -/// Flush all TLB entries on this CPU, all PCIDs. +/// Flush all TLB entries on this CPU, all PCIDs, global ones included. pub fn flush_tlb_all() { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::AllIncludingGlobal, 0, 0); } else { - // SAFETY: the value is exactly what `read_cr3` just read on this - // CPU, so writing it back is a valid CR3 write that only flushes - // the TLB (there is no `PAGE_GLOBAL` in this kernel). - unsafe { - let cr3 = crate::arch::cpu::read_cr3(); - crate::arch::cpu::write_cr3(cr3); - } + crate::arch::control_regs::flush_global(); } } @@ -667,7 +678,7 @@ impl AddressSpace { let end = (phys + size + PAGE_2M - 1) & !(PAGE_2M - 1); let mut cur = start; while cur < end { - self.map_2m(cur, PAGE_PRESENT | PAGE_WRITE | cache.pde_bits()); + self.map_2m(cur, kernel_leaf(cache)); cur += PAGE_2M; } crate::mm::Mmio::new(crate::mm::DirectMap::from_phys(phys), size) @@ -720,6 +731,11 @@ impl AddressSpace { let pd = unsafe { PageTablePage::from_phys_mut(pd_phys) }; let pde = pd[pd_idx]; assert!(pde & PAGE_PRESENT != 0, "guard_4k: {phys:#x} is not in the direct map"); + // Both writes discharge on this CPU alone: the guard belongs to one + // CPU, so only it dereferences the removed page; `alloc_idle_stack` + // runs on the BSP before an AP's TLB exists, and a sibling's stale + // 2 MiB entry stays correct for the other 511 pages it still maps. + let target = self.root(); // Already split: an earlier guard in the same 2 MiB region. if pde & PAGE_SIZE_BIT != 0 { @@ -735,9 +751,7 @@ impl AddressSpace { } let pt_phys = pt.phys(); self.children.push(pt); - // Covered by the flush below, wider than either write could owe. - pd.write_pde(pd_idx, virt, pt_phys | PAGE_PRESENT | PAGE_WRITE) - .subsumed_by_flush(); + pd.write_pde(pd_idx, virt, pt_phys | PAGE_PRESENT | PAGE_WRITE).discharge(target); } // SAFETY: `pd[pd_idx]` names a page table either way, permanently — @@ -745,13 +759,7 @@ impl AddressSpace { let pt = unsafe { PageTablePage::from_phys_mut(pd[pd_idx] & ADDR_MASK) }; let idx = ((phys >> 12) & 0x1FF) as usize; assert!(pt[idx] & PAGE_PRESENT != 0, "guard_4k: {phys:#x} is already unmapped"); - pt.write(idx, virt, 0).subsumed_by_flush(); - - // Local only: the guard belongs to one CPU, so only it dereferences - // the removed page; `alloc_idle_stack` runs on the BSP before an - // AP's TLB exists, and a sibling's stale 2 MiB entry stays correct - // for the other 511 pages it still maps unchanged. - flush_tlb_all(); + pt.write(idx, virt, 0).discharge(target); } /// Replaces whatever is there; a page `guard_4k` already split must not @@ -789,7 +797,7 @@ impl AddressSpace { self.children.push(child); } else { // Upper-level entries: widen only (OR), never narrow. - self.root.widen(pml4_idx, flags).discharge(target); + self.root.widen(pml4_idx, va, flags).discharge(target); } // SAFETY: the branch above guaranteed PRESENT, so `& ADDR_MASK` @@ -803,7 +811,7 @@ impl AddressSpace { .expect_install("ensure_table: pdpt"); self.children.push(child); } else { - pdpt.widen(pdpt_idx, flags).discharge(target); + pdpt.widen(pdpt_idx, va, flags).discharge(target); } // SAFETY: same argument as `pdpt` above, one level down. @@ -904,6 +912,83 @@ pub fn map_mmio(phys: u64, size: u64, policy: MmioPolicy) -> crate::mm::Mmio { mmio } +/// Where [`debug_global_leaf`] maps: the kernel half's last 2 MiB, under a root +/// slot [`seal_kernel_half`] installed, as far past the direct map as it goes. +#[cfg(feature = "test-actuators")] +const SCRATCH: u64 = 0u64.wrapping_sub(PAGE_2M); + +/// `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`: a [`kernel_leaf`] at [`SCRATCH`] over one +/// frame, read so this CPU caches it global; the leaf pointed at a second frame +/// and invalidated as `how` asks; read again. Answers which frame that read +/// saw. The kernel's lock keeps it on one CPU and interrupts stay closed, so no +/// flush but `how`'s lands between the two reads. +#[cfg(feature = "test-actuators")] +pub fn debug_global_leaf(how: u64) -> u64 { + use toyos_abi::syscall::debug_action as da; + use toyos_abi::syscall::SyscallError; + + enum How { + Withheld, + Discharged, + ShotDown, + } + let how = match how { + da::GLOBAL_LEAF_WITHHELD => How::Withheld, + da::GLOBAL_LEAF_DISCHARGED => How::Discharged, + da::GLOBAL_LEAF_SHOT_DOWN => How::ShotDown, + _ => return SyscallError::InvalidArgument.to_u64(), + }; + let (Some(old), Some(new)) = (crate::mm::pmm::alloc_page(), crate::mm::pmm::alloc_page()) else { + return SyscallError::ResourceExhausted.to_u64(); + }; + // Each frame's first word is its own address, so a read names its frame. + for page in [&old, &new] { + let at = page.direct_map(); + // SAFETY: a frame this call owns, through the direct map. + unsafe { core::ptr::write_volatile(at.as_mut_ptr::(), at.phys()) }; + } + let (old_phys, new_phys) = (old.direct_map().phys(), new.direct_map().phys()); + let leaf = kernel_leaf(CachePolicy::Normal) | PAGE_SIZE_BIT; + // SAFETY: every read below is made while `SCRATCH` maps one of the two + // frames this call owns, or a translation of one of them. + let read = || unsafe { core::ptr::read_volatile(SCRATCH as *const u64) }; + + let seen = { + let mut space = kernel().lock(); + let _closed = crate::arch::IrqGuard::close(); + let target = space.root(); + let pd = space.ensure_table(SCRATCH, leaf); + let idx = indices(SCRATCH).2; + if pd[idx] & PAGE_PRESENT != 0 { + return SyscallError::AlreadyExists.to_u64(); + } + pd.write_pde(idx, SCRATCH, old_phys | leaf).expect_install("debug_global_leaf"); + let first = read(); + assert!(first == old_phys, "debug_global_leaf: a fresh leaf over {old_phys:#x} read {first:#x}"); + let owed = pd.write_pde(idx, SCRATCH, new_phys | leaf); + match how { + // The control: what this CPU still holds with nothing invalidated. + How::Withheld => {} + How::Discharged => owed.discharge(target), + How::ShotDown => { + owed.subsumed_by_flush(); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Staged); + } + } + let seen = read(); + pd.write_pde(idx, SCRATCH, 0).discharge(target); + seen + }; + // No CPU keeps a translation of either frame past its free. + crate::arch::tlb::shootdown(crate::invalidation::Origin::Staged); + drop((old, new)); + match seen { + _ if seen == old_phys => da::GLOBAL_LEAF_SAW_OLD, + _ if seen == new_phys => da::GLOBAL_LEAF_SAW_NEW, + _ => panic!("debug_global_leaf: the leaf over {old_phys:#x} then {new_phys:#x} read {seen:#x}"), + } +} + /// Build kernel page tables: the direct map in the high half, in 2 MiB pages, /// as far as [`toyos_bootmap::x86_64::direct_map_end`] reaches, and answer /// that end. The scanout is not mapped for itself: in the low map the MTRRs @@ -928,7 +1013,7 @@ pub(crate) fn init(memory_map: &[MemoryMapEntry], _scanout: Option<(u64, u64)>) } let mut addr: u64 = 0; while addr < end { - kernel.map_2m(addr, PAGE_PRESENT | PAGE_WRITE); + kernel.map_2m(addr, kernel_leaf(CachePolicy::Normal)); addr += PAGE_2M; } crate::log!("paging: the direct map covers 0x0..{end:#x}"); diff --git a/kernel/src/arch/x86_64/pat.rs b/kernel/src/arch/x86_64/pat.rs index 71201b75c88..ca0c1323352 100644 --- a/kernel/src/arch/x86_64/pat.rs +++ b/kernel/src/arch/x86_64/pat.rs @@ -1,6 +1,7 @@ //! Programs `IA32_PAT` with one write-combining entry for the GOP scanout; //! SDM Vol. 3A Table 11-7 makes an MTRR range register unnecessary. +use crate::arch::control_regs::flush_global; use crate::arch::cpu; const IA32_PAT: u32 = 0x277; @@ -57,15 +58,14 @@ pub fn init() { unsafe { core::arch::asm!("pushfq", "pop {}", "cli", out(reg) flags); let cr0 = cpu::read_cr0(); - let cr4 = cpu::read_cr4(); // CD set with NW clear is no-fill mode: writes still hit, nothing new caches. cpu::write_cr0((cr0 | CR0_CD) & !CR0_NW); cpu::wbinvd(); - flush_tlb(cr4); + flush_global(); cpu::wrmsr(IA32_PAT, PAT_VALUE); - flush_tlb(cr4); + flush_global(); cpu::wbinvd(); cpu::write_cr0(cr0); } @@ -88,21 +88,8 @@ pub fn check() { const CR0_NW: u64 = 1 << 29; const CR0_CD: u64 = 1 << 30; -const CR4_PGE: u64 = 1 << 7; const RFLAGS_IF: u64 = 1 << 9; -/// Flush every TLB entry, including global ones (SDM Vol. 3A §4.10.4.1). -/// # Safety -/// `cr4` must be this CPU's live `CR4`; both arms restore it verbatim. -unsafe fn flush_tlb(cr4: u64) { - if cr4 & CR4_PGE != 0 { - cpu::write_cr4(cr4 & !CR4_PGE); - cpu::write_cr4(cr4); - } else { - cpu::write_cr3(cpu::read_cr3()); - } -} - /// This CPU's live `IA32_PAT`. pub fn msr() -> u64 { cpu::rdmsr(IA32_PAT) diff --git a/kernel/src/invalidation.rs b/kernel/src/invalidation.rs index bf6511946c7..4bfd094ceca 100644 --- a/kernel/src/invalidation.rs +++ b/kernel/src/invalidation.rs @@ -3,7 +3,7 @@ /// Which path issued a shootdown, so the census names who pays: `Dlopen` (a /// `Shared` window or rollback unmap), `Pcid` (pool reclaim), `Mmio`, `Unmap` -/// (`Unmapped::drop`), `Pipe`, `Staged` (the ack-delay actuator), `Bench` +/// (`Unmapped::drop`), `Pipe`, `Staged` (a test actuator's), `Bench` /// (`arch::tlb::bench`'s own, so a measured shootdown is never counted as one /// a path in this kernel needed), `Stop` (the power-off's, for the answer from /// Ring 0 every other CPU owes it, not for a flush). diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index e61b604b560..3a3b3d6541c 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -662,6 +662,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> // An interrupt inside a syscall's body, which nothing a guest does puts there on demand. DA::RING0_TIMER_IN_SYSCALL => ring0_timer_in_syscall(), DA::ACPI_FIRMWARE_LOCK => crate::arch::acpi_mode::debug_firmware_lock(a2), + // A translation outliving its entry, which only a TLB that keeps one shows. + DA::GLOBAL_LEAF_REWRITE => crate::mm::paging::debug_global_leaf(a2), _ => SyscallError::InvalidArgument.to_u64(), }, SYS_SCHED_INFO => match ctx.copy_out(UserAddr::new(a1), &sys_sched_info()) { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 906e12a9d36..691950a478d 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -884,6 +884,21 @@ pub mod debug_action { pub const RING0_FIRE_REARMED: u64 = 0; pub const RING0_FIRE_NEVER: u64 = 1; pub const RING0_FIRE_OTHER_SPAN: u64 = 2; + /// Map a global 2 MiB kernel leaf over one frame and read through it, point + /// the leaf at a second frame, invalidate as the argument says, and read + /// again, all on one CPU with interrupts closed. The argument is + /// [`GLOBAL_LEAF_WITHHELD`], which invalidates nothing, + /// [`GLOBAL_LEAF_DISCHARGED`], the write's own discharge, or + /// [`GLOBAL_LEAF_SHOT_DOWN`], a machine-wide shootdown. Answers + /// [`GLOBAL_LEAF_SAW_OLD`] or [`GLOBAL_LEAF_SAW_NEW`], the frame the second + /// read saw; `NotSupported` on AArch64, `AlreadyExists` where something is + /// already mapped where the leaf goes. + pub const GLOBAL_LEAF_REWRITE: u64 = 28; + pub const GLOBAL_LEAF_WITHHELD: u64 = 0; + pub const GLOBAL_LEAF_DISCHARGED: u64 = 1; + pub const GLOBAL_LEAF_SHOT_DOWN: u64 = 2; + pub const GLOBAL_LEAF_SAW_OLD: u64 = 0; + pub const GLOBAL_LEAF_SAW_NEW: u64 = 1; } /// Every kind of kernel object, in the order the kernel's own `kobject!` diff --git a/tests/checks.rs b/tests/checks.rs index 546e548bae3..88008392047 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -341,7 +341,7 @@ mod checks { #[test] fn control_regs_verdict() -> Result<(), String> { const AP_BEFORE: (u64, u64) = (0xe000_0011, 0x0031_0620); - const DECLARED: (u64, u64) = (0x8001_0033, 0x0030_0668); + const DECLARED: (u64, u64) = (0x8001_0033, 0x0030_06e8); fn log(cpus: &[(u64, u64)]) -> String { cpus.iter() @@ -382,15 +382,16 @@ mod checks { refused("OSXSAVE set", &[(DECLARED.0, DECLARED.1 | (1 << 18)); 4], "OSXSAVE")?; // Two bits a machine could hold uniformly, each one line of kernel diff // away, and neither reachable by an actuator. `AM` is named clear above and - // answers by name; `PGE` is named nowhere, which is the case the whole - // never-named rule exists for — `TSD` and `PKE` are the same case. + // answers by name; `PKE` is named nowhere, which is the case the whole + // never-named rule exists for — `TSD` is the same case. refused("every CPU with AM set", &[(DECLARED.0 | (1 << 18), DECLARED.1); 4], "AM")?; refused( - "every CPU with PGE set", - &[(DECLARED.0, DECLARED.1 | (1 << 7)); 4], + "every CPU with PKE set", + &[(DECLARED.0, DECLARED.1 | (1 << 22)); 4], "never named", )?; refused("every CPU without SMEP", &[(DECLARED.0, DECLARED.1 & !(1 << 20)); 4], "SMEP")?; + refused("every CPU without PGE", &[(DECLARED.0, DECLARED.1 & !(1 << 7)); 4], "PGE")?; // A CPU that agrees about every named bit and differs in one the CPU is // allowed to withhold, so nothing above it can object. refused("one CPU with PCID and three without", &[DECLARED, DECLARED, DECLARED, (DECLARED.0, DECLARED.1 | (1 << 17))], "cpu3")?; @@ -398,7 +399,7 @@ mod checks { // died before the check looks like. refused("three lines for four CPUs", &[DECLARED; 3], "{0, 1, 2, 3}")?; - eprintln!(" [control_regs] the verdict refuses 10 machines and accepts the declared one"); + eprintln!(" [control_regs] the verdict refuses 11 machines and accepts the declared one"); Ok(()) } diff --git a/tests/guest/rust-tests/src/bin/global_leaf_discharge.rs b/tests/guest/rust-tests/src/bin/global_leaf_discharge.rs new file mode 100644 index 00000000000..10e241c3fa1 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/global_leaf_discharge.rs @@ -0,0 +1,14 @@ +//! A global kernel leaf rewritten through its own discharge is read fresh by +//! the CPU that wrote it: the discharge dropped the global translation, which +//! `INVPCID` type 0 and a `CR3` write both leave. + +#[path = "../global_leaf.rs"] +mod global_leaf; + +use toyos_abi::syscall::debug_action as da; + +fn main() { + let saw = global_leaf::rewritten(da::GLOBAL_LEAF_DISCHARGED); + assert_eq!(saw, da::GLOBAL_LEAF_SAW_NEW, "the write's discharge left the old frame's global translation"); + println!("global_leaf_discharge: the discharge dropped the global translation"); +} diff --git a/tests/guest/rust-tests/src/bin/global_leaf_flush.rs b/tests/guest/rust-tests/src/bin/global_leaf_flush.rs new file mode 100644 index 00000000000..96b425decde --- /dev/null +++ b/tests/guest/rust-tests/src/bin/global_leaf_flush.rs @@ -0,0 +1,14 @@ +//! A global kernel leaf rewritten under a machine-wide shootdown is read fresh +//! by the CPU that issued it: the flush every CPU of the shootdown runs drops +//! global translations, which `INVPCID` type 3 and a `CR3` write both leave. + +#[path = "../global_leaf.rs"] +mod global_leaf; + +use toyos_abi::syscall::debug_action as da; + +fn main() { + let saw = global_leaf::rewritten(da::GLOBAL_LEAF_SHOT_DOWN); + assert_eq!(saw, da::GLOBAL_LEAF_SAW_NEW, "the shootdown's flush left the old frame's global translation"); + println!("global_leaf_flush: the shootdown's flush dropped the global translation"); +} diff --git a/tests/guest/rust-tests/src/global_leaf.rs b/tests/guest/rust-tests/src/global_leaf.rs new file mode 100644 index 00000000000..4af5e32374d --- /dev/null +++ b/tests/guest/rust-tests/src/global_leaf.rs @@ -0,0 +1,25 @@ +//! `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`, for the two metal rows that judge how a +//! global kernel translation is dropped, and the control both run first. +//! +//! The action maps a global 2 MiB kernel leaf, reads through it, points it at a +//! second frame, invalidates as asked and reads again, on one CPU with +//! interrupts closed. **Metal rows and not guest tests**: a metal row is the +//! cheaper tier, and this host's TCG guest drops every translation on a `CR3` +//! write and has no `INVPCID`, so both wrong machine flushes read fresh there. + +use toyos_abi::syscall::{self, debug_action as da}; + +/// The frame the second read saw after `how`, once this CPU has shown it reads +/// the first one with nothing invalidated: a CPU that read the second frame +/// then held no translation to go stale, and its fresh read after an +/// invalidation would be no evidence of that invalidation. +pub fn rewritten(how: u64) -> u64 { + let withheld = syscall::debug_with(da::GLOBAL_LEAF_REWRITE, da::GLOBAL_LEAF_WITHHELD); + assert_eq!( + withheld, + da::GLOBAL_LEAF_SAW_OLD, + "the control: with nothing invalidated the second read did not see the first frame, so \ + this CPU held no translation to go stale and the row judges nothing", + ); + syscall::debug_with(da::GLOBAL_LEAF_REWRITE, how) +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 36f92181a34..fe8933c0131 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -87,6 +87,12 @@ const RUST_SKIP: &[&str] = &[ // is read off the clock around the syscall. The `tlb_shootdown_waits` // metal row runs it on the T14. "tlb_shootdown_waits", + // Each reads a global translation its kernel may leave stale, which a TCG + // guest's TLB drops on any `CR3` write and the T14's keeps: the + // `a_global_kernel_leafs_discharge_drops_it` and + // `the_machine_flush_drops_a_global_kernel_translation` metal rows run them. + "global_leaf_discharge", + "global_leaf_flush", // Its product is the windows a `mask-windows` kernel reports under it, and // its 256 threads' stacks would crowd every member after it in one boot. // The `mask_windows` metal row runs it. @@ -695,6 +701,29 @@ const METAL: &[(&str, metal::Metal)] = &[ judge: |b| b[0].job_passed("test_rs_tlb_shootdown_waits"), }, ), + ( + // A global kernel translation outlives its entry until something drops + // it, and the kernel that carries `SYS_DEBUG` stages one: the verdict + // is the guest's own exit, its control first. + "a_global_kernel_leafs_discharge_drops_it", + metal::Metal { + arms: &[metal::Arm { + features: ACTUATOR_KERNEL, + ..metal::once("shared-debug", "tests/images/testcases.toml", SELFTEST_PARAMS, &["test_rs_global_leaf_discharge"]) + }], + judge: |b| b[0].job_passed("test_rs_global_leaf_discharge"), + }, + ), + ( + "the_machine_flush_drops_a_global_kernel_translation", + metal::Metal { + arms: &[metal::Arm { + features: ACTUATOR_KERNEL, + ..metal::once("shared-debug", "tests/images/testcases.toml", SELFTEST_PARAMS, &["test_rs_global_leaf_flush"]) + }], + judge: |b| b[0].job_passed("test_rs_global_leaf_flush"), + }, + ), ( // What one diary record costs its writer: a million written back to // back by `SYS_DEBUG`'s flood, so the kernel that carries it, timed @@ -5469,6 +5498,8 @@ fn control_regs(log: &str, cpus: u32) -> Result<(), String> { (3, "DE", true), (5, "PAE", true), (6, "MCE", true), + // Every kernel leaf is global, and only with `PGE` set is one cached so. + (7, "PGE", true), (9, "OSFXSR", true), (10, "OSXMMEXCPT", true), (12, "LA57", false), @@ -5511,7 +5542,7 @@ fn control_regs(log: &str, cpus: u32) -> Result<(), String> { } // A bit named nowhere above is as much of the declaration as a named one, - // and the kernel writes both registers whole — so `UMIP`, `PGE`, `TSD` or + // and the kernel writes both registers whole — so `UMIP`, `TSD` or // `PKE` would reach every CPU with nothing here to say so. Which is why // what follows is the set this gate has an opinion about, rather than a // second list of bits to forbid: a forbid-list fails open on the next bit. From 86877feb6c83e3f0b0cda12758a02bdd9d84269a Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 15:48:45 +0200 Subject: [PATCH 2/3] Intel's INVLPG erratum withholds PCID, and a user-half shootdown keeps the kernel's global translations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fix round on #873 after its first review (SEND BACK) and the owner's ruling to follow the manufacturers' documents. Intel's ADL063 (682436-040) and RPL043 (740518-031), "INVLPG May Invalidate Global TLB Entries Only For The Current PCID": with PCIDs enabled, INVLPG may leave a global translation cached for another PCID. Every kernel-half discharge of this branch is an INVLPG of a global translation, so on such a CPU below the microcode that fixed it, `toyos-cpuvuln::invlpg_misses_global` answers yes and `control_regs` withholds PCIDE; on an Intel CPU it logs the signature and the revision it read, so a boot can be held to Linux's reading of the same CPU, and says where it withholds PCID. The rows are the signatures each document's Table 2-1 names beside its erratum: every Alder Lake and Raptor Lake signature but the 6P+0E Raptor Lake die (0xB06F5), which RPL043's summary marks N/A, plus Alder Lake-N (0xB06E0), which its own specification update (764616-004) does not list. The fixed revisions are Intel's, posted to linux-kernel as Message-ID 20240325231300.qrltbzf6twm43ftb@desk; no specification update carries one. The erratum does not apply in VMX non-root operation, so a guest keeps PCID, and a revision is compared signed (SDM Vol. 3A Table 12-7). The kernel reads the revision on an Intel CPU alone, by §12.11.7.1's procedure (`cpu::intel_microcode_revision`). A shootdown now names the half it changed (`invalidation::Half`). The protocol (`shootdown.rs`) counts wide generations before issuing them, and a serve answers wide where a wide generation was issued since its last wide answer, so a coalesced answer reaches as far as the widest generation it covers. x86-64 maps a wide answer to `flush_tlb_all` (INVPCID type 2, or the PGE toggle) and a narrow one to the new `flush_tlb_user` (INVPCID type 3, or a CR3 reload), which keeps global translations. Only `map_mmio` and the global-leaf actuator change the kernel half; `join` stays wide, since a kernel-half change made before the machine answered issued no generation. The user-half `Owed::Context` arm without PCID is a CR3 reload again. The global-leaf actuator runs its control and its arm in one IRQ-closed section and answers UNHELD where the control read the second frame. A third metal row, `a_user_half_shootdown_keeps_a_global_kernel_translation`, holds the user-half flush; the kernel-half row is renamed `a_kernel_half_shootdown_drops_a_global_kernel_translation`. Four loom models hold the wide reach. The track issue is deleted: this branch builds it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...ge-global-is-a-decision-nobody-has-made.md | 44 ---- kernel/cpuvuln/Cargo.toml | 2 +- kernel/cpuvuln/src/invlpg.rs | 139 ++++++++++++ kernel/cpuvuln/src/lib.rs | 4 + kernel/cpuvuln/src/table.rs | 32 ++- kernel/loom/tests/counters_round.rs | 6 +- kernel/loom/tests/tlb_shootdown.rs | 214 +++++++++++++++--- kernel/src/arch/aarch64/tlb.rs | 4 +- kernel/src/arch/x86_64/control_regs.rs | 35 ++- kernel/src/arch/x86_64/cpu.rs | 14 ++ kernel/src/arch/x86_64/paging.rs | 69 ++++-- kernel/src/arch/x86_64/pio.rs | 2 +- kernel/src/arch/x86_64/smi_cmd.rs | 6 +- kernel/src/arch/x86_64/tlb.rs | 52 +++-- kernel/src/counters.rs | 6 +- kernel/src/invalidation.rs | 16 +- kernel/src/mm/unmapped.rs | 2 +- kernel/src/process.rs | 2 +- kernel/src/shootdown.rs | 42 +++- kernel/src/syscall/vm.rs | 4 +- sdk/abi/src/syscall.rs | 26 ++- .../rust-tests/src/bin/global_leaf_flush.rs | 8 +- .../src/bin/global_leaf_user_flush.rs | 15 ++ tests/guest/rust-tests/src/global_leaf.rs | 31 +-- tests/toyos.rs | 18 +- 25 files changed, 619 insertions(+), 174 deletions(-) delete mode 100644 issues/page-global-is-a-decision-nobody-has-made.md create mode 100644 kernel/cpuvuln/src/invlpg.rs create mode 100644 tests/guest/rust-tests/src/bin/global_leaf_user_flush.rs diff --git a/issues/page-global-is-a-decision-nobody-has-made.md b/issues/page-global-is-a-decision-nobody-has-made.md deleted file mode 100644 index 3e753abec1f..00000000000 --- a/issues/page-global-is-a-decision-nobody-has-made.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -status: open -kind: track -opened: 2026-08-19 ---- - -# There is no `PAGE_GLOBAL` anywhere, and the owner decided: turn it on - -**The owner ruled on 2026-08-20: `PGE` goes on**, sequenced into the -interrupt-distribution/scheduler performance era -(`issues/every-interrupt-lands-on-the-boot-cpu.md`'s wave), never as a -drive-by — because the work below is real: every `Owed::discharge` in -`mm/paging.rs` must answer for global translations, `guard_4k`'s local full -flush too, the bit joins `control_regs.rs`'s one declaration (never a -read-modify-write at a use site), and the gain is measured before/after on -the KVM instrument so the improvement is a number against a number. TCG -cannot price it (tests/CLAUDE.md: the local guest has no such cache model), -which is one more reason it rides the measured era. - -`CR4.PGE` is absent from `kernel/src/arch/x86_64/control_regs.rs`'s declaration and no -`PAGE_GLOBAL` exists in `kernel/src/mm/`. With PCID active, every address space -therefore caches a private copy of the kernel's direct map, and every -`flush_tlb_all` — which is `INVPCID` all-context — throws away the kernel's own -translations on every CPU along with the process's. - -This is not an oversight to fix silently. 2020+ hardware is not Meltdown- -affected and this kernel has no KPTI, so global kernel pages are available. -`control_regs.rs`'s own doctrine — every bit of both registers is decided in one -place, and the bits left out are as much of the declaration as the bits in — -means `PGE`'s absence should be *stated* whichever way it goes. - -**What the answer would have to revisit.** `kernel/src/arch/x86_64/paging.rs` now derives -every invalidation from the entry a write replaced and discharges it with -`INVPCID` type 0 or `INVLPG`. Neither touches a global translation, which is -sound only because no entry in this kernel is global; the module header says so -in as many words, and a `PGE` that goes in has to answer for every `discharge` -there and for `guard_4k`'s local full flush. - -This file's earlier heading said TLB invalidation was chosen, not derived; that -question's other half — an -invalidation chosen by each caller, an unconditional `invlpg` over a not-present -entry, a duplicate one on the demand-paging path, and one aimed at the parent's -PCID while writing a child's tables — is resolved. `git log --follow` on this -file carries the evidence; `Owed` in `mm/paging.rs` is what it became. diff --git a/kernel/cpuvuln/Cargo.toml b/kernel/cpuvuln/Cargo.toml index 6ce1bb2b9f3..04d49b44d32 100644 --- a/kernel/cpuvuln/Cargo.toml +++ b/kernel/cpuvuln/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "toyos-cpuvuln" -description = "Linux's verdicts from a CPU's CPUID and MSR facts, pure: each speculative-execution vulnerability and its mitigation choice, which model-specific counters the CPU has, and the HWP request intel_pstate asks of it." +description = "Linux's verdicts from a CPU's CPUID and MSR facts, pure: each speculative-execution vulnerability and its mitigation choice, which model-specific counters the CPU has, and the HWP request intel_pstate asks of it; and Intel's, whether its INVLPG drops a global translation in every PCID." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/kernel/cpuvuln/src/invlpg.rs b/kernel/cpuvuln/src/invlpg.rs new file mode 100644 index 00000000000..9c970249e1d --- /dev/null +++ b/kernel/cpuvuln/src/invlpg.rs @@ -0,0 +1,139 @@ +//! Whether a CPU's `INVLPG` drops a global translation in every PCID, which a +//! kernel whose kernel-half leaves are global relies on while `CR4.PCIDE` is +//! set: Intel's erratum "INVLPG May Invalidate Global TLB Entries Only For The +//! Current PCID" (ADL063, RPL043), on the CPUs [`table`](crate::table) names, +//! below the microcode revision that fixed it. +//! +//! **Intel's verdict, not Linux's**: the specification updates are the +//! source, and where they part from `invlpg_miss_ids` (`arch/x86/mm/init.c`) +//! they decide. RPL043 marks the 6P+0E die N/A, which Linux's model match +//! takes in; the erratum "does not apply in VMX non-root operation", which +//! Linux does not read; and an update revision is signed (SDM Vol. 3A Table +//! 12-7), where Linux compares unsigned. + +use crate::{table, Ident, Vendor, CPUID_1_ECX_HYPERVISOR}; + +/// What one CPU says about itself, read on that CPU. +#[derive(Clone, Copy, Debug)] +pub struct InvlpgFacts { + pub vendor: Vendor, + /// CPUID.1:EAX. + pub signature: u32, + pub cpuid_1_ecx: u32, + /// `IA32_BIOS_SIGN_ID[63:32]` after the read SDM Vol. 3A §12.11.7.1 + /// defines, present exactly where the vendor is Intel: the procedure is + /// Intel's. + pub microcode: Option, +} + +/// Whether `INVLPG` may leave a global translation cached for another PCID: a +/// CPU Intel names, outside VMX non-root operation (CPUID.1:ECX[31]), running +/// microcode older than the fix. +/// +/// Panics where the facts contradict their own documentation: a revision read +/// on a vendor that is not Intel, or none on one that is. +pub fn invlpg_misses_global(facts: &InvlpgFacts) -> bool { + assert_eq!( + facts.microcode.is_some(), + facts.vendor == Vendor::Intel, + "the microcode revision is read exactly where the vendor is Intel", + ); + let guest = facts.cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR != 0; + table::invlpg_fixed_at(&Ident::new(facts.vendor, facts.signature)) + .is_some_and(|fixed| !guest && (facts.microcode.expect("asserted above") as i32) < fixed) +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::vec::Vec; + + use super::*; + + /// A native Intel CPU of `signature` running `microcode`. + fn native(signature: u32, microcode: u32) -> InvlpgFacts { + InvlpgFacts { vendor: Vendor::Intel, signature, cpuid_1_ecx: 0, microcode: Some(microcode) } + } + + /// Every signature the two specification updates name, from each one's + /// Table 2-1 beside its erratum's summary row, and Alder Lake-N's from its + /// own Table 2-1, each with Intel's fixed revision for its model. + const NAMED: [(u32, u32, &str); 9] = [ + // 682436-040: ADL063 "Fixed" on S, H/P, U and HX. + (0x0009_0672, 0x2e, "ADL-S 8+8, ADL-HX 8+8"), + (0x0009_0675, 0x2e, "ADL-S 6+0"), + (0x0009_06a3, 0x42c, "ADL-H 6+8, ADL-P 6+8"), + (0x0009_06a4, 0x42c, "ADL-U15W 2+8, ADL-U9W 2+8"), + // 740518-031: RPL043 "Fixed" on every column but S 6+0. + (0x000b_0671, 0x118, "RPL-S 8P+16E, RPL-HX 8P+16E, RPL-E 8P+0E"), + (0x000b_06f2, 0x2e, "RPL-S 8P+8E, RPL-HX 8P+8E"), + (0x000b_06a2, 0x4117, "RPL-P 6P+8E, RPL-H 6P+8E"), + (0x000b_06a3, 0x4117, "RPL-U 2P+8E"), + // 764616-004 lists the N-series by this signature alone. + (0x000b_06e0, 0x11, "Alder Lake-N"), + ]; + + #[test] + fn every_cpu_intel_names_withholds_pcid_below_its_fix_and_only_there() { + for (signature, fixed, name) in NAMED { + assert!(invlpg_misses_global(&native(signature, fixed - 1)), "{name} one revision below its fix"); + assert!(!invlpg_misses_global(&native(signature, fixed)), "{name} at its fix"); + } + } + + /// RPL043's summary row reads N/A under S 6+0, signature `0xB06F5`. + #[test] + fn the_die_rpl043_marks_not_applicable_keeps_pcid() { + assert!(!invlpg_misses_global(&native(0x000b_06f5, 0))); + } + + /// "This erratum does not apply to a guest operating system running in VMX + /// non-root operation" (ADL063, RPL043). + #[test] + fn a_guest_keeps_pcid() { + let guest = InvlpgFacts { cpuid_1_ecx: CPUID_1_ECX_HYPERVISOR, ..native(0x0009_06a4, 0) }; + assert!(!invlpg_misses_global(&guest)); + } + + /// Table 12-7: an update revision is a signed 32-bit number. + #[test] + fn a_revision_with_its_sign_bit_set_is_older_than_the_fix() { + assert!(invlpg_misses_global(&native(0x000b_06a2, 0x8000_4117))); + } + + /// The T14's CPU 0 off `fixtures/t14/`: its signature from `cpuid.txt`, its + /// revision from `msr.txt`'s 0x8B, as Linux read both. Tiger Lake is in + /// neither table. + #[test] + fn the_t14_keeps_pcid() { + let words = |text: &'static str, first: &str| -> Vec { + text.lines() + .map(|l| l.split(' ').map(|w| u64::from_str_radix(w, 16).expect(w)).collect::>()) + .find(|l| l[..2] == [u64::from_str_radix(first, 16).unwrap(), 0]) + .expect("a line the capture read") + }; + let leaf_1 = words(include_str!("../fixtures/t14/cpuid.txt"), "1"); + let bios_sign_id = words(include_str!("../fixtures/t14/msr.txt"), "8b")[2]; + let t14 = InvlpgFacts { + vendor: Vendor::Intel, + signature: leaf_1[2] as u32, + cpuid_1_ecx: leaf_1[4] as u32, + microcode: Some((bios_sign_id >> 32) as u32), + }; + assert_eq!((t14.signature, t14.microcode), (0x0008_06c1, Some(0xbe))); + assert!(!invlpg_misses_global(&t14)); + } + + #[test] + fn another_vendor_reads_no_revision_and_keeps_pcid() { + let amd = InvlpgFacts { vendor: Vendor::Amd, microcode: None, ..native(0x0009_06a4, 0) }; + assert!(!invlpg_misses_global(&amd)); + } + + #[test] + #[should_panic(expected = "read exactly where the vendor is Intel")] + fn an_intel_cpu_without_its_revision_is_a_contradiction() { + invlpg_misses_global(&InvlpgFacts { microcode: None, ..native(0x0008_06c1, 0) }); + } +} diff --git a/kernel/cpuvuln/src/lib.rs b/kernel/cpuvuln/src/lib.rs index 20c9e7270d8..63298abff4e 100644 --- a/kernel/cpuvuln/src/lib.rs +++ b/kernel/cpuvuln/src/lib.rs @@ -18,6 +18,8 @@ //! [`counters`] is the same tag's other verdict on a CPU: which of its //! model-specific counters exist. [`hwp`] is the third: whether its //! performance request is declared, and [`hwp_request`] the request. +//! [`invlpg_misses_global`] is Intel's: whether its `INVLPG` leaves another +//! PCID's global translation. //! //! Pure: no I/O, no allocation, no `unsafe`. The caller reads the facts. @@ -28,10 +30,12 @@ use core::fmt; mod counters; mod hwp; +mod invlpg; mod table; pub use counters::{counters, CounterFacts, Counters}; pub use hwp::{hwp, hwp_request, Hwp, HwpFacts, HwpRefusal, HWP_EPP}; +pub use invlpg::{invlpg_misses_global, InvlpgFacts}; #[cfg(test)] mod tests; diff --git a/kernel/cpuvuln/src/table.rs b/kernel/cpuvuln/src/table.rs index 8b748b5fe73..6337c2dea27 100644 --- a/kernel/cpuvuln/src/table.rs +++ b/kernel/cpuvuln/src/table.rs @@ -1,7 +1,8 @@ //! Linux's five CPU tables, row for row: `cpu_vuln_whitelist` and //! `cpu_vuln_blacklist` (`common.c:1182-1344`), `spectre_bad_microcodes` //! (`intel.c:141-163`), `amd_check_tsa_microcode`'s (`amd.c:472-515`) and -//! `test_intel`'s (`arch/x86/events/msr.c:40-122`). A +//! `test_intel`'s (`arch/x86/events/msr.c:40-122`); and one of Intel's, +//! [`invlpg_fixed_at`]'s. A //! lookup in the first two is `x86_match_cpu` (`match.c:36-57`): the //! first row whose vendor, family, model and stepping all match, `0` matching //! any. @@ -356,6 +357,35 @@ pub(crate) fn tsa_microcode(id: &Ident, microcode: u32) -> bool { ROWS.iter().find(|&&(k, _)| k == key).is_some_and(|&(_, min)| microcode >= min) } +/// The microcode revision from which `INVLPG` drops a global translation in +/// every PCID, on a CPU Intel names for "INVLPG May Invalidate Global TLB +/// Entries Only For The Current PCID": ADL063 (682436-040), on every signature +/// of its Table 2-1; RPL043 (740518-031), on every signature of its Table 2-1 +/// but the 6P+0E die's, where its summary reads N/A; and Alder Lake-N, which +/// its own specification update (764616-004) does not list and whose fix Intel +/// names beside the others. The revisions are Intel's, posted to linux-kernel +/// in Message-ID `20240325231300.qrltbzf6twm43ftb@desk`, one per model: no +/// specification update carries a revision. +pub(crate) fn invlpg_fixed_at(id: &Ident) -> Option { + const ROWS: &[(u32, u32, i32)] = &[ + (ALDERLAKE, 0x2, 0x2e), + (ALDERLAKE, 0x5, 0x2e), + (ALDERLAKE_L, 0x3, 0x42c), + (ALDERLAKE_L, 0x4, 0x42c), + (ATOM_GRACEMONT, 0x0, 0x11), + (RAPTORLAKE, 0x1, 0x118), + (RAPTORLAKE_P, 0x2, 0x4117), + (RAPTORLAKE_P, 0x3, 0x4117), + (RAPTORLAKE_S, 0x2, 0x2e), + ]; + if id.vendor != Vendor::Intel || id.family != 6 { + return None; + } + ROWS.iter() + .find(|&&(model, stepping, _)| model == id.model && stepping == id.stepping) + .map(|&(_, _, fixed)| fixed) +} + /// `test_intel` for `PERF_MSR_SMI` (`arch/x86/events/msr.c:40-122`): an Intel /// family 6 CPU of one of these models counts its SMIs in `MSR_SMI_COUNT`. pub(crate) fn counts_smis(id: &Ident) -> bool { diff --git a/kernel/loom/tests/counters_round.rs b/kernel/loom/tests/counters_round.rs index 866407c82bb..5ef886da49b 100644 --- a/kernel/loom/tests/counters_round.rs +++ b/kernel/loom/tests/counters_round.rs @@ -20,7 +20,7 @@ use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; use kernel_loom::seqlock::Published; -use kernel_loom::shootdown::Shootdown; +use kernel_loom::shootdown::{Reach, Shootdown}; use loom::sync::Arc; /// What the CPU's block held before the round, and what it answers the round with. @@ -44,11 +44,11 @@ fn a_cpu_read_as_answered_is_read_with_its_answer() { let rounds = Arc::new(Shootdown::new()); let block = Arc::new(Published::new()); block.publish(BEFORE); - let generation = rounds.issue(); + let generation = rounds.issue(Reach::Narrow); let cpu1 = { let (rounds, block) = (rounds.clone(), block.clone()); - loom::thread::spawn(move || rounds.serve(1, || block.publish(ANSWER))) + loom::thread::spawn(move || rounds.serve(1, |_| block.publish(ANSWER))) }; for _ in 0..POLLS { diff --git a/kernel/loom/tests/tlb_shootdown.rs b/kernel/loom/tests/tlb_shootdown.rs index 939d6948a96..d956e94e9da 100644 --- a/kernel/loom/tests/tlb_shootdown.rs +++ b/kernel/loom/tests/tlb_shootdown.rs @@ -10,6 +10,8 @@ //! to mean and the only one expressible here. So `UNMAPPED` after an //! acknowledged shootdown is "cpu 1 can no longer reach the page the initiator //! is about to free", and `MAPPED` is the use-after-free this stage removes. +//! A kernel-half entry is a second cell, re-walked only by a wide answer, as a +//! global translation is dropped only by the flush that drops global ones. //! //! The edge the property rests on is `serve`'s load of what it owes: it //! synchronizes with `issue`'s release, and that is what puts the initiator's @@ -39,7 +41,7 @@ use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; -use kernel_loom::shootdown::Shootdown; +use kernel_loom::shootdown::{Reach, Shootdown}; use loom::sync::Arc; use loom::sync::atomic::{AtomicU64, Ordering::Relaxed}; @@ -85,10 +87,14 @@ static ACKED: AtomicBool = AtomicBool::new(false); struct Machine { shootdown: Shootdown, - /// The page-table entry the initiator unmaps. + /// The user-half page-table entry the initiator unmaps. pte: AtomicU64, - /// What cpu 1's last page walk found. A flush re-walks. + /// What cpu 1's last page walk of it found. Every flush re-walks. tlb: AtomicU64, + /// The kernel-half entry, whose translation is global. + kernel_pte: AtomicU64, + /// What cpu 1's last walk of it found. Only a wide flush re-walks. + kernel_tlb: AtomicU64, } impl Machine { @@ -97,28 +103,47 @@ impl Machine { shootdown: Shootdown::new(), pte: AtomicU64::new(MAPPED), tlb: AtomicU64::new(MAPPED), + kernel_pte: AtomicU64::new(MAPPED), + kernel_tlb: AtomicU64::new(MAPPED), }) } + /// cpu 1's flush, as far as `reach` asks. + fn flush(&self, reach: Reach) { + self.tlb.store(self.pte.load(Relaxed), Relaxed); + if reach == Reach::Wide { + self.kernel_tlb.store(self.kernel_pte.load(Relaxed), Relaxed); + } + } + + /// One serve of vector 0xFE's body on cpu 1. + fn serve(&self) { + self.shootdown.serve(1, |reach| self.flush(reach)); + } + /// cpu 1's side: take the interrupt and run vector 0xFE's body. fn take_interrupts(&self) { for _ in 0..SERVES { loom::thread::yield_now(); - self.shootdown - .serve(1, || self.tlb.store(self.pte.load(Relaxed), Relaxed)); + self.serve(); } } - /// The initiator's side, up to the point where it would free the pages. - /// `None` when the bounded poll ran out, which is a schedule this model - /// says nothing about. - fn unmap_and_wait(&self) -> Option { - self.pte.store(UNMAPPED, Relaxed); - let generation = self.shootdown.issue(); + /// The initiator's side, up to the point where it would free the pages: + /// `pte` written for a user-half change, `kernel_pte` for a kernel-half + /// one, and what cpu 1 holds of it once it has answered. `None` when the + /// bounded poll ran out, which is a schedule this model says nothing about. + fn unmap_and_wait(&self, reach: Reach) -> Option { + let (pte, tlb) = match reach { + Reach::Narrow => (&self.pte, &self.tlb), + Reach::Wide => (&self.kernel_pte, &self.kernel_tlb), + }; + pte.store(UNMAPPED, Relaxed); + let generation = self.shootdown.issue(reach); for _ in 0..POLLS { if self.shootdown.served(1, generation) { ACKED.store(true, SeqCst); - return Some(self.tlb.load(Relaxed)); + return Some(tlb.load(Relaxed)); } loom::thread::yield_now(); } @@ -138,7 +163,7 @@ fn an_acknowledged_flush_postdates_the_page_table_write() { loom::thread::spawn(move || m.take_interrupts()) }; - if let Some(reachable) = m.unmap_and_wait() { + if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) { assert_eq!( reachable, UNMAPPED, "cpu 1 acknowledged the shootdown while still holding a \ @@ -174,10 +199,10 @@ fn one_serve_answers_two_concurrent_shootdowns() { let second = { let m = m.clone(); - loom::thread::spawn(move || m.unmap_and_wait()) + loom::thread::spawn(move || m.unmap_and_wait(Reach::Narrow)) }; - if let Some(reachable) = m.unmap_and_wait() { + if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) { assert_eq!(reachable, UNMAPPED, "the first initiator freed a page cpu 1 could still reach"); } if let Some(reachable) = second.join().unwrap() { @@ -213,15 +238,15 @@ fn an_initiator_answers_while_it_waits() { // cpu 0 unmaps and answers for itself; cpu 1 has not issued yet, so the // generation cpu 0 publishes is its own. - let g0 = s.issue(); - s.serve(0, || {}); + let g0 = s.issue(Reach::Narrow); + s.serve(0, |_| {}); // cpu 1 unmaps while cpu 0 is between its own `serve` and its wait. - let g1 = s.issue(); + let g1 = s.issue(Reach::Narrow); // cpu 0's wait is not satisfied — cpu 1 has not flushed — and that turn // is nevertheless where cpu 1's answer comes from. - assert!(!s.wait_turn(0, 1, g0, || {})); + assert!(!s.wait_turn(0, 1, g0, |_| {})); assert!( s.served(0, g1), "cpu 0 is waiting for cpu 1 and has stopped answering it, which is \ @@ -229,9 +254,9 @@ fn an_initiator_answers_while_it_waits() { ); // And cpu 1's own wait then has an answer to find. - s.serve(1, || {}); - assert!(s.wait_turn(1, 0, g1, || {})); - assert!(s.wait_turn(0, 1, g0, || {})); + s.serve(1, |_| {}); + assert!(s.wait_turn(1, 0, g1, |_| {})); + assert!(s.wait_turn(0, 1, g0, |_| {})); }); } @@ -245,16 +270,16 @@ fn an_initiator_answers_while_it_waits() { fn a_nested_serve_is_not_undone_by_the_one_it_interrupted() { model(|| { let s = Arc::new(Shootdown::new()); - let first = s.issue(); + let first = s.issue(Reach::Narrow); let nested = { let s = Arc::clone(&s); loom::thread::spawn(move || { - let later = s.issue(); - s.serve(1, || {}); + let later = s.issue(Reach::Narrow); + s.serve(1, |_| {}); later }) }; - s.serve(1, || {}); + s.serve(1, |_| {}); let later = nested.join().unwrap(); assert!(s.served(1, first)); assert!( @@ -264,3 +289,140 @@ fn a_nested_serve_is_not_undone_by_the_one_it_interrupted() { ); }); } + +/// A kernel-half shootdown acknowledged means cpu 1 has dropped its global +/// translation since the write. +/// +/// The edge is `issue`'s count of wide generations, raised before the +/// generation exists: a serve that owes the generation reads the count, so its +/// answer is wide. Reds when the count is raised after the generation is +/// issued, where a serve can owe it and still answer narrow. +#[test] +fn an_acknowledged_kernel_half_shootdown_dropped_the_global_translation() { + ACKED.store(false, SeqCst); + model(|| { + let m = Machine::new(); + + let target = { + let m = m.clone(); + loom::thread::spawn(move || m.take_interrupts()) + }; + + if let Some(reachable) = m.unmap_and_wait(Reach::Wide) { + assert_eq!( + reachable, UNMAPPED, + "cpu 1 acknowledged a kernel-half shootdown while still holding the \ + global translation the initiator changed", + ); + } + + target.join().unwrap(); + }); + assert!( + ACKED.load(SeqCst), + "no interleaving completed the wait, so the assertion never ran", + ); +} + +/// One serve may answer a user-half and a kernel-half shootdown at once, and +/// its answer is then the wide one: the user-half initiator's page is gone +/// either way, and the kernel-half initiator's global translation with it. +#[test] +fn one_serve_answering_a_user_half_and_a_kernel_half_shootdown_answers_wide() { + ACKED.store(false, SeqCst); + model(|| { + let m = Machine::new(); + + let target = { + let m = m.clone(); + loom::thread::spawn(move || m.take_interrupts()) + }; + + let kernel = { + let m = m.clone(); + loom::thread::spawn(move || m.unmap_and_wait(Reach::Wide)) + }; + + if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) { + assert_eq!(reachable, UNMAPPED, "the user-half initiator freed a page cpu 1 could still reach"); + } + if let Some(reachable) = kernel.join().unwrap() { + assert_eq!(reachable, UNMAPPED, "the kernel-half initiator's global translation outlived cpu 1's answer"); + } + + target.join().unwrap(); + }); + assert!( + ACKED.load(SeqCst), + "no interleaving completed a wait, so the assertions never ran", + ); +} + +/// A serve nested inside a wide one that has not yet flushed answers wide +/// itself: the record of a wide answer is raised only once its flush is done. +/// +/// The two serves are two threads on cpu 1, so loom places the nested one at +/// every point of the outer one. Reds when the record is raised before the +/// flush, where the nested serve reads it, answers narrow, and publishes the +/// generation while the global translation is still held. +#[test] +fn a_serve_nested_in_an_unfinished_wide_one_answers_wide() { + ACKED.store(false, SeqCst); + model(|| { + let m = Machine::new(); + + let outer = { + let m = m.clone(); + loom::thread::spawn(move || m.serve()) + }; + let nested = { + let m = m.clone(); + loom::thread::spawn(move || m.serve()) + }; + + if let Some(reachable) = m.unmap_and_wait(Reach::Wide) { + assert_eq!( + reachable, UNMAPPED, + "a nested serve published the kernel-half shootdown while the wide \ + answer it nested in had not yet dropped the global translation", + ); + } + + outer.join().unwrap(); + nested.join().unwrap(); + }); + assert!( + ACKED.load(SeqCst), + "no interleaving completed the wait, so the assertion never ran", + ); +} + +/// A serve owing only user-half shootdowns answers narrow, before and after a +/// kernel-half one, which it answers wide once: a user-half change keeps the +/// kernel's global translations. +/// +/// **Not an interleaving question**, as `an_initiator_answers_while_it_waits` +/// is not; reds when a serve answers wide with no wide generation owed. +#[test] +fn only_a_kernel_half_shootdown_asks_a_wide_answer() { + model(|| { + let s = Shootdown::new(); + let reach_of = |s: &Shootdown| { + let mut answered = None; + s.serve(1, |reach| answered = Some(reach)); + answered.expect("serve runs its flush") + }; + + s.issue(Reach::Narrow); + assert_eq!(reach_of(&s), Reach::Narrow, "a user-half shootdown was answered wide"); + s.issue(Reach::Wide); + s.issue(Reach::Narrow); + assert_eq!(reach_of(&s), Reach::Wide, "a kernel-half shootdown was answered narrow"); + s.issue(Reach::Narrow); + assert_eq!( + reach_of(&s), + Reach::Narrow, + "a user-half shootdown after an answered kernel-half one was answered wide", + ); + }); +} diff --git a/kernel/src/arch/aarch64/tlb.rs b/kernel/src/arch/aarch64/tlb.rs index b8a028933cc..8d302fc90d7 100644 --- a/kernel/src/arch/aarch64/tlb.rs +++ b/kernel/src/arch/aarch64/tlb.rs @@ -12,7 +12,7 @@ use core::sync::atomic::{AtomicU64, Ordering}; -use crate::invalidation::Origin; +use crate::invalidation::{Half, Origin}; /// Issuer-side census, as x86-64's counts it; there is no receiver side. static ISSUED: [AtomicU64; Origin::COUNT] = [const { AtomicU64::new(0) }; Origin::COUNT]; @@ -61,7 +61,7 @@ pub(super) fn all(origin: Origin) { /// Nothing to drop: every caller follows an `AddressSpace` edit whose own /// `page` or `asid` already reached every CPU before it returned. -pub fn shootdown(_origin: Origin) {} +pub fn shootdown(_origin: Origin, _half: Half) {} /// Nothing to answer: no CPU waits on another's acknowledgement here. pub fn poll() {} diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 23062f3ebc5..f1bf295754c 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -20,7 +20,7 @@ use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; -use toyos_cpuvuln::HwpFacts; +use toyos_cpuvuln::{HwpFacts, InvlpgFacts, Vendor}; use super::cpu; use crate::log; @@ -389,7 +389,7 @@ impl PcidActive { /// against what the BSP said. Recomputed per CPU rather than trusted from /// the BSP, so a divergent machine names the CPU instead of faulting blind. fn declaration(cpu_id: u32) -> u64 { - let have = supported(); + let have = supported(cpu_id); let missing = CR4_REQUIRED & !have; assert!( missing == 0, @@ -433,7 +433,7 @@ fn declaration(cpu_id: u32) -> u64 { } /// The `CR4` bits this CPU will accept, as CPUID reports them. -fn supported() -> u64 { +fn supported(cpu_id: u32) -> u64 { const CPUID_1_EDX: [(u32, u64); 6] = [ (2, cr4::DE), (6, cr4::PAE), @@ -466,12 +466,39 @@ fn supported() -> u64 { } } // PCID without INVPCID is not worth having: nothing could flush by ASID. - if ecx1 & (1 << 17) != 0 && ebx7 & (1 << 10) != 0 { + if ecx1 & (1 << 17) != 0 && ebx7 & (1 << 10) != 0 && !invlpg_misses_global(cpu_id, ecx1) { have |= cr4::PCIDE; } have } +/// Whether this CPU's `INVLPG` leaves another PCID's global translation +/// (`toyos_cpuvuln::invlpg_misses_global`), which every global discharge is +/// (`paging::Owed::discharge`): PCID is withheld where it does. The facts are +/// logged where a revision was read, so a reader can hold the read to another's. +fn invlpg_misses_global(cpu_id: u32, cpuid_1_ecx: u32) -> bool { + let vendor = cpu::vendor(); + let facts = InvlpgFacts { + vendor, + signature: cpu::cpuid(1, 0).0, + cpuid_1_ecx, + microcode: (vendor == Vendor::Intel).then(cpu::intel_microcode_revision), + }; + let misses = toyos_cpuvuln::invlpg_misses_global(&facts); + if let Some(microcode) = facts.microcode { + log!( + "control_regs: cpu{cpu_id} signature {:#x} microcode {microcode:#x}{}", + facts.signature, + if misses { + ": PCID withheld, Intel's erratum ADL063/RPL043 leaves another PCID's global translation" + } else { + "" + }, + ); + } + misses +} + /// Logs what this CPU holds, then asserts it against the declaration — /// logged first so a failing CPU still leaves its value in the log. fn self_check(cpu_id: u32, declared_cr4: u64) { diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 65cf3444524..0a3b9f7f802 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -134,6 +134,18 @@ pub fn vendor() -> toyos_cpuvuln::Vendor { toyos_cpuvuln::Vendor::from_id(&core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4])) } +/// This CPU's microcode revision as Intel defines its read: `IA32_BIOS_SIGN_ID` +/// (MSR 8BH) written 0, CPUID.01H run, and the MSR's upper dword read (SDM +/// Vol. 3A §12.11.7.1). An Intel CPU's alone: the procedure is Intel's. +pub fn intel_microcode_revision() -> u32 { + const IA32_BIOS_SIGN_ID: u32 = 0x8B; + // SAFETY: MSR 8BH is architectural on every Intel 64 processor (SDM Vol. 4 + // Table 2-2), and 0 is the value the procedure writes before its read. + unsafe { wrmsr(IA32_BIOS_SIGN_ID, 0) }; + let _ = cpuid(1, 0); + (rdmsr(IA32_BIOS_SIGN_ID) >> 32) as u32 +} + /// CPUID.6's EAX and ECX, the thermal and power leaf; zero where CPUID.0 /// does not reach it, as `init_scattered_cpuid_features` reads it: a level /// past the range's own is no level. @@ -248,6 +260,8 @@ pub enum Invpcid { SinglePcid = 1, /// Every entry in every PCID, global pages included. AllIncludingGlobal = 2, + /// Every entry in every PCID, global pages excepted. + AllButGlobal = 3, } /// Discard TLB entries by descriptor type; both instruction faults are excluded by the signature ([`Invpcid`], [`PcidActive`]). diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 5497911c22b..ce37c92feb0 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -210,7 +210,9 @@ impl Owed { fn discharge(self, target: Cr3) { match self { Self::Nothing => {} - // `INVLPG` drops the page's global translation in every PCID. + // `INVLPG` drops the page's global translation in every PCID, but + // on a CPU with Intel's ADL063/RPL043, which `control_regs` declares + // without PCID. Self::Address { va, prior } if prior & PAGE_GLOBAL != 0 => crate::arch::cpu::invlpg(va), Self::Address { va, .. } => { if let Some(have) = pcid_active() { @@ -225,7 +227,7 @@ impl Owed { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::SinglePcid, target.pcid(), 0); } else if Cr3::current().phys() == target.phys() { - flush_tlb_all(); + flush_tlb_user(); } } } @@ -273,7 +275,8 @@ pub fn pcid_active() -> Option { crate::arch::control_regs::PcidActive::ask() } -/// Flush all TLB entries on this CPU, all PCIDs, global ones included. +/// Flush all TLB entries on this CPU, all PCIDs, global ones included: what a +/// kernel-half change owes. pub fn flush_tlb_all() { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::AllIncludingGlobal, 0, 0); @@ -282,6 +285,18 @@ pub fn flush_tlb_all() { } } +/// Flush every TLB entry on this CPU but the global ones, all PCIDs: what a +/// user-half change owes, since no user leaf is global. +pub fn flush_tlb_user() { + if let Some(have) = pcid_active() { + crate::arch::cpu::invpcid(have, Invpcid::AllButGlobal, 0, 0); + } else { + // SAFETY: the value is exactly what `read_cr3` just read on this CPU, + // so writing it back is a valid CR3 write that only flushes the TLB. + unsafe { crate::arch::cpu::write_cr3(crate::arch::cpu::read_cr3()) }; + } +} + /// CR3 register value: PML4 physical address | PCID. #[derive(Clone, Copy)] pub struct Cr3(u64); @@ -362,7 +377,7 @@ fn alloc_pcid() -> Option { match pool.alloc() { Alloc::Ready(p) => return Some(PcidGuard(p)), Alloc::NeedsFlush => { - crate::arch::tlb::shootdown(crate::invalidation::Origin::Pcid); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Pcid, crate::invalidation::Half::User); pool.reclaim(); } Alloc::Exhausted => return None, @@ -899,7 +914,7 @@ pub fn map_mmio(phys: u64, size: u64, policy: MmioPolicy) -> crate::mm::Mmio { driven.push((phys, phys + size)); } } - crate::arch::tlb::shootdown(crate::invalidation::Origin::Mmio); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Mmio, crate::invalidation::Half::Kernel); // Read back off the table and logged beside firmware's MTRR verdict: the // boot's own evidence that no register window trusts firmware. let installed = @@ -919,23 +934,24 @@ const SCRATCH: u64 = 0u64.wrapping_sub(PAGE_2M); /// `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`: a [`kernel_leaf`] at [`SCRATCH`] over one /// frame, read so this CPU caches it global; the leaf pointed at a second frame -/// and invalidated as `how` asks; read again. Answers which frame that read -/// saw. The kernel's lock keeps it on one CPU and interrupts stay closed, so no -/// flush but `how`'s lands between the two reads. +/// and read again with nothing invalidated, the control; invalidated as `how` +/// asks; read a third time. Answers which frame the third read saw, or that the +/// control already saw the second. The kernel's lock keeps it on one CPU and +/// interrupts stay closed, so no flush but `how`'s lands between the reads. #[cfg(feature = "test-actuators")] pub fn debug_global_leaf(how: u64) -> u64 { + use crate::invalidation::{Half, Origin}; use toyos_abi::syscall::debug_action as da; use toyos_abi::syscall::SyscallError; enum How { - Withheld, Discharged, - ShotDown, + ShotDown(Half), } let how = match how { - da::GLOBAL_LEAF_WITHHELD => How::Withheld, da::GLOBAL_LEAF_DISCHARGED => How::Discharged, - da::GLOBAL_LEAF_SHOT_DOWN => How::ShotDown, + da::GLOBAL_LEAF_KERNEL_SHOT_DOWN => How::ShotDown(Half::Kernel), + da::GLOBAL_LEAF_USER_SHOT_DOWN => How::ShotDown(Half::User), _ => return SyscallError::InvalidArgument.to_u64(), }; let (Some(old), Some(new)) = (crate::mm::pmm::alloc_page(), crate::mm::pmm::alloc_page()) else { @@ -953,7 +969,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { // frames this call owns, or a translation of one of them. let read = || unsafe { core::ptr::read_volatile(SCRATCH as *const u64) }; - let seen = { + let (held, seen) = { let mut space = kernel().lock(); let _closed = crate::arch::IrqGuard::close(); let target = space.root(); @@ -966,26 +982,31 @@ pub fn debug_global_leaf(how: u64) -> u64 { let first = read(); assert!(first == old_phys, "debug_global_leaf: a fresh leaf over {old_phys:#x} read {first:#x}"); let owed = pd.write_pde(idx, SCRATCH, new_phys | leaf); + let held = read(); match how { - // The control: what this CPU still holds with nothing invalidated. - How::Withheld => {} How::Discharged => owed.discharge(target), - How::ShotDown => { - owed.subsumed_by_flush(); - crate::arch::tlb::shootdown(crate::invalidation::Origin::Staged); + // The write's own invalidation is withheld: the arm asks what the shootdown drops. + How::ShotDown(half) => { + let _ = owed; + crate::arch::tlb::shootdown(Origin::Staged, half); } } let seen = read(); pd.write_pde(idx, SCRATCH, 0).discharge(target); - seen + (held, seen) }; // No CPU keeps a translation of either frame past its free. - crate::arch::tlb::shootdown(crate::invalidation::Origin::Staged); + crate::arch::tlb::shootdown(Origin::Staged, Half::Kernel); drop((old, new)); - match seen { - _ if seen == old_phys => da::GLOBAL_LEAF_SAW_OLD, - _ if seen == new_phys => da::GLOBAL_LEAF_SAW_NEW, - _ => panic!("debug_global_leaf: the leaf over {old_phys:#x} then {new_phys:#x} read {seen:#x}"), + let old_frame = |at: u64| match at { + _ if at == old_phys => true, + _ if at == new_phys => false, + _ => panic!("debug_global_leaf: the leaf over {old_phys:#x} then {new_phys:#x} read {at:#x}"), + }; + match (old_frame(held), old_frame(seen)) { + (false, _) => da::GLOBAL_LEAF_UNHELD, + (true, true) => da::GLOBAL_LEAF_SAW_OLD, + (true, false) => da::GLOBAL_LEAF_SAW_NEW, } } diff --git a/kernel/src/arch/x86_64/pio.rs b/kernel/src/arch/x86_64/pio.rs index c27e1d7615b..320d23a3426 100644 --- a/kernel/src/arch/x86_64/pio.rs +++ b/kernel/src/arch/x86_64/pio.rs @@ -146,7 +146,7 @@ pub struct TakenBack(()); /// shootdown returns only once every other CPU has answered it from Ring 0, /// so no thread that was in Ring 3 before is there still. pub fn take_back(_stopping: &crate::quiesce::Stopping) -> TakenBack { - super::tlb::shootdown(crate::invalidation::Origin::Stop); + super::tlb::shootdown(crate::invalidation::Origin::Stop, crate::invalidation::Half::User); TakenBack(()) } diff --git a/kernel/src/arch/x86_64/smi_cmd.rs b/kernel/src/arch/x86_64/smi_cmd.rs index 3f17585892f..c513f05aa4a 100644 --- a/kernel/src/arch/x86_64/smi_cmd.rs +++ b/kernel/src/arch/x86_64/smi_cmd.rs @@ -54,7 +54,7 @@ use toyos_userbound::{KeptCommands, Mediated, Ports, Undeclared}; use super::pio::{self, Declared, Slot, TakenBack}; use super::{apic, cpu, percpu, IrqGuard}; -use crate::shootdown::Shootdown; +use crate::shootdown::{Reach, Shootdown}; use crate::sync::Lock; use crate::time::{Duration, DEAF_CPU}; use kernel::bootwrite::{self, Turn}; @@ -143,7 +143,7 @@ pub fn write(value: u8) -> Option { let asked_from = percpu::cpu_id(); ASKED.store(value, Relaxed); TAKEN_NS.store(UNTAKEN, Relaxed); - let generation = ROUND.issue(); + let generation = ROUND.issue(Reach::Narrow); // The boot processor's own ask, unless a kick taken since the issue has answered it already. answer(); if !ROUND.served(BOOT as usize, generation) { @@ -196,7 +196,7 @@ fn answer() { if percpu::cpu_id() != BOOT { return; } - ROUND.serve_if_owed(BOOT as usize, || { + ROUND.serve_if_owed(BOOT as usize, |_| { let port = PORT.get().expect("smi_cmd: a write asked of a machine whose FADT names no SMI_CMD").port(0); let value = ASKED.load(Relaxed); let smi = || super::counters::read().smi.unwrap_or(UNREAD); diff --git a/kernel/src/arch/x86_64/tlb.rs b/kernel/src/arch/x86_64/tlb.rs index 902752dc763..6337cd9c9f6 100644 --- a/kernel/src/arch/x86_64/tlb.rs +++ b/kernel/src/arch/x86_64/tlb.rs @@ -9,17 +9,22 @@ //! **The target set is every other CPU the machine brought up**, not the CPUs //! sharing the initiator's address space: nothing a process does puts a CPU //! into it or takes one out. +//! +//! **A user-half change keeps global translations**: its flush is `INVPCID` +//! type 3 or a `CR3` reload, and only a kernel-half change, whose leaves are +//! global, asks the flush that drops them (SDM Vol. 3A §4.10.4.1, APM Vol. 2 +//! §5.5.2). An answer covering both is the wider. use core::sync::atomic::{AtomicU64, Ordering}; -use crate::shootdown::{Generation, Shootdown}; +use crate::shootdown::{Generation, Reach, Shootdown}; use super::{apic, percpu}; use crate::smp; static SHOOTDOWN: Shootdown = Shootdown::new(); -use crate::invalidation::Origin; +use crate::invalidation::{Half, Origin}; /// Issuer-side census; `irq_census`'s `tlb` column is the receiver side, and a /// delivery the two disagree on is an uncounted issuing path. @@ -58,20 +63,25 @@ pub fn census(say: &mut impl FnMut(core::fmt::Arguments<'_>)) { const SPINS_PER_DEADLINE_CHECK: u32 = 1024; /// Write the page table, then call this, then free — it returns only once every -/// CPU has flushed. The local-flush early return is uncounted: no IPI, no wait. -pub fn shootdown(origin: Origin) { +/// CPU has flushed what a change to `half` owes. The local-flush early return +/// is uncounted: no IPI, no wait. +pub fn shootdown(origin: Origin, half: Half) { + let reach = match half { + Half::User => Reach::Narrow, + Half::Kernel => Reach::Wide, + }; let cpus = smp::cpu_count(); if !smp::answering() || cpus <= 1 { - crate::mm::paging::flush_tlb_all(); + flush(reach); return; } // Counted before the IPI, so a delivery can never precede its issue's count. ISSUED[origin as usize].fetch_add(1, Ordering::Relaxed); let began = crate::clock::nanos_since_boot(); let me = percpu::cpu_id() as usize; - let generation = SHOOTDOWN.issue(); + let generation = SHOOTDOWN.issue(reach); // This CPU answers itself locally instead of by self-IPI. - SHOOTDOWN.serve(me, crate::mm::paging::flush_tlb_all); + SHOOTDOWN.serve(me, flush); apic::tlb_ipi(); for cpu in 0..cpus { if cpu as usize != me { @@ -103,7 +113,7 @@ pub fn bench() { let mut took = [0u64; ROUNDS]; for sample in took.iter_mut() { let began = crate::clock::nanos_since_boot(); - shootdown(Origin::Bench); + shootdown(Origin::Bench, Half::User); *sample = crate::clock::nanos_since_boot().saturating_sub(began); } took.sort_unstable(); @@ -123,7 +133,7 @@ pub fn bench() { fn wait_for(me: usize, cpu: u32, generation: Generation) { let mut spins = 0u32; let mut deadline = None; - while !SHOOTDOWN.wait_turn(me, cpu as usize, generation, crate::mm::paging::flush_tlb_all) { + while !SHOOTDOWN.wait_turn(me, cpu as usize, generation, flush) { core::hint::spin_loop(); spins += 1; if spins == SPINS_PER_DEADLINE_CHECK { @@ -142,11 +152,20 @@ fn wait_for(me: usize, cpu: u32, generation: Generation) { } } +/// What `reach` asks of this CPU: every translation, or every one but the +/// global ones a user-half change leaves. +fn flush(reach: Reach) { + match reach { + Reach::Wide => crate::mm::paging::flush_tlb_all(), + Reach::Narrow => crate::mm::paging::flush_tlb_user(), + } +} + /// Vector 0xFE's whole body: flush this CPU and say which generation it covers. pub fn serve_ipi() { let cpu = percpu::cpu_id() as usize; - SHOOTDOWN.serve(cpu, || { - crate::mm::paging::flush_tlb_all(); + SHOOTDOWN.serve(cpu, |reach| { + flush(reach); stage_ack_delay(); }); } @@ -161,17 +180,18 @@ pub fn poll() { return; } let cpu = percpu::cpu_id() as usize; - SHOOTDOWN.serve_if_owed(cpu, || { - crate::mm::paging::flush_tlb_all(); + SHOOTDOWN.serve_if_owed(cpu, |reach| { + flush(reach); stage_ack_delay(); }); } /// Settle every shootdown issued before this CPU could answer one; called -/// once after the machine is released. +/// once after the machine is released. Wide whatever is owed: a kernel-half +/// change made before the machine answered issued no generation to say so. pub fn join() { let cpu = percpu::cpu_id() as usize; - SHOOTDOWN.serve(cpu, crate::mm::paging::flush_tlb_all); + SHOOTDOWN.serve(cpu, |_| crate::mm::paging::flush_tlb_all()); } #[cfg(not(feature = "test-actuators"))] @@ -244,7 +264,7 @@ pub fn debug_arm_ack_delay(nanos: u64) -> u64 { delay::TARGET.store(cpu, Ordering::Relaxed); open_arm_window(); let began = crate::clock::nanos_since_boot(); - shootdown(Origin::Staged); + shootdown(Origin::Staged, Half::User); least = least.min(crate::clock::nanos_since_boot().saturating_sub(began)); } }); diff --git a/kernel/src/counters.rs b/kernel/src/counters.rs index 7aef37d90ec..6017190b963 100644 --- a/kernel/src/counters.rs +++ b/kernel/src/counters.rs @@ -25,7 +25,7 @@ use crate::arch::{irqchip, percpu, IrqGuard}; use crate::irq_census::Source; use crate::scheduler::{Parkable, MAX_CPUS}; use crate::seqlock::Published; -use crate::shootdown::{Generation, Shootdown}; +use crate::shootdown::{Generation, Reach, Shootdown}; use crate::sync::Lock; use crate::time::{Budget, Cadence, Deadline, Duration, Instant}; use crate::user_ptr::UserBytesMut; @@ -151,7 +151,7 @@ fn answer(kick: bool) { if deaf::DEAF.load(core::sync::atomic::Ordering::Relaxed) == me { return; } - ROUNDS.serve_if_owed(me, || BLOCKS[me].publish(sample(me))) + ROUNDS.serve_if_owed(me, |_| BLOCKS[me].publish(sample(me))) }; if served { ANSWERED.post_in_place(); @@ -175,7 +175,7 @@ pub fn read(rights: Rights, out: &mut UserBytesMut) -> Result (generation, issued, false), _ => { - let generation = ROUNDS.issue(); + let generation = ROUNDS.issue(Reach::Narrow); *last = Some((generation, now)); (generation, now, true) } diff --git a/kernel/src/invalidation.rs b/kernel/src/invalidation.rs index 4bfd094ceca..13bc01b8ab8 100644 --- a/kernel/src/invalidation.rs +++ b/kernel/src/invalidation.rs @@ -1,5 +1,6 @@ -//! Why a machine-wide translation invalidation was asked for: the census every -//! architecture's shootdown (`arch::tlb`) counts its callers by. +//! Why a machine-wide translation invalidation was asked for, the census every +//! architecture's shootdown (`arch::tlb`) counts its callers by, and which half +//! of the address space it answers for. /// Which path issued a shootdown, so the census names who pays: `Dlopen` (a /// `Shared` window or rollback unmap), `Pcid` (pool reclaim), `Mmio`, `Unmap` @@ -24,6 +25,17 @@ pub enum Origin { Stop, } +/// Which half of every address space a shootdown's write changed: x86-64's +/// kernel-half leaves are global, so only a change there asks a flush that +/// drops global translations. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Half { + User, + // The kernel half is global on x86-64 alone, so AArch64 builds a variant it never issues. + #[allow(dead_code)] + Kernel, +} + impl Origin { pub const COUNT: usize = 8; /// Order matches the variants; `tests/toyos.rs`'s `irq_census_conservation` reads the line back. diff --git a/kernel/src/mm/unmapped.rs b/kernel/src/mm/unmapped.rs index 5741e04619c..29b9c9655da 100644 --- a/kernel/src/mm/unmapped.rs +++ b/kernel/src/mm/unmapped.rs @@ -16,7 +16,7 @@ impl Unmapped { impl Drop for Unmapped { fn drop(&mut self) { - crate::arch::tlb::shootdown(crate::invalidation::Origin::Unmap); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Unmap, crate::invalidation::Half::User); // SAFETY: the wrapped value is never taken before this drop. unsafe { ManuallyDrop::drop(&mut self.0) }; } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 78bfe36fbe2..2318afee90c 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -661,7 +661,7 @@ pub fn revoke_pipe_maps(maps: &mut Vec, pt: &PageTables, pipe: pipe::Pi }); } // Outside the block: it waits, and a sibling can be spinning on this lock. - crate::arch::tlb::shootdown(crate::invalidation::Origin::Pipe); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Pipe, crate::invalidation::Half::User); } /// One live `mmap` and its physical pages; the range's registration in the address space's `regions` is separate (placement search, `munmap`). diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 068f41b8956..f6b77013777 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -3,6 +3,8 @@ //! Compiled a second time into `kernel/loom/` against loom's atomics, so this file must hold no `crate::` references. //! The read must happen before the answer, or a target could publish a generation its answer does not cover; //! and an initiator that sees a generation served sees everything that answer wrote. +//! A generation is narrow or wide, and an answer reaches as far as the widest generation it covers: one +//! answer covers every generation owed, so the reach is the target's to read and never the IPI's. #[cfg(not(feature = "loom"))] use core::sync::atomic::{AtomicU64, Ordering}; @@ -31,9 +33,20 @@ const SERVED: Ordering = Ordering::Relaxed; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct Generation(u64); +/// How far an answer must reach: what a wide generation asks of it, a narrow one does not. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Reach { + Narrow, + Wide, +} + pub struct Shootdown { requested: AtomicU64, + /// Wide generations issued, each counted before it is issued. + wide: AtomicU64, flushed: [AtomicU64; MAX_CPUS], + /// Per CPU, `wide` as the last wide answer it finished read it. + widened: [AtomicU64; MAX_CPUS], } impl Shootdown { @@ -42,7 +55,9 @@ impl Shootdown { pub const fn new() -> Self { Self { requested: AtomicU64::new(0), + wide: AtomicU64::new(0), flushed: [const { AtomicU64::new(0) }; MAX_CPUS], + widened: [const { AtomicU64::new(0) }; MAX_CPUS], } } @@ -53,20 +68,33 @@ impl Shootdown { pub fn new() -> Self { Self { requested: AtomicU64::new(0), + wide: AtomicU64::new(0), flushed: core::array::from_fn(|_| AtomicU64::new(0)), + widened: core::array::from_fn(|_| AtomicU64::new(0)), } } - /// Names the generation every other CPU now owes a flush for. - pub fn issue(&self) -> Generation { + /// Names the generation every other CPU now owes a flush for, of `reach`. + pub fn issue(&self, reach: Reach) -> Generation { + if reach == Reach::Wide { + // Before the generation exists, and Release: a serve that owes the generation reads this + // count, and an answer that reads the count follows the write it was raised for. + self.wide.fetch_add(1, Ordering::Release); + } // AcqRel, not Release: the acquire half stops a later load being hoisted above this fetch_add. Generation(self.requested.fetch_add(1, Ordering::AcqRel) + 1) } - /// A target's whole side of the protocol. - pub fn serve(&self, cpu: usize, flush: impl FnOnce()) { + /// A target's whole side of the protocol: `flush` is told how far its answer must reach. + pub fn serve(&self, cpu: usize, flush: impl FnOnce(Reach)) { let owed = self.requested.load(OWED); - flush(); + let wide = self.wide.load(Ordering::Acquire); + let reach = if wide > self.widened[cpu].load(Ordering::Acquire) { Reach::Wide } else { Reach::Narrow }; + flush(reach); + if reach == Reach::Wide { + // After the flush: a serve nested inside this one, reading this record first, answers wide itself. + self.widened[cpu].fetch_max(wide, Ordering::Release); + } // A max, not a store: a serve nested inside this one may already have published a later generation. self.flushed[cpu].fetch_max(owed, Ordering::Release); } @@ -83,7 +111,7 @@ impl Shootdown { } /// [`serve`](Self::serve) for a CPU not taking the interrupt; whether it served. - pub fn serve_if_owed(&self, cpu: usize, flush: impl FnOnce()) -> bool { + pub fn serve_if_owed(&self, cpu: usize, flush: impl FnOnce(Reach)) -> bool { let owed = self.owes(cpu); if owed { self.serve(cpu, flush); @@ -97,7 +125,7 @@ impl Shootdown { me: usize, cpu: usize, generation: Generation, - flush: impl FnOnce(), + flush: impl FnOnce(Reach), ) -> bool { // An initiator inside an `IrqGuard`, a handler or the tick's pass takes no IPI, so this CPU must serve itself or two initiators waiting on each other deadlock. self.serve_if_owed(me, flush); diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index 0b465c8c4b3..9a26e689e9a 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -293,7 +293,7 @@ pub(super) fn sys_dlopen(ctx: &crate::user_ptr::SyscallContext, path: &str, init // `map_window`'s shootdown reached only this CPU, so the rest of the // machine is told here. if matches!(lib.memory, crate::elf::LibMemory::Shared { .. }) { - crate::arch::tlb::shootdown(crate::invalidation::Origin::Dlopen); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Dlopen, crate::invalidation::Half::User); } if vaddr != lib.user_base { lib.user_base = vaddr; @@ -316,7 +316,7 @@ pub(super) fn sys_dlopen(ctx: &crate::user_ptr::SyscallContext, path: &str, init process::with_process_data(|_data| { pt.lock().free_and_unmap(base); }); - crate::arch::tlb::shootdown(crate::invalidation::Origin::Dlopen); + crate::arch::tlb::shootdown(crate::invalidation::Origin::Dlopen, crate::invalidation::Half::User); }); let lib_tls = lib.tls().and_then(toyos_elf::TlsSegment::occupied); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 691950a478d..84e835eeaed 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -885,20 +885,24 @@ pub mod debug_action { pub const RING0_FIRE_NEVER: u64 = 1; pub const RING0_FIRE_OTHER_SPAN: u64 = 2; /// Map a global 2 MiB kernel leaf over one frame and read through it, point - /// the leaf at a second frame, invalidate as the argument says, and read - /// again, all on one CPU with interrupts closed. The argument is - /// [`GLOBAL_LEAF_WITHHELD`], which invalidates nothing, - /// [`GLOBAL_LEAF_DISCHARGED`], the write's own discharge, or - /// [`GLOBAL_LEAF_SHOT_DOWN`], a machine-wide shootdown. Answers - /// [`GLOBAL_LEAF_SAW_OLD`] or [`GLOBAL_LEAF_SAW_NEW`], the frame the second - /// read saw; `NotSupported` on AArch64, `AlreadyExists` where something is - /// already mapped where the leaf goes. + /// the leaf at a second frame and read again with nothing invalidated, the + /// control, invalidate as the argument says, and read a third time, all on + /// one CPU with interrupts closed. The argument is + /// [`GLOBAL_LEAF_DISCHARGED`], the write's own discharge, + /// [`GLOBAL_LEAF_KERNEL_SHOT_DOWN`], the machine-wide shootdown a + /// kernel-half change asks, or [`GLOBAL_LEAF_USER_SHOT_DOWN`], the one a + /// user-half change asks. Answers [`GLOBAL_LEAF_SAW_OLD`] or + /// [`GLOBAL_LEAF_SAW_NEW`], the frame the third read saw, or + /// [`GLOBAL_LEAF_UNHELD`] where the control already saw the second frame; + /// `NotSupported` on AArch64, `AlreadyExists` where something is already + /// mapped where the leaf goes. pub const GLOBAL_LEAF_REWRITE: u64 = 28; - pub const GLOBAL_LEAF_WITHHELD: u64 = 0; - pub const GLOBAL_LEAF_DISCHARGED: u64 = 1; - pub const GLOBAL_LEAF_SHOT_DOWN: u64 = 2; + pub const GLOBAL_LEAF_DISCHARGED: u64 = 0; + pub const GLOBAL_LEAF_KERNEL_SHOT_DOWN: u64 = 1; + pub const GLOBAL_LEAF_USER_SHOT_DOWN: u64 = 2; pub const GLOBAL_LEAF_SAW_OLD: u64 = 0; pub const GLOBAL_LEAF_SAW_NEW: u64 = 1; + pub const GLOBAL_LEAF_UNHELD: u64 = 2; } /// Every kind of kernel object, in the order the kernel's own `kobject!` diff --git a/tests/guest/rust-tests/src/bin/global_leaf_flush.rs b/tests/guest/rust-tests/src/bin/global_leaf_flush.rs index 96b425decde..317b6025726 100644 --- a/tests/guest/rust-tests/src/bin/global_leaf_flush.rs +++ b/tests/guest/rust-tests/src/bin/global_leaf_flush.rs @@ -1,6 +1,6 @@ -//! A global kernel leaf rewritten under a machine-wide shootdown is read fresh -//! by the CPU that issued it: the flush every CPU of the shootdown runs drops -//! global translations, which `INVPCID` type 3 and a `CR3` write both leave. +//! A global kernel leaf rewritten under the machine-wide shootdown a kernel-half +//! change asks is read fresh by the CPU that issued it: that flush drops global +//! translations, which `INVPCID` type 3 and a `CR3` write both leave. #[path = "../global_leaf.rs"] mod global_leaf; @@ -8,7 +8,7 @@ mod global_leaf; use toyos_abi::syscall::debug_action as da; fn main() { - let saw = global_leaf::rewritten(da::GLOBAL_LEAF_SHOT_DOWN); + let saw = global_leaf::rewritten(da::GLOBAL_LEAF_KERNEL_SHOT_DOWN); assert_eq!(saw, da::GLOBAL_LEAF_SAW_NEW, "the shootdown's flush left the old frame's global translation"); println!("global_leaf_flush: the shootdown's flush dropped the global translation"); } diff --git a/tests/guest/rust-tests/src/bin/global_leaf_user_flush.rs b/tests/guest/rust-tests/src/bin/global_leaf_user_flush.rs new file mode 100644 index 00000000000..f4454f184c1 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/global_leaf_user_flush.rs @@ -0,0 +1,15 @@ +//! A global kernel leaf rewritten under the machine-wide shootdown a user-half +//! change asks is still read stale by the CPU that issued it: that flush is +//! `INVPCID` type 3 or a `CR3` reload, which keep global translations, and only +//! a kernel-half change asks the flush that drops them. + +#[path = "../global_leaf.rs"] +mod global_leaf; + +use toyos_abi::syscall::debug_action as da; + +fn main() { + let saw = global_leaf::rewritten(da::GLOBAL_LEAF_USER_SHOT_DOWN); + assert_eq!(saw, da::GLOBAL_LEAF_SAW_OLD, "the user-half shootdown dropped a global kernel translation"); + println!("global_leaf_user_flush: the user-half shootdown kept the global translation"); +} diff --git a/tests/guest/rust-tests/src/global_leaf.rs b/tests/guest/rust-tests/src/global_leaf.rs index 4af5e32374d..76d096ff5ec 100644 --- a/tests/guest/rust-tests/src/global_leaf.rs +++ b/tests/guest/rust-tests/src/global_leaf.rs @@ -1,25 +1,26 @@ -//! `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`, for the two metal rows that judge how a -//! global kernel translation is dropped, and the control both run first. +//! `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`, for the three metal rows that judge +//! which flush drops a global kernel translation and which keeps it. //! //! The action maps a global 2 MiB kernel leaf, reads through it, points it at a -//! second frame, invalidates as asked and reads again, on one CPU with -//! interrupts closed. **Metal rows and not guest tests**: a metal row is the -//! cheaper tier, and this host's TCG guest drops every translation on a `CR3` -//! write and has no `INVPCID`, so both wrong machine flushes read fresh there. +//! second frame, reads again with nothing invalidated (the control), +//! invalidates as asked and reads a third time, on one CPU with interrupts +//! closed. **Metal rows and not guest tests**: a metal row is the cheaper tier, +//! and this host's TCG guest drops every translation on a `CR3` write and has +//! no `INVPCID`, so it reads fresh where a CPU that keeps a global translation +//! reads stale. use toyos_abi::syscall::{self, debug_action as da}; -/// The frame the second read saw after `how`, once this CPU has shown it reads -/// the first one with nothing invalidated: a CPU that read the second frame -/// then held no translation to go stale, and its fresh read after an -/// invalidation would be no evidence of that invalidation. +/// The frame the third read saw after `how`, refused where the control saw the +/// second frame: that CPU held no translation to go stale, and what it read +/// after `how` would be no evidence of `how`. pub fn rewritten(how: u64) -> u64 { - let withheld = syscall::debug_with(da::GLOBAL_LEAF_REWRITE, da::GLOBAL_LEAF_WITHHELD); - assert_eq!( - withheld, - da::GLOBAL_LEAF_SAW_OLD, + let saw = syscall::debug_with(da::GLOBAL_LEAF_REWRITE, how); + assert_ne!( + saw, + da::GLOBAL_LEAF_UNHELD, "the control: with nothing invalidated the second read did not see the first frame, so \ this CPU held no translation to go stale and the row judges nothing", ); - syscall::debug_with(da::GLOBAL_LEAF_REWRITE, how) + saw } diff --git a/tests/toyos.rs b/tests/toyos.rs index fe8933c0131..dfe0b56ddae 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -89,10 +89,12 @@ const RUST_SKIP: &[&str] = &[ "tlb_shootdown_waits", // Each reads a global translation its kernel may leave stale, which a TCG // guest's TLB drops on any `CR3` write and the T14's keeps: the - // `a_global_kernel_leafs_discharge_drops_it` and - // `the_machine_flush_drops_a_global_kernel_translation` metal rows run them. + // `a_global_kernel_leafs_discharge_drops_it`, + // `a_kernel_half_shootdown_drops_a_global_kernel_translation` and + // `a_user_half_shootdown_keeps_a_global_kernel_translation` metal rows run them. "global_leaf_discharge", "global_leaf_flush", + "global_leaf_user_flush", // Its product is the windows a `mask-windows` kernel reports under it, and // its 256 threads' stacks would crowd every member after it in one boot. // The `mask_windows` metal row runs it. @@ -715,7 +717,7 @@ const METAL: &[(&str, metal::Metal)] = &[ }, ), ( - "the_machine_flush_drops_a_global_kernel_translation", + "a_kernel_half_shootdown_drops_a_global_kernel_translation", metal::Metal { arms: &[metal::Arm { features: ACTUATOR_KERNEL, @@ -724,6 +726,16 @@ const METAL: &[(&str, metal::Metal)] = &[ judge: |b| b[0].job_passed("test_rs_global_leaf_flush"), }, ), + ( + "a_user_half_shootdown_keeps_a_global_kernel_translation", + metal::Metal { + arms: &[metal::Arm { + features: ACTUATOR_KERNEL, + ..metal::once("shared-debug", "tests/images/testcases.toml", SELFTEST_PARAMS, &["test_rs_global_leaf_user_flush"]) + }], + judge: |b| b[0].job_passed("test_rs_global_leaf_user_flush"), + }, + ), ( // What one diary record costs its writer: a million written back to // back by `SYS_DEBUG`'s flood, so the kernel that carries it, timed From 806130ce9a4a0e395391b1d3bcd90fab8d89fbde Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 12 Oct 2026 22:29:52 +0200 Subject: [PATCH 3/3] Under PCID a global discharge is INVPCID type 2, Intel's own workaround, and PCID stays on every CPU that has INVPCID Intel's specification updates for Alder Lake (682436, ADL063) and Raptor Lake (740518, RPL043) say INVLPG "may fail to invalidate TLB entries for global pages with PCIDs different from the current PCID value", and give the software workaround: "this can be worked around by software using INVPCID type 2 instead of INVLPG". Round 2 instead withheld PCIDE on the CPUs those documents name, below a fixed microcode revision; the owner's ruling on #873 takes Intel's workaround, everywhere PCID is on. - paging: Owed::discharge's global arm is INVPCID type 2 wherever PCIDE is set, and INVLPG only without it, where the erratum does not apply (it "applies only when PCIDs are enabled"). Every other global discharge already went through flush_tlb_all, which is type 2 under PCID. Its one caller today is guard_4k, at boot. - The verdict goes whole: toyos-cpuvuln's invlpg.rs, its table rows and description; cpu::intel_microcode_revision; control_regs' per-CPU signature and microcode line; the seven host tests. - control_regs: a CPU with PCID but no INVPCID gets no PCIDE, as before, and now says so on the log by name, since every global discharge under PCID needs INVPCID. The review's two NOTEs: - the_t14_keeps_pcid went with the verdict. - The keeping row's action refuses, answering GLOBAL_LEAF_WIDENED, when this CPU's own answers inside the user-half shootdown reached wide because another CPU's kernel-half generation landed in it. Shootdown::widened, test-actuators only, reads the per-CPU record the protocol already keeps; kernel-loom declares the feature for cfg checking. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- kernel/cpuvuln/Cargo.toml | 2 +- kernel/cpuvuln/src/invlpg.rs | 139 ---------------------- kernel/cpuvuln/src/lib.rs | 5 +- kernel/cpuvuln/src/table.rs | 32 +---- kernel/loom/Cargo.toml | 2 +- kernel/src/arch/x86_64/control_regs.rs | 41 ++----- kernel/src/arch/x86_64/cpu.rs | 12 -- kernel/src/arch/x86_64/paging.rs | 41 ++++--- kernel/src/arch/x86_64/tlb.rs | 10 ++ kernel/src/shootdown.rs | 6 + sdk/abi/src/syscall.rs | 9 +- tests/guest/rust-tests/src/global_leaf.rs | 13 +- 12 files changed, 71 insertions(+), 241 deletions(-) delete mode 100644 kernel/cpuvuln/src/invlpg.rs diff --git a/kernel/cpuvuln/Cargo.toml b/kernel/cpuvuln/Cargo.toml index a7dda8638f0..74e3a0a386b 100644 --- a/kernel/cpuvuln/Cargo.toml +++ b/kernel/cpuvuln/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "toyos-cpuvuln" -description = "Verdicts from a CPU's CPUID and MSR facts, pure: Linux's on each speculative-execution vulnerability and its mitigation choice, which model-specific counters the CPU has and the HWP request intel_pstate asks of it; AMD's revision guides' on whether erratum 1386 forbids the CPU XSAVES; and Intel's specification updates' on whether its INVLPG drops a global translation in every PCID." +description = "Verdicts from a CPU's CPUID and MSR facts, pure: Linux's on each speculative-execution vulnerability and its mitigation choice, which model-specific counters the CPU has and the HWP request intel_pstate asks of it; AMD's revision guides' on whether erratum 1386 forbids the CPU XSAVES." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/kernel/cpuvuln/src/invlpg.rs b/kernel/cpuvuln/src/invlpg.rs deleted file mode 100644 index 9c970249e1d..00000000000 --- a/kernel/cpuvuln/src/invlpg.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Whether a CPU's `INVLPG` drops a global translation in every PCID, which a -//! kernel whose kernel-half leaves are global relies on while `CR4.PCIDE` is -//! set: Intel's erratum "INVLPG May Invalidate Global TLB Entries Only For The -//! Current PCID" (ADL063, RPL043), on the CPUs [`table`](crate::table) names, -//! below the microcode revision that fixed it. -//! -//! **Intel's verdict, not Linux's**: the specification updates are the -//! source, and where they part from `invlpg_miss_ids` (`arch/x86/mm/init.c`) -//! they decide. RPL043 marks the 6P+0E die N/A, which Linux's model match -//! takes in; the erratum "does not apply in VMX non-root operation", which -//! Linux does not read; and an update revision is signed (SDM Vol. 3A Table -//! 12-7), where Linux compares unsigned. - -use crate::{table, Ident, Vendor, CPUID_1_ECX_HYPERVISOR}; - -/// What one CPU says about itself, read on that CPU. -#[derive(Clone, Copy, Debug)] -pub struct InvlpgFacts { - pub vendor: Vendor, - /// CPUID.1:EAX. - pub signature: u32, - pub cpuid_1_ecx: u32, - /// `IA32_BIOS_SIGN_ID[63:32]` after the read SDM Vol. 3A §12.11.7.1 - /// defines, present exactly where the vendor is Intel: the procedure is - /// Intel's. - pub microcode: Option, -} - -/// Whether `INVLPG` may leave a global translation cached for another PCID: a -/// CPU Intel names, outside VMX non-root operation (CPUID.1:ECX[31]), running -/// microcode older than the fix. -/// -/// Panics where the facts contradict their own documentation: a revision read -/// on a vendor that is not Intel, or none on one that is. -pub fn invlpg_misses_global(facts: &InvlpgFacts) -> bool { - assert_eq!( - facts.microcode.is_some(), - facts.vendor == Vendor::Intel, - "the microcode revision is read exactly where the vendor is Intel", - ); - let guest = facts.cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR != 0; - table::invlpg_fixed_at(&Ident::new(facts.vendor, facts.signature)) - .is_some_and(|fixed| !guest && (facts.microcode.expect("asserted above") as i32) < fixed) -} - -#[cfg(test)] -mod tests { - extern crate std; - - use std::vec::Vec; - - use super::*; - - /// A native Intel CPU of `signature` running `microcode`. - fn native(signature: u32, microcode: u32) -> InvlpgFacts { - InvlpgFacts { vendor: Vendor::Intel, signature, cpuid_1_ecx: 0, microcode: Some(microcode) } - } - - /// Every signature the two specification updates name, from each one's - /// Table 2-1 beside its erratum's summary row, and Alder Lake-N's from its - /// own Table 2-1, each with Intel's fixed revision for its model. - const NAMED: [(u32, u32, &str); 9] = [ - // 682436-040: ADL063 "Fixed" on S, H/P, U and HX. - (0x0009_0672, 0x2e, "ADL-S 8+8, ADL-HX 8+8"), - (0x0009_0675, 0x2e, "ADL-S 6+0"), - (0x0009_06a3, 0x42c, "ADL-H 6+8, ADL-P 6+8"), - (0x0009_06a4, 0x42c, "ADL-U15W 2+8, ADL-U9W 2+8"), - // 740518-031: RPL043 "Fixed" on every column but S 6+0. - (0x000b_0671, 0x118, "RPL-S 8P+16E, RPL-HX 8P+16E, RPL-E 8P+0E"), - (0x000b_06f2, 0x2e, "RPL-S 8P+8E, RPL-HX 8P+8E"), - (0x000b_06a2, 0x4117, "RPL-P 6P+8E, RPL-H 6P+8E"), - (0x000b_06a3, 0x4117, "RPL-U 2P+8E"), - // 764616-004 lists the N-series by this signature alone. - (0x000b_06e0, 0x11, "Alder Lake-N"), - ]; - - #[test] - fn every_cpu_intel_names_withholds_pcid_below_its_fix_and_only_there() { - for (signature, fixed, name) in NAMED { - assert!(invlpg_misses_global(&native(signature, fixed - 1)), "{name} one revision below its fix"); - assert!(!invlpg_misses_global(&native(signature, fixed)), "{name} at its fix"); - } - } - - /// RPL043's summary row reads N/A under S 6+0, signature `0xB06F5`. - #[test] - fn the_die_rpl043_marks_not_applicable_keeps_pcid() { - assert!(!invlpg_misses_global(&native(0x000b_06f5, 0))); - } - - /// "This erratum does not apply to a guest operating system running in VMX - /// non-root operation" (ADL063, RPL043). - #[test] - fn a_guest_keeps_pcid() { - let guest = InvlpgFacts { cpuid_1_ecx: CPUID_1_ECX_HYPERVISOR, ..native(0x0009_06a4, 0) }; - assert!(!invlpg_misses_global(&guest)); - } - - /// Table 12-7: an update revision is a signed 32-bit number. - #[test] - fn a_revision_with_its_sign_bit_set_is_older_than_the_fix() { - assert!(invlpg_misses_global(&native(0x000b_06a2, 0x8000_4117))); - } - - /// The T14's CPU 0 off `fixtures/t14/`: its signature from `cpuid.txt`, its - /// revision from `msr.txt`'s 0x8B, as Linux read both. Tiger Lake is in - /// neither table. - #[test] - fn the_t14_keeps_pcid() { - let words = |text: &'static str, first: &str| -> Vec { - text.lines() - .map(|l| l.split(' ').map(|w| u64::from_str_radix(w, 16).expect(w)).collect::>()) - .find(|l| l[..2] == [u64::from_str_radix(first, 16).unwrap(), 0]) - .expect("a line the capture read") - }; - let leaf_1 = words(include_str!("../fixtures/t14/cpuid.txt"), "1"); - let bios_sign_id = words(include_str!("../fixtures/t14/msr.txt"), "8b")[2]; - let t14 = InvlpgFacts { - vendor: Vendor::Intel, - signature: leaf_1[2] as u32, - cpuid_1_ecx: leaf_1[4] as u32, - microcode: Some((bios_sign_id >> 32) as u32), - }; - assert_eq!((t14.signature, t14.microcode), (0x0008_06c1, Some(0xbe))); - assert!(!invlpg_misses_global(&t14)); - } - - #[test] - fn another_vendor_reads_no_revision_and_keeps_pcid() { - let amd = InvlpgFacts { vendor: Vendor::Amd, microcode: None, ..native(0x0009_06a4, 0) }; - assert!(!invlpg_misses_global(&amd)); - } - - #[test] - #[should_panic(expected = "read exactly where the vendor is Intel")] - fn an_intel_cpu_without_its_revision_is_a_contradiction() { - invlpg_misses_global(&InvlpgFacts { microcode: None, ..native(0x0008_06c1, 0) }); - } -} diff --git a/kernel/cpuvuln/src/lib.rs b/kernel/cpuvuln/src/lib.rs index 2bc1f4c0e02..1ad1af1b297 100644 --- a/kernel/cpuvuln/src/lib.rs +++ b/kernel/cpuvuln/src/lib.rs @@ -19,8 +19,7 @@ //! model-specific counters exist. [`hwp`] is the third: whether its //! performance request is declared, and [`hwp_request`] the request. //! [`erratum_1386`] is AMD's, not Linux's: whether its `XSAVES` erratum names -//! the CPU. [`invlpg_misses_global`] is Intel's: whether its `INVLPG` leaves -//! another PCID's global translation. +//! the CPU. //! //! Pure: no I/O, no allocation, no `unsafe`. The caller reads the facts. @@ -32,13 +31,11 @@ use core::fmt; mod counters; mod erratum; mod hwp; -mod invlpg; mod table; pub use counters::{counters, CounterFacts, Counters}; pub use erratum::erratum_1386; pub use hwp::{hwp, hwp_request, Hwp, HwpFacts, HwpRefusal, HWP_EPP}; -pub use invlpg::{invlpg_misses_global, InvlpgFacts}; #[cfg(test)] mod tests; diff --git a/kernel/cpuvuln/src/table.rs b/kernel/cpuvuln/src/table.rs index 6337c2dea27..8b748b5fe73 100644 --- a/kernel/cpuvuln/src/table.rs +++ b/kernel/cpuvuln/src/table.rs @@ -1,8 +1,7 @@ //! Linux's five CPU tables, row for row: `cpu_vuln_whitelist` and //! `cpu_vuln_blacklist` (`common.c:1182-1344`), `spectre_bad_microcodes` //! (`intel.c:141-163`), `amd_check_tsa_microcode`'s (`amd.c:472-515`) and -//! `test_intel`'s (`arch/x86/events/msr.c:40-122`); and one of Intel's, -//! [`invlpg_fixed_at`]'s. A +//! `test_intel`'s (`arch/x86/events/msr.c:40-122`). A //! lookup in the first two is `x86_match_cpu` (`match.c:36-57`): the //! first row whose vendor, family, model and stepping all match, `0` matching //! any. @@ -357,35 +356,6 @@ pub(crate) fn tsa_microcode(id: &Ident, microcode: u32) -> bool { ROWS.iter().find(|&&(k, _)| k == key).is_some_and(|&(_, min)| microcode >= min) } -/// The microcode revision from which `INVLPG` drops a global translation in -/// every PCID, on a CPU Intel names for "INVLPG May Invalidate Global TLB -/// Entries Only For The Current PCID": ADL063 (682436-040), on every signature -/// of its Table 2-1; RPL043 (740518-031), on every signature of its Table 2-1 -/// but the 6P+0E die's, where its summary reads N/A; and Alder Lake-N, which -/// its own specification update (764616-004) does not list and whose fix Intel -/// names beside the others. The revisions are Intel's, posted to linux-kernel -/// in Message-ID `20240325231300.qrltbzf6twm43ftb@desk`, one per model: no -/// specification update carries a revision. -pub(crate) fn invlpg_fixed_at(id: &Ident) -> Option { - const ROWS: &[(u32, u32, i32)] = &[ - (ALDERLAKE, 0x2, 0x2e), - (ALDERLAKE, 0x5, 0x2e), - (ALDERLAKE_L, 0x3, 0x42c), - (ALDERLAKE_L, 0x4, 0x42c), - (ATOM_GRACEMONT, 0x0, 0x11), - (RAPTORLAKE, 0x1, 0x118), - (RAPTORLAKE_P, 0x2, 0x4117), - (RAPTORLAKE_P, 0x3, 0x4117), - (RAPTORLAKE_S, 0x2, 0x2e), - ]; - if id.vendor != Vendor::Intel || id.family != 6 { - return None; - } - ROWS.iter() - .find(|&&(model, stepping, _)| model == id.model && stepping == id.stepping) - .map(|&(_, _, fixed)| fixed) -} - /// `test_intel` for `PERF_MSR_SMI` (`arch/x86/events/msr.c:40-122`): an Intel /// family 6 CPU of one of these models counts its SMIs in `MSR_SMI_COUNT`. pub(crate) fn counts_smis(id: &Ident) -> bool { diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index 757e1e1b9a9..70bc4a60a1d 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -322,4 +322,4 @@ test = false # library's tests and the simulator run. A `cfg` name and not a feature, since a # feature here is a model control. [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators", "mask-windows", "sched-check", "sched-tripwire", "protocol-port", "placement-ignores-staleness"))'] } +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators", "test-actuators", "mask-windows", "sched-check", "sched-tripwire", "protocol-port", "placement-ignores-staleness"))'] } diff --git a/kernel/src/arch/x86_64/control_regs.rs b/kernel/src/arch/x86_64/control_regs.rs index 35c562a8057..f228321d3ba 100644 --- a/kernel/src/arch/x86_64/control_regs.rs +++ b/kernel/src/arch/x86_64/control_regs.rs @@ -22,7 +22,7 @@ use core::sync::atomic::{AtomicU64, AtomicU8, Ordering}; -use toyos_cpuvuln::{HwpFacts, InvlpgFacts, Vendor}; +use toyos_cpuvuln::HwpFacts; use super::cpu; use crate::log; @@ -536,40 +536,19 @@ fn supported(cpu_id: u32) -> u64 { have |= flag; } } - // PCID without INVPCID is not worth having: nothing could flush by ASID. - if ecx1 & (1 << 17) != 0 && ebx7 & (1 << 10) != 0 && !invlpg_misses_global(cpu_id, ecx1) { - have |= cr4::PCIDE; + // PCID without INVPCID is refused: with `PCIDE` set every global discharge + // is `INVPCID` type 2 (`paging::Owed::discharge`), Intel's workaround for + // ADL063/RPL043's `INVLPG`, and nothing else could flush by PCID. + if ecx1 & (1 << 17) != 0 { + if ebx7 & (1 << 10) != 0 { + have |= cr4::PCIDE; + } else { + log!("control_regs: cpu{cpu_id} has PCID but no INVPCID: PCIDE refused"); + } } have } -/// Whether this CPU's `INVLPG` leaves another PCID's global translation -/// (`toyos_cpuvuln::invlpg_misses_global`), which every global discharge is -/// (`paging::Owed::discharge`): PCID is withheld where it does. The facts are -/// logged where a revision was read, so a reader can hold the read to another's. -fn invlpg_misses_global(cpu_id: u32, cpuid_1_ecx: u32) -> bool { - let vendor = cpu::vendor(); - let facts = InvlpgFacts { - vendor, - signature: cpu::cpuid(1, 0).0, - cpuid_1_ecx, - microcode: (vendor == Vendor::Intel).then(cpu::intel_microcode_revision), - }; - let misses = toyos_cpuvuln::invlpg_misses_global(&facts); - if let Some(microcode) = facts.microcode { - log!( - "control_regs: cpu{cpu_id} signature {:#x} microcode {microcode:#x}{}", - facts.signature, - if misses { - ": PCID withheld, Intel's erratum ADL063/RPL043 leaves another PCID's global translation" - } else { - "" - }, - ); - } - misses -} - /// Logs what this CPU holds, then asserts it against the declaration — /// logged first so a failing CPU still leaves its value in the log. fn self_check(cpu_id: u32, declared_cr4: u64, declared_xcr0: u64, form: kernel::xsave::Form, has_xss: bool) { diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 986a77a05d3..2f5dc664c7a 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -134,18 +134,6 @@ pub fn vendor() -> toyos_cpuvuln::Vendor { toyos_cpuvuln::Vendor::from_id(&core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4])) } -/// This CPU's microcode revision as Intel defines its read: `IA32_BIOS_SIGN_ID` -/// (MSR 8BH) written 0, CPUID.01H run, and the MSR's upper dword read (SDM -/// Vol. 3A §12.11.7.1). An Intel CPU's alone: the procedure is Intel's. -pub fn intel_microcode_revision() -> u32 { - const IA32_BIOS_SIGN_ID: u32 = 0x8B; - // SAFETY: MSR 8BH is architectural on every Intel 64 processor (SDM Vol. 4 - // Table 2-2), and 0 is the value the procedure writes before its read. - unsafe { wrmsr(IA32_BIOS_SIGN_ID, 0) }; - let _ = cpuid(1, 0); - (rdmsr(IA32_BIOS_SIGN_ID) >> 32) as u32 -} - /// CPUID.6's EAX and ECX, the thermal and power leaf; zero where CPUID.0 /// does not reach it, as `init_scattered_cpuid_features` reads it: a level /// past the range's own is no level. diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index ce37c92feb0..a9e3e28ff8e 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -210,10 +210,13 @@ impl Owed { fn discharge(self, target: Cr3) { match self { Self::Nothing => {} - // `INVLPG` drops the page's global translation in every PCID, but - // on a CPU with Intel's ADL063/RPL043, which `control_regs` declares - // without PCID. - Self::Address { va, prior } if prior & PAGE_GLOBAL != 0 => crate::arch::cpu::invlpg(va), + // With PCID, `INVPCID` type 2: Intel's ADL063 and RPL043 let an + // `INVLPG` keep the global translation in every PCID but the current + // one, and name type 2 "instead of INVLPG" as the software workaround. + Self::Address { va, prior } if prior & PAGE_GLOBAL != 0 => match pcid_active() { + Some(have) => crate::arch::cpu::invpcid(have, Invpcid::AllIncludingGlobal, 0, 0), + None => crate::arch::cpu::invlpg(va), + }, Self::Address { va, .. } => { if let Some(have) = pcid_active() { crate::arch::cpu::invpcid(have, Invpcid::Address, target.pcid(), va); @@ -935,8 +938,10 @@ const SCRATCH: u64 = 0u64.wrapping_sub(PAGE_2M); /// `SYS_DEBUG`'s `GLOBAL_LEAF_REWRITE`: a [`kernel_leaf`] at [`SCRATCH`] over one /// frame, read so this CPU caches it global; the leaf pointed at a second frame /// and read again with nothing invalidated, the control; invalidated as `how` -/// asks; read a third time. Answers which frame the third read saw, or that the -/// control already saw the second. The kernel's lock keeps it on one CPU and +/// asks; read a third time. Answers which frame the third read saw, or refuses: +/// where the control already saw the second frame, or where a user-half +/// shootdown's answer here reached wide because another CPU's kernel-half +/// generation landed inside it. The kernel's lock keeps it on one CPU and /// interrupts stay closed, so no flush but `how`'s lands between the reads. #[cfg(feature = "test-actuators")] pub fn debug_global_leaf(how: u64) -> u64 { @@ -969,7 +974,7 @@ pub fn debug_global_leaf(how: u64) -> u64 { // frames this call owns, or a translation of one of them. let read = || unsafe { core::ptr::read_volatile(SCRATCH as *const u64) }; - let (held, seen) = { + let (held, widened, seen) = { let mut space = kernel().lock(); let _closed = crate::arch::IrqGuard::close(); let target = space.root(); @@ -983,17 +988,20 @@ pub fn debug_global_leaf(how: u64) -> u64 { assert!(first == old_phys, "debug_global_leaf: a fresh leaf over {old_phys:#x} read {first:#x}"); let owed = pd.write_pde(idx, SCRATCH, new_phys | leaf); let held = read(); - match how { - How::Discharged => owed.discharge(target), + let widened = match how { + How::Discharged => { + owed.discharge(target); + false + } // The write's own invalidation is withheld: the arm asks what the shootdown drops. How::ShotDown(half) => { let _ = owed; - crate::arch::tlb::shootdown(Origin::Staged, half); + crate::arch::tlb::debug_shootdown_answered_wide(Origin::Staged, half) && matches!(half, Half::User) } - } + }; let seen = read(); pd.write_pde(idx, SCRATCH, 0).discharge(target); - (held, seen) + (held, widened, seen) }; // No CPU keeps a translation of either frame past its free. crate::arch::tlb::shootdown(Origin::Staged, Half::Kernel); @@ -1003,10 +1011,11 @@ pub fn debug_global_leaf(how: u64) -> u64 { _ if at == new_phys => false, _ => panic!("debug_global_leaf: the leaf over {old_phys:#x} then {new_phys:#x} read {at:#x}"), }; - match (old_frame(held), old_frame(seen)) { - (false, _) => da::GLOBAL_LEAF_UNHELD, - (true, true) => da::GLOBAL_LEAF_SAW_OLD, - (true, false) => da::GLOBAL_LEAF_SAW_NEW, + match (old_frame(held), widened, old_frame(seen)) { + (false, _, _) => da::GLOBAL_LEAF_UNHELD, + (true, true, _) => da::GLOBAL_LEAF_WIDENED, + (true, false, true) => da::GLOBAL_LEAF_SAW_OLD, + (true, false, false) => da::GLOBAL_LEAF_SAW_NEW, } } diff --git a/kernel/src/arch/x86_64/tlb.rs b/kernel/src/arch/x86_64/tlb.rs index 6337cd9c9f6..4e7f9d007da 100644 --- a/kernel/src/arch/x86_64/tlb.rs +++ b/kernel/src/arch/x86_64/tlb.rs @@ -279,6 +279,16 @@ pub fn debug_arm_ack_delay(nanos: u64) -> u64 { } } +/// [`shootdown`], answering whether this CPU's own answers inside it reached +/// wide: a kernel-half generation another CPU issued meanwhile widens them. +#[cfg(feature = "test-actuators")] +pub fn debug_shootdown_answered_wide(origin: Origin, half: Half) -> bool { + let me = percpu::cpu_id() as usize; + let before = SHOOTDOWN.widened(me); + shootdown(origin, half); + SHOOTDOWN.widened(me) != before +} + /// Give the machine its ordinary latency back before the window lapses. #[cfg(feature = "test-actuators")] pub fn debug_disarm_ack_delay() -> u64 { diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index f6b77013777..eb3c5f4b3ee 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -99,6 +99,12 @@ impl Shootdown { self.flushed[cpu].fetch_max(owed, Ordering::Release); } + /// `wide` as `cpu`'s last wide answer read it: a change across a span is a wide answer inside it. + #[cfg(feature = "test-actuators")] + pub fn widened(&self, cpu: usize) -> u64 { + self.widened[cpu].load(Ordering::Relaxed) + } + /// Does `cpu` owe anyone a flush? pub fn owes(&self, cpu: usize) -> bool { // A hint: `serve` carries the real ordering, so a stale `false` costs nothing. diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index fd15ac6391e..d2e9c5a3512 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -900,10 +900,12 @@ pub mod debug_action { /// [`GLOBAL_LEAF_KERNEL_SHOT_DOWN`], the machine-wide shootdown a /// kernel-half change asks, or [`GLOBAL_LEAF_USER_SHOT_DOWN`], the one a /// user-half change asks. Answers [`GLOBAL_LEAF_SAW_OLD`] or - /// [`GLOBAL_LEAF_SAW_NEW`], the frame the third read saw, or + /// [`GLOBAL_LEAF_SAW_NEW`], the frame the third read saw; /// [`GLOBAL_LEAF_UNHELD`] where the control already saw the second frame; - /// `NotSupported` on AArch64, `AlreadyExists` where something is already - /// mapped where the leaf goes. + /// [`GLOBAL_LEAF_WIDENED`] where the user-half shootdown's answer on this + /// CPU covered another CPU's kernel-half one and so dropped global + /// translations; `NotSupported` on AArch64, `AlreadyExists` where + /// something is already mapped where the leaf goes. pub const GLOBAL_LEAF_REWRITE: u64 = 28; pub const GLOBAL_LEAF_DISCHARGED: u64 = 0; pub const GLOBAL_LEAF_KERNEL_SHOT_DOWN: u64 = 1; @@ -911,6 +913,7 @@ pub mod debug_action { pub const GLOBAL_LEAF_SAW_OLD: u64 = 0; pub const GLOBAL_LEAF_SAW_NEW: u64 = 1; pub const GLOBAL_LEAF_UNHELD: u64 = 2; + pub const GLOBAL_LEAF_WIDENED: u64 = 3; } /// Create a pipe. Returns the read and write ends. diff --git a/tests/guest/rust-tests/src/global_leaf.rs b/tests/guest/rust-tests/src/global_leaf.rs index 76d096ff5ec..2a05ebc3710 100644 --- a/tests/guest/rust-tests/src/global_leaf.rs +++ b/tests/guest/rust-tests/src/global_leaf.rs @@ -11,9 +11,10 @@ use toyos_abi::syscall::{self, debug_action as da}; -/// The frame the third read saw after `how`, refused where the control saw the -/// second frame: that CPU held no translation to go stale, and what it read -/// after `how` would be no evidence of `how`. +/// The frame the third read saw after `how`, refused where what it read after +/// `how` would be no evidence of `how`: the control saw the second frame, so +/// that CPU held no translation to go stale; or a user-half shootdown answered +/// wide, covering a kernel-half one another CPU issued inside it. pub fn rewritten(how: u64) -> u64 { let saw = syscall::debug_with(da::GLOBAL_LEAF_REWRITE, how); assert_ne!( @@ -22,5 +23,11 @@ pub fn rewritten(how: u64) -> u64 { "the control: with nothing invalidated the second read did not see the first frame, so \ this CPU held no translation to go stale and the row judges nothing", ); + assert_ne!( + saw, + da::GLOBAL_LEAF_WIDENED, + "the user-half shootdown's answer covered another CPU's kernel-half one, which drops \ + global translations, so the row judges nothing", + ); saw }