diff --git a/issues/placement-is-blind-to-caches-and-topology.md b/issues/placement-is-blind-to-caches-and-topology.md index c1a29123cc9..6dbca620b30 100644 --- a/issues/placement-is-blind-to-caches-and-topology.md +++ b/issues/placement-is-blind-to-caches-and-topology.md @@ -27,3 +27,10 @@ placement story has three parts none of which we have: Composes with the reservation model (placement decides *where*, reservations decide *how much*) and with the scheduler simulator (`kernel/sim`), which would need a topology model to gate any of this. + +The simulator models the tree before the kernel has one, so it compares the +keys itself: `Shape::level` (`kernel/sim/src/topology.rs`), and the CPUID +decoding of `scouted` (`kernel/sim/tests/topology.rs`), which reads the benches' +recorded words. The stage that lands the kernel's `Topology::from_keys` and its +CPUID decoder moves both onto them in the same change, or the simulator keeps a +second tree and a second decoder beside the kernel's. diff --git a/issues/the-push-bounds-an-idle-cpus-wait-only-for-one-victim.md b/issues/the-push-bounds-an-idle-cpus-wait-only-for-one-victim.md new file mode 100644 index 00000000000..dff6d1560a1 --- /dev/null +++ b/issues/the-push-bounds-an-idle-cpus-wait-only-for-one-victim.md @@ -0,0 +1,26 @@ +--- +status: open +kind: finding +opened: 2026-10-12 +--- + +# The push bounds an idle CPU's wait only for one victim + +The simulator's invariant I15 (`kernel/sim/src/invariants.rs`, +`work_conservation_bound`) holds the shipped balance path to `cpus + 2` +protocol rounds of an idle CPU beside stealable work. The bound is derived for +one victim: that victim passes once a round and its push walks a cursor over +every sleeper, so the idle CPU is rung within `cpus − 1` of its passes. + +With several victims nothing derives a bound. Each victim walks its own cursor, +so a sleeper can be passed over by each in turn, and a thief probes the CPU +publishing the most surplus, so a probe sent to a victim that runs dry before it +answers leaves the thief waiting for a ring again. Neither is shown to happen: +over the suite at every simulator shape, and over storm and interactive +workloads at 8, 16 and 64 CPUs, the worst stretch measured is 17 rounds against +a bound of 66. It is the derivation that is missing, not a schedule that +crosses it. + +Exit: stage S2 of the scheduler design (#872) replaces the most-surplus victim +with two random choices and the cursor push with a nearest-first one, and +derives I15's bound for that protocol over any number of victims. diff --git a/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace b/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace index c6416f656cd..c2ac9baeecd 100644 --- a/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace +++ b/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace @@ -4,5 +4,7 @@ # note: InTransit(CpuId(1)). soundserver boost-wakes a client thread whose process # note: has just been torn down, on the CPU soundserver itself is running, with a # note: sibling asleep: place() forwards the corpse instead of reaping it. -1 2 2 0 1 1 1 1 2 0 0 1 1 1 1 1 0 0 2 1 2 1 2 1 0 1 0 0 2 2 1 2 -0 1 3 1 1 1 0 3 +# note: re-shrunk after an awake CPU with nothing loaded came to owe its pass at +# note: once, which renumbers every decision. +1 1 1 2 0 0 0 0 1 2 2 2 0 1 1 2 0 1 2 0 1 0 1 0 0 3 1 2 0 0 2 0 +2 0 0 0 2 0 2 2 1 1 0 3 diff --git a/kernel/sim/src/explore.rs b/kernel/sim/src/explore.rs index 19e0526e740..410cb949f09 100644 --- a/kernel/sim/src/explore.rs +++ b/kernel/sim/src/explore.rs @@ -11,6 +11,7 @@ use kernel::sched::hw::CpuId; use crate::choice::ChoiceStream; use crate::invariants; use crate::latency::{ReadyCause, RunWait}; +use crate::topology::LevelCounts; use crate::vm::{build_queues, Step, Vm}; use crate::workload::Scenario; @@ -111,6 +112,12 @@ pub struct Outcome { /// The longest a halted CPU sat beside a published surplus with no probe /// outstanding — see [`crate::vm::Vm::probe_gap_ns`]. pub probe_gap_ns: u64, + /// Invariant I15's measurement: the most rounds a CPU stayed idle beside + /// stealable work, and the most a ready task waited to run. + pub wc_rounds: u64, + pub rounds_to_run: u64, + /// What the run did at each level of its machine's cache tree. + pub levels: LevelCounts, /// What each CPU's passes cost, as the core's `feature = "check"` recorder /// measured them — the on-target instrument, driven here by the scenario's /// modelled `pass_cost_ns`. @@ -311,6 +318,9 @@ fn outcome_of(scenario: &'static str, vm: &Vm<'_>, choices: &ChoiceStream) -> Ou never_ran: vm.spawned.difference(&vm.ran).count(), idle_wakes: vm.idle_wakes.clone(), probe_gap_ns: vm.probe_gap_ns, + wc_rounds: vm.wc_rounds, + rounds_to_run: vm.rounds_to_run, + levels: vm.levels.clone(), pass_costs: (0..vm.handles.len()) .map(|cpu| { let cpu = CpuId(cpu as u32); diff --git a/kernel/sim/src/hw_impl.rs b/kernel/sim/src/hw_impl.rs index 3fc8beea8e6..1f10f775a3f 100644 --- a/kernel/sim/src/hw_impl.rs +++ b/kernel/sim/src/hw_impl.rs @@ -33,6 +33,9 @@ pub struct SimHwState { pub released: Vec<(TaskKey, TaskAccounting)>, pub trace: Vec, pub kicks: u64, + /// Each kick's target since the VM last drained it, which it does once a + /// step to say what level each one crossed. + pub kicked: Vec, pub switches: u64, /// Which CPU's pass is running. The core has no `cpu_id()` by design, so /// the driver is what knows where a timer or a halt lands. @@ -70,6 +73,7 @@ impl SimHw { released: Vec::new(), trace: Vec::new(), kicks: 0, + kicked: Vec::new(), switches: 0, current: None, pass_cost_ns: 0, @@ -118,6 +122,7 @@ impl Kicker for SimHw { self.with(|s| { s.pending_ipi[target.0 as usize] += 1; s.kicks += 1; + s.kicked.push(target); }); } } diff --git a/kernel/sim/src/invariants.rs b/kernel/sim/src/invariants.rs index cb926f110a8..cc07365b106 100644 --- a/kernel/sim/src/invariants.rs +++ b/kernel/sim/src/invariants.rs @@ -18,10 +18,10 @@ use kernel::sched::fair::{MAX_VRUNTIME_LAG_NS, QUANTUM_NS}; use kernel::sched::invariants::{residents, Container}; use kernel::sched::task::{TaskKey, TaskState}; -use kernel::sched::cpu::{DYING_AGE_NS, DYING_CHUNK_NS}; +use kernel::sched::cpu::{Balance, DYING_AGE_NS, DYING_CHUNK_NS}; use crate::latency::ReadyCause; -use crate::vm::{FairEpoch, Vm, IPI_LATENCY_NS, RUN_CHUNK_NS, UNWIND_NS}; +use crate::vm::{FairEpoch, Vm, IPI_LATENCY_NS, PROBE_WORTH_IT, RUN_CHUNK_NS, UNWIND_NS}; /// How long a CPU may keep running a normal task while an RT task is ready on /// it (invariant I4): the interrupt's own delivery bound, plus the preempt-off @@ -127,6 +127,7 @@ pub fn check_all(vm: &mut Vm<'_>) { check_share_refcounts(vm); check_address_spaces(vm); check_boost_windows(vm); + check_work_conservation(vm); note_run_latency(vm); } @@ -140,7 +141,7 @@ pub fn check_all(vm: &mut Vm<'_>) { /// share's refcount, and it is exactly a thread whose competition is changing, /// which is the one thing I13 declines to measure across. fn runnable_now(vm: &Vm<'_>) -> (Vec, Vec, BTreeSet) { - let cpus = vm.scenario.cpus; + let cpus = vm.scenario.cpus(); let mut counted = vec![0u32; vm.procs.len()]; let mut per_cpu = vec![0u32; vm.procs.len() * cpus]; let mut threads = BTreeSet::new(); @@ -166,7 +167,7 @@ fn runnable_now(vm: &Vm<'_>) -> (Vec, Vec, BTreeSet) { /// and I6 both need it and it is the more expensive half of either. fn runnable_per_process(vm: &Vm<'_>) -> Vec { let mut counted = vec![0u32; vm.procs.len()]; - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { for (key, container) in residents(&vm.cpus[cpu]) { if !matches!( container, @@ -192,7 +193,7 @@ fn check_single_ownership(vm: &mut Vm<'_>) { let mut seen: BTreeMap = BTreeMap::new(); let mut problems = Vec::new(); - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { for (key, container) in residents(&vm.cpus[cpu]) { if let Some((other, other_container)) = seen.insert(key, (cpu, container)) { problems.push(format!( @@ -257,7 +258,7 @@ fn check_single_ownership(vm: &mut Vm<'_>) { fn check_sleeping_cpus(vm: &mut Vm<'_>) { let mut problems = Vec::new(); let (halted, pending) = vm.hw.with(|s| (s.halted.clone(), s.pending_ipi.clone())); - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { // The stopped CPU is the modelled fault, not a protocol break: it is // *given* work it never takes, which is the whole of what the scenario // stages. Every other CPU still answers for itself. @@ -304,17 +305,14 @@ fn check_sleeping_cpus(vm: &mut Vm<'_>) { fn check_retires(vm: &mut Vm<'_>) { let mut problems = Vec::new(); - let migrated: Vec<(TaskKey, u32)> = vm.hw.with(|s| { - let fresh = s.trace[vm.trace_cursor..] - .iter() - .filter_map(|ev| match ev.kind { - kernel::sched::hw::TraceKind::Migrate { task, to, .. } => Some((task, to.0)), - _ => None, - }) - .collect(); - vm.trace_cursor = s.trace.len(); - fresh - }); + let migrated: Vec<(TaskKey, u32)> = vm + .step_trace + .iter() + .filter_map(|ev| match ev.kind { + kernel::sched::hw::TraceKind::Migrate { task, to, .. } => Some((task, to.0)), + _ => None, + }) + .collect(); // Counted on the same walk that judges them: the policy suite's wakeup-storm // case claims a machine drains in parallel, and that claim is only about the // balance path if the balance path moved something. @@ -409,7 +407,7 @@ fn retire_elapsed(vm: &Vm<'_>, key: TaskKey) -> u64 { /// thing the CPU owes. Delegated to the core's own checker, which /// is the same code a kernel `feature="check"` build runs. fn check_timers(vm: &mut Vm<'_>) { - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { let armed = vm.cpus[cpu].armed(); let quantum = vm.cpus[cpu].running().map(|_| vm.cpus[cpu].quantum_end()); let deadline = vm.cpus[cpu].parked().filter_map(|p| p.deadline()).min(); @@ -435,7 +433,7 @@ fn check_timers(vm: &mut Vm<'_>) { fn check_rt_latency(vm: &mut Vm<'_>) { let bound = rt_latency_bound(vm.max_kernel_section()); let mut problems = Vec::new(); - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { // `serves_rt_band` and not `is_rt`, for the reason that method's own doc // gives: a killed thread that holds the RT right is unwinding, not doing // real-time work, so a real-time sibling waiting behind it **is** being @@ -477,7 +475,7 @@ fn check_rt_latency(vm: &mut Vm<'_>) { fn note_rt_service(vm: &mut Vm<'_>) -> bool { let mut rt = Vec::new(); let mut occupied = false; - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { let sched = &vm.cpus[cpu]; if let Some(task) = sched.running() { if task.rt().is_rt() { @@ -572,7 +570,7 @@ fn check_fairness(vm: &mut Vm<'_>, rt_present: bool) { vm.fair_window_open = false; vm.thread_window_open = false; let (runnable, per_cpu, live_threads) = runnable_now(vm); - let saturated = (0..vm.scenario.cpus).all(|cpu| vm.cpus[cpu].running().is_some()); + let saturated = (0..vm.scenario.cpus()).all(|cpu| vm.cpus[cpu].running().is_some()); let mut members: Vec = if rt_present || !saturated { Vec::new() } else { @@ -588,7 +586,7 @@ fn check_fairness(vm: &mut Vm<'_>, rt_present: bool) { // can still fill the machine. if members .iter() - .any(|&p| runnable[p] as usize * members.len() < vm.scenario.cpus) + .any(|&p| runnable[p] as usize * members.len() < vm.scenario.cpus()) { members.clear(); } @@ -603,14 +601,14 @@ fn check_fairness(vm: &mut Vm<'_>, rt_present: bool) { // them sharing a CPU with a solo thread that is catching up, which is I5's // entitlement being honoured and not a sibling being cheated. let balanced = members.iter().all(|&process| { - let row = &per_cpu[process * vm.scenario.cpus..(process + 1) * vm.scenario.cpus]; + let row = &per_cpu[process * vm.scenario.cpus()..(process + 1) * vm.scenario.cpus()]; row.iter().min() == row.iter().max() }); let threads: u32 = members.iter().map(|&p| runnable[p]).sum(); // What one CPU's fair band holds. Read off CPU 0 rather than divided out of // the machine-wide total, so it is the quantity itself and not an average // that happens to equal it; `balanced` is what makes every CPU's the same. - let rivals: u32 = members.iter().map(|&p| per_cpu[p * vm.scenario.cpus]).sum(); + let rivals: u32 = members.iter().map(|&p| per_cpu[p * vm.scenario.cpus()]).sum(); if members != vm.fair_epoch.members { vm.fair_epoch = FairEpoch { @@ -852,7 +850,7 @@ fn check_thread_service(vm: &mut Vm<'_>, members: &[usize]) { // Which run queue each thread is in, because that is the first // question a per-thread spread raises and the answer is one // scan on a path that has already failed. - let at = (0..vm.scenario.cpus) + let at = (0..vm.scenario.cpus()) .find(|&cpu| residents(&vm.cpus[cpu]).any(|(k, _)| k == *key)) .map_or_else(|| "?".to_string(), |cpu| cpu.to_string()); Some(format!("{}@cpu{at}={served}", key.0)) @@ -871,6 +869,99 @@ fn check_thread_service(vm: &mut Vm<'_>, members: &[usize]) { } } +/// I15: **work conservation.** No CPU spends more than +/// [`work_conservation_bound`] protocol rounds ([`crate::vm::Vm::round`]) idle +/// while another holds a task a probe would be handed — a fair band at least +/// [`PROBE_WORTH_IT`] deep. +/// +/// The bounded form of the rule rather than a predicate over one step, and the +/// reason is what a step lets the simulator see. Its safety form — no CPU parks +/// unprobed beside stealable work that nobody rang it for — turns on whether a +/// victim's push rang a parked CPU, and a ring on a doorbell whose edge is +/// already raised changes nothing outside the core: it reads exactly as no ring. +/// What is visible is the outcome, an idle CPU and a queue that stays stealable, +/// and a protocol that leaves work unannounced or refuses it keeps that outcome +/// standing round after round. So the bound is on the outcome, and every +/// failure the safety form names fails it: an idle CPU nobody rings, one that is +/// rung and declines to probe, a probe that is never answered. +/// +/// **Nothing pauses the count.** A ring or a task on its way to the idle CPU +/// lands after an interrupt's latency, a matter of time, which the rest of the +/// machine could fill with rounds that take none; a round waits on every CPU an +/// IPI is on its way to ([`crate::vm::Vm::round`]), so an answer in flight costs +/// at most the round it was sent in. A CPU that is rung and declines to probe, +/// or whose probe is never answered, accrues every round it stays idle however +/// often it is rung. +/// +/// Rounds and not nanoseconds, because the clock measures the explorer as much +/// as the protocol; and a stopped CPU is neither idle nor a victim, its numbers +/// being the modelled fault `stopped_cpu` measures. +fn check_work_conservation(vm: &mut Vm<'_>) { + let cpus = vm.scenario.cpus(); + let victim = (0..cpus).find(|&cpu| { + !vm.is_stopped(cpu) && vm.cpus[cpu].rq().fair_len() as u32 >= PROBE_WORTH_IT + }); + let bound = work_conservation_bound(vm.scenario.balance, cpus).unwrap_or(u64::MAX); + let completed = vm.round - std::mem::replace(&mut vm.wc_round_seen, vm.round); + let mut problems = Vec::new(); + for cpu in 0..cpus { + let idle = !vm.is_stopped(cpu) && vm.cpus[cpu].running().is_none(); + let Some(victim) = victim.filter(|_| idle) else { + vm.wc_idle[cpu] = None; + continue; + }; + let rounds = vm.wc_idle[cpu].get_or_insert(0); + *rounds += completed; + let rounds = *rounds; + vm.wc_rounds = vm.wc_rounds.max(rounds); + if rounds > bound { + problems.push(format!( + "I15: cpu{cpu} has been idle for {rounds} rounds while cpu{victim} holds {} \ + queued fair task(s) (bound {bound} rounds)", + vm.cpus[victim].rq().fair_len(), + )); + } + } + for problem in problems { + vm.violate(problem); + } +} + +/// How many rounds a CPU may spend idle beside stealable work: `cpus + 2` under +/// [`Balance::PushOnSurplus`], and no bound under a policy that claims none — +/// the pull alone, no balance path and the declined re-arm, the controls +/// `sim/tests/policy.rs` prices the push against. +/// +/// **Derived for one victim.** An idle CPU `p` beside a victim `v`'s stealable +/// work counts: +/// +/// 1. `cpus` rounds until `p` is rung: the round open when `p` began counting, +/// whose pass of `v` may have come before it, and `cpus − 1` more. `v` runs +/// a task, so it is awake when every round opens and passes in each; every +/// such pass that leaves it stealable rings the first sleeping CPU at or +/// after its cursor and moves the cursor past it, so `p` is reached within +/// `cpus − 1` of them. A `p` the cursor passes because it is awake owes the +/// next round its pass, which probes as term 2's does. +/// 2. One round for `p`'s idle pass. A ring still in flight when the next round +/// opens holds that round open until `p` has passed, so `p` has probed `v`, +/// the only CPU publishing a surplus worth a probe, before it completes. +/// 3. One round for `v`'s answer: `v` passes in the round after the probe was +/// posted, and answers. The answer hands over a task unless that leaves `v` +/// with none to spare — and then nothing is stealable and the count is over +/// — and its post leaves `p` awake or rung, so the next round waits on the +/// pass in which `p` takes it and runs it. +/// +/// With several victims the derivation does not hold: each walks its own +/// cursor, and a probe sent to one that runs dry before it answers leaves `p` +/// to wait for a ring again — the flat push's weakness, recorded in +/// `issues/the-push-bounds-an-idle-cpus-wait-only-for-one-victim.md`. +fn work_conservation_bound(balance: Balance, cpus: usize) -> Option { + match balance { + Balance::PushOnSurplus { .. } => Some(cpus as u64 + 2), + Balance::None | Balance::Pull | Balance::PullWithRearm { .. } => None, + } +} + /// The measured policy suite's instrument: how long each task waited between /// being owed a dispatch and getting one (`crate::latency`, `sim/tests/policy.rs`). /// @@ -895,7 +986,7 @@ fn check_thread_service(vm: &mut Vm<'_>, members: &[usize]) { fn note_run_latency(vm: &mut Vm<'_>) { let now = vm.clock; let mut seen: BTreeMap = BTreeMap::new(); - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { for (key, container) in residents(&vm.cpus[cpu]) { seen.insert(key, container); } @@ -925,10 +1016,11 @@ fn note_run_latency(vm: &mut Vm<'_>) { | Container::Zombie, ) => ReadyCause::Fresh, }; - vm.awaiting.insert(key, (now, cause)); + vm.awaiting.insert(key, (now, cause, vm.round)); } } else if container == Container::Running { - if let Some((since, cause)) = vm.awaiting.remove(&key) { + if let Some((since, cause, round)) = vm.awaiting.remove(&key) { + vm.rounds_to_run = vm.rounds_to_run.max(vm.round - round); dispatched.push((key, cause, now.since(since))); } } else { @@ -1091,7 +1183,7 @@ pub fn check_final(vm: &mut Vm<'_>) { stuck.join(", "), )); } - for cpu in 0..vm.scenario.cpus { + for cpu in 0..vm.scenario.cpus() { if vm.is_stopped(cpu) { continue; } diff --git a/kernel/sim/src/lib.rs b/kernel/sim/src/lib.rs index c0d10a20e6b..7efdbd4b320 100644 --- a/kernel/sim/src/lib.rs +++ b/kernel/sim/src/lib.rs @@ -33,5 +33,6 @@ pub mod payload; pub mod scenarios; pub mod shrink; pub mod sweep; +pub mod topology; pub mod vm; pub mod workload; diff --git a/kernel/sim/src/main.rs b/kernel/sim/src/main.rs index 7eebe451970..4a2f8741c64 100644 --- a/kernel/sim/src/main.rs +++ b/kernel/sim/src/main.rs @@ -67,7 +67,7 @@ fn main() -> ExitCode { }; let seed: u64 = args.get(2).map_or(0, |s| s.parse().unwrap_or(0)); let mut choices = if cmd == "pct" { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) } else { ChoiceStream::from_seed(seed) }; @@ -214,7 +214,7 @@ fn main() -> ExitCode { for seed in 0..seeds { for pct in [false, true] { let mut choices = if pct { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) } else { ChoiceStream::from_seed(seed) }; @@ -236,7 +236,7 @@ fn main() -> ExitCode { }; let seed: u64 = args.get(2).map_or(0, |s| s.parse().unwrap_or(0)); let mut choices = if args.get(3).is_some_and(|d| d == "pct") { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) } else { ChoiceStream::from_seed(seed) }; diff --git a/kernel/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs index 2e82c8ec6ba..166699d25b0 100644 --- a/kernel/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -9,10 +9,12 @@ use kernel::sched::cpu::Balance; use kernel::sched::queue::FairOrder; use kernel::sched::task::WaitClass; +use crate::topology::Shape; use crate::vm::RUN_CHUNK_NS; use crate::workload::{ AgeShape, BlockShape, ChargeShape, IrqSpec, MigrateShape, Op, ParkShape, - PlacementShape, ProcSpec, Protocol, QueueSpec, Scenario, Script, ShareShape, WindowShape, + PlacementShape, ProcSpec, Protocol, QueueSpec, Scenario, Script, ShareShape, SurplusShape, + WindowShape, }; const MS: u64 = 1_000_000; @@ -29,7 +31,7 @@ fn scenario( ) -> Scenario { Scenario { name, - cpus, + topology: Shape::flat(cpus), queues, procs, irqs: Vec::new(), @@ -48,6 +50,7 @@ fn scenario( balance: Balance::PushOnSurplus { threshold: kernel::sched::cpu::PUSH_THRESHOLD, }, + surplus: SurplusShape::Published, order: FairOrder::InsertSequence, pass_cost_ns: 0, fair_allowance_ns: 0, @@ -1428,6 +1431,27 @@ pub fn lopsided_placement(cpus: usize, threads: usize, work: u64) -> Scenario { scenario } +/// **Wasted Cores' group imbalance, staged**: an idle CPU whose neighbours all +/// look busy, beside a victim with work to spare, on the AMD bench's two CCXs. +/// +/// cpu8 is idle while every other CPU of its CCX holds a task queued behind the +/// one it runs — load no probe takes — and cpu0, in the other CCX, holds two. A +/// rule that compares the two CCXs' average load sees cpu8's as the busier and +/// never probes cpu0, which is how an average in a summary hides an idle CPU, +/// and work conservation (invariant I15) is the one check that sees it. The +/// work outlasts I15's bound many times over, so the idle CPU must be reached +/// rather than outwaited. +pub fn group_imbalance() -> Scenario { + const ON: &[usize] = &[ + 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, + ]; + let mut scenario = lopsided_placement(16, ON.len(), 4 * WORK) + .with_placement(PlacementShape::Listed(ON)) + .with_topology(Shape::amd_bench()); + scenario.name = "group_imbalance"; + scenario +} + /// Negative gate for invariant I9: [`lend_then_block`] under commit `9c2fc4d`'s /// park, which cleared the borrowed window only `if now >= until`. /// diff --git a/kernel/sim/src/sweep.rs b/kernel/sim/src/sweep.rs index 04988d0bb3c..20a9c7eb47c 100644 --- a/kernel/sim/src/sweep.rs +++ b/kernel/sim/src/sweep.rs @@ -164,7 +164,7 @@ pub fn seed_sweep(scenario: &Scenario, seeds: u64, keep_failures: usize) -> Swee let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); result.runs += 1; @@ -190,7 +190,7 @@ pub fn abort_gate(scenario: &Scenario, seeds: u64) -> Option<(u64, String)> { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; if let Err(message) = run_catching(scenario.clone(), &mut choices) { return Some((seed, message)); diff --git a/kernel/sim/src/topology.rs b/kernel/sim/src/topology.rs new file mode 100644 index 00000000000..692e5dd02b6 --- /dev/null +++ b/kernel/sim/src/topology.rs @@ -0,0 +1,299 @@ +//! The machine's shape: which core, last-level cache and package each dense CPU +//! id sits in, so that a run can say which level of the cache tree each +//! migration, wake, failed steal and IPI crossed. +//! +//! A shape is what each CPU reads of itself: its x2APIC id, shifted right by the +//! width of each level — the SMT and package widths of CPUID leaf 0xB, and the +//! last-level cache's width, the base-2 log of how many ids share it, from the +//! deterministic cache leaf. Two CPUs share a level when their keys at it are +//! equal, and the lowest level they share is the one a move between them +//! crosses; a level whose domains are its parent's collapses into it, because no +//! pair of CPUs then has it as its lowest. +//! +//! Dense ids are the order the firmware listed the CPUs in, which is no topology +//! order: the T14's SMT siblings are four ids apart and the AMD bench's +//! adjacent, so a synthetic shape is enumerated under a seeded permutation and +//! nothing may read locality off an id's value. +//! +//! The scheduler core reads none of this. It is the simulator's instrument, and +//! it prices no level: there is no cache, frequency or SMT cost here. + +use rand::rngs::SmallRng; +use rand::seq::SliceRandom; +use rand::SeedableRng; + +/// The lowest domain two distinct CPUs share. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum Level { + /// SMT siblings: one core's hardware threads. + Core, + /// One last-level cache. + Llc, + /// One package. + Package, + /// Different packages. + Machine, +} + +/// A count's column in [`LevelCounts`]: 0 is one CPU, then each [`Level`]. +pub fn slot(level: Option) -> usize { + level.map_or(0, |level| 1 + level as usize) +} + +/// What one run did at each level, each array indexed by [`slot`]. Counts, never +/// times: whether a level is worth crossing is metal's verdict. +#[derive(Clone, Default, PartialEq, Eq, Debug)] +pub struct LevelCounts { + /// Tasks moved between CPUs, by steal or wake forward. + pub migrations: [u64; 5], + /// Wakes — a task taken out of its CPU's parked set, by a message, its + /// deadline or a retire — by the level between that CPU and the one the + /// waking pass left the task with: its own, unless the pass sent it on. + pub wakes: [u64; 5], + /// Steal probes their victim answered with no task. + pub failed_steals: [u64; 5], + /// Kicks, by the level between the CPU whose step sent one and its target. + pub ipis: [u64; 5], + /// Kicks no CPU's step sent: a device's. + pub device_ipis: u64, +} + +impl LevelCounts { + pub fn add(&mut self, other: &Self) { + for (sum, row) in [ + (&mut self.migrations, &other.migrations), + (&mut self.wakes, &other.wakes), + (&mut self.failed_steals, &other.failed_steals), + (&mut self.ipis, &other.ipis), + ] { + for (a, b) in sum.iter_mut().zip(row) { + *a += b; + } + } + self.device_ipis += other.device_ipis; + } + + /// One line per count, one column per level. + pub fn table(&self) -> String { + let mut out = String::from(" same core llc package machine\n"); + for (name, row) in [ + ("migrations", &self.migrations), + ("wakes placed", &self.wakes), + ("failed steals", &self.failed_steals), + ("ipis", &self.ipis), + ] { + out.push_str(&format!("{name:<14}")); + for count in row { + out.push_str(&format!("{count:>8}")); + } + out.push('\n'); + } + out.push_str(&format!("device ipis {:>8}\n", self.device_ipis)); + out + } +} + +/// One CPU's key at each level. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Keys { + pub core: u32, + pub llc: u32, + pub package: u32, +} + +#[derive(Clone, Debug)] +pub struct Shape { + pub name: &'static str, + /// Indexed by dense CPU id. + keys: Vec, +} + +/// How many low bits of an id a level `count` wide takes. +fn width(count: u32) -> u32 { + count.next_power_of_two().trailing_zeros() +} + +impl Shape { + /// Each dense CPU's x2APIC id, in the order the firmware listed them, and + /// each level's width in bits. + pub fn from_apic_ids( + name: &'static str, + ids: &[u32], + smt_shift: u32, + llc_shift: u32, + package_shift: u32, + ) -> Self { + assert!( + smt_shift <= llc_shift && llc_shift <= package_shift, + "{name}: a core must sit inside its LLC and an LLC inside its package", + ); + let mut seen = ids.to_vec(); + seen.sort_unstable(); + seen.dedup(); + assert_eq!(seen.len(), ids.len(), "{name}: two CPUs carry one x2APIC id"); + let keys = ids + .iter() + .map(|&id| Keys { + core: id >> smt_shift, + llc: id >> llc_shift, + package: id >> package_shift, + }) + .collect(); + Self { name, keys } + } + + /// `cpus` cores of one thread each, sharing one LLC in one package: the + /// machine of a scenario that names no shape. + pub fn flat(cpus: usize) -> Self { + let cpus = u32::try_from(cpus).expect("a CPU count fits an x2APIC id"); + let ids: Vec = (0..cpus).collect(); + Self::from_apic_ids("flat", &ids, 0, width(cpus), width(cpus)) + } + + /// `packages × llcs × cores × threads`, the ids laid out as x2APIC ids are, + /// listed under the permutation `seed` draws. + pub fn tree( + name: &'static str, + packages: u32, + llcs: u32, + cores: u32, + threads: u32, + seed: u64, + ) -> Self { + let (t, c, l) = (width(threads), width(cores), width(llcs)); + let mut ids = Vec::new(); + for package in 0..packages { + for llc in 0..llcs { + for core in 0..cores { + for thread in 0..threads { + ids.push(package << (l + c + t) | llc << (c + t) | core << t | thread); + } + } + } + } + ids.shuffle(&mut SmallRng::seed_from_u64(seed)); + Self::from_apic_ids(name, &ids, t, c + t, l + c + t) + } + + /// The T14 as its scout read it: four cores of two threads under one 8 MiB + /// L3, listed first threads first, so that SMT siblings are four ids apart. + pub fn t14() -> Self { + Self::from_apic_ids("t14", &[0, 2, 4, 6, 1, 3, 5, 7], 1, 4, 4) + } + + /// The AMD bench as its scout read it: two CCXs of four cores of two + /// threads, each CCX under its own 4 MiB L3, listed in APIC order so that + /// SMT siblings are adjacent. + pub fn amd_bench() -> Self { + let ids: Vec = (0..16).collect(); + Self::from_apic_ids("amd-bench", &ids, 1, 3, 7) + } + + pub fn cpus(&self) -> usize { + self.keys.len() + } + + pub fn keys(&self, cpu: usize) -> Keys { + self.keys[cpu] + } + + /// The lowest level `a` and `b` share, or `None` when they are one CPU. + pub fn level(&self, a: usize, b: usize) -> Option { + // By id: SMT siblings share every key. + if a == b { + return None; + } + let (a, b) = (self.keys[a], self.keys[b]); + Some(if a.core == b.core { + Level::Core + } else if a.llc == b.llc { + Level::Llc + } else if a.package == b.package { + Level::Package + } else { + Level::Machine + }) + } + + /// How many domains the machine has at `level`. + pub fn domains(&self, level: Level) -> usize { + let mut keys: Vec = self + .keys + .iter() + .map(|k| match level { + Level::Core => k.core, + Level::Llc => k.llc, + Level::Package => k.package, + Level::Machine => 0, + }) + .collect(); + keys.sort_unstable(); + keys.dedup(); + keys.len() + } + + /// `cpus=… cores=… llcs=… packages=…`, the shape as counts per level. + pub fn describe(&self) -> String { + format!( + "{}: cpus={} cores={} llcs={} packages={}", + self.name, + self.cpus(), + self.domains(Level::Core), + self.domains(Level::Llc), + self.domains(Level::Package), + ) + } +} + +/// Every shape the topology sweep runs every scenario at: the two benches as +/// their scouts read them, and two synthetic trees under a permuted listing — +/// the second 768 CPUs wide, two packages of 24 eight-core LLCs. +pub fn shapes() -> Vec { + vec![ + Shape::t14(), + Shape::amd_bench(), + Shape::tree("4x8x2", 1, 4, 8, 2, 1), + Shape::tree("2x24x8x2", 2, 24, 8, 2, 2), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_tree_listed_under_a_permutation_keeps_its_counts_per_level() { + let shape = Shape::tree("2x24x8x2", 2, 24, 8, 2, 2); + assert_eq!(shape.cpus(), 768); + assert_eq!(shape.domains(Level::Core), 384); + assert_eq!(shape.domains(Level::Llc), 48); + assert_eq!(shape.domains(Level::Package), 2); + } + + #[test] + fn a_pair_is_at_the_lowest_level_it_shares() { + let t14 = Shape::t14(); + assert_eq!(t14.level(0, 0), None); + assert_eq!(t14.level(0, 4), Some(Level::Core)); + assert_eq!(t14.level(0, 1), Some(Level::Llc)); + let amd = Shape::amd_bench(); + assert_eq!(amd.level(0, 1), Some(Level::Core)); + assert_eq!(amd.level(0, 2), Some(Level::Llc)); + assert_eq!(amd.level(0, 8), Some(Level::Package)); + let wide = Shape::tree("2x24x8x2", 2, 24, 8, 2, 2); + let other_package = (1..wide.cpus()).find(|&cpu| wide.keys(cpu).package != wide.keys(0).package); + assert_eq!(other_package.map(|cpu| wide.level(0, cpu)), Some(Some(Level::Machine))); + } + + #[test] + fn the_permutation_moves_siblings_apart() { + let shape = Shape::tree("4x8x2", 1, 4, 8, 2, 1); + let adjacent_siblings = (0..shape.cpus() / 2) + .filter(|&pair| shape.level(2 * pair, 2 * pair + 1) == Some(Level::Core)) + .count(); + assert!( + adjacent_siblings < shape.cpus() / 2, + "every even id's sibling is the id after it: the listing was not permuted", + ); + } +} diff --git a/kernel/sim/src/vm.rs b/kernel/sim/src/vm.rs index f7e85f7c042..1bc2b873e25 100644 --- a/kernel/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -16,7 +16,7 @@ use std::sync::Arc as StdArc; use kernel::sched::cpu::{Action, CpuHandle, CpuSched, Env, SchedPass}; use kernel::sched::fair::{FairShare, Frontier, ShareState}; -use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; +use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent, TraceKind}; use kernel::sched::invariants::Container; use kernel::sched::mailbox::{mailbox, Kick, Urgency}; use kernel::sched::msg::Msg; @@ -34,9 +34,10 @@ use crate::hw_impl::SimHw; use crate::latency::{ReadyCause, RunWait}; use crate::msg::{SimHandles, SimMsg, SimQueue}; use crate::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, SimShareLock, StdLock}; +use crate::topology::LevelCounts; use crate::workload::{ AgeShape, BlockShape, ChargeShape, MigrateShape, Op, ParkShape, PlacementShape, - Protocol, Scenario, Script, ShareShape, WindowShape, + Protocol, Scenario, Script, ShareShape, SurplusShape, WindowShape, }; /// How finely a `Run(ns)` op is chopped. Small enough that a 10 ms quantum @@ -285,10 +286,12 @@ pub struct Vm<'q> { pub busy_ns: Vec, /// When each CPU's pending IPI must be taken. pub ipi_due: Vec>, - /// When each CPU last acquired an unserved `need_resched`. A CPU that - /// owes a rescheduling pass takes it at IRQ exit, i.e. essentially at - /// once; letting the explorer defer it while the clock ran would measure - /// the explorer's freedom rather than the protocol's latency. + /// When each CPU began to owe a pass: an unserved `need_resched`, or being + /// awake with no task loaded. A CPU that owes a rescheduling pass takes it + /// at IRQ exit, and one on its idle context is running the idle loop, so + /// either takes it essentially at once; letting the explorer defer it while + /// the clock ran would measure the explorer's freedom rather than the + /// protocol's latency. pub resched_at: Vec>, /// When each CPU last acquired — or last discharged and re-acquired — an /// unfinished unwind on its loaded task. @@ -352,8 +355,9 @@ pub struct Vm<'q> { /// `RUN_CHUNK_NS` at a time, so the unwind is preemptible at exactly the /// granularity every other run is. pub unwind_left: BTreeMap, - /// How far into `SimHwState::trace` invariant I14 has read. - pub trace_cursor: usize, + /// What the last step traced, drained out of `SimHwState::trace` once per + /// step: invariant I14 and the per-level counts read the same events. + pub step_trace: Vec, /// I14's measurement: the longest a retire has taken to reach `release`, /// and the bound in force when that happened. A number as well as a /// verdict, because the kernel's own guard is a wall clock and the honest @@ -433,8 +437,9 @@ pub struct Vm<'q> { pub thread_bound: u64, pub thread_over_bound: u64, /// Every task currently owed a dispatch and not getting one, with the - /// instant it became owed one and why — the measured policy suite's - /// instrument (`crate::latency`, `sim/tests/policy.rs`). + /// instant it became owed one, why, and the [`Vm::round`] it was in — the + /// measured policy suite's instrument (`crate::latency`, + /// `sim/tests/policy.rs`). /// /// "Owed a dispatch" is two states and not one: a task in a run queue, and a /// *parked* task a wake has already claimed (`TaskState::WakeQueued`), whose @@ -448,7 +453,7 @@ pub struct Vm<'q> { /// per-CPU instrument would lose. A task in the *dying* list drops it — a /// corpse's wait for the CPU is invariant I14's quantity, measured on I14's /// clock against I14's bound. - pub awaiting: BTreeMap, + pub awaiting: BTreeMap, /// Where each task sat at the end of the previous step, which is what says /// whether an arrival in a run queue is a wake, a preemption or a spawn. pub prev_container: BTreeMap, @@ -520,12 +525,42 @@ pub struct Vm<'q> { /// Per CPU: when its current probe gap opened, or `None` if it is not in /// one. probe_gap_since: Vec>, + /// **Protocol rounds completed.** A round opens on the CPUs that owe a + /// pass — awake, or halted with an IPI on its way, and not stopped — and + /// completes when each of them has run one scheduler pass, the only point + /// at which a CPU drains, answers a probe, publishes and pushes. + /// + /// Counted in passes and not in time because the model's clock measures the + /// explorer as much as the protocol: a CPU whose pass the explorer defers + /// holds its round open, and no work-conservation bound runs while it does. + /// An IPI's latency is time too, so a CPU rung before a round opens holds + /// that round open until its pass. + pub round: u64, + /// Which CPUs the open round still waits on, and how many; a count of zero + /// is no round open. + round_owed: Vec, + round_owed_count: usize, + /// Per CPU: how many rounds it has spent idle beside stealable work since + /// it last was not (invariant I15), or `None`; and the [`Vm::round`] I15 + /// last read, which says whether a round completed since. + pub wc_idle: Vec>, + pub wc_round_seen: u64, + /// I15's measurement: the most rounds any CPU stayed idle beside stealable + /// work, and the most any ready task waited to be dispatched. + pub wc_rounds: u64, + pub rounds_to_run: u64, + /// What the run did at each level of [`Scenario::topology`]. + pub levels: LevelCounts, + /// Per CPU: a steal probe outstanding before the step being executed, so a + /// probe the step consumed without handing a task over is a failed steal. + probing: Vec, } /// The surplus at which a victim is worth probing — /// `SchedPass::post_steal_probe`'s own inequality, the core's constant because -/// [`Vm::probe_gap_ns`] has to ask the same question from outside the core. -const PROBE_WORTH_IT: u32 = kernel::sched::cpu::PUSH_THRESHOLD; +/// [`Vm::probe_gap_ns`] and invariant I15 have to ask the same question from +/// outside the core. +pub const PROBE_WORTH_IT: u32 = kernel::sched::cpu::PUSH_THRESHOLD; /// One contention window: a maximal interval over which the same set of /// fair-band processes was continuously runnable. @@ -566,7 +601,7 @@ pub struct FairEpoch { impl<'q> Vm<'q> { pub fn new(scenario: Scenario, queues: &'q [QueueState]) -> Self { - let n = scenario.cpus; + let n = scenario.cpus(); let hw = SimHw::new(n); let mut handles = Vec::with_capacity(n); let mut cpus = Vec::with_capacity(n); @@ -651,6 +686,17 @@ impl<'q> Vm<'q> { woken_from_halt: vec![false; n], probe_gap_ns: 0, probe_gap_since: vec![None; n], + // Every CPU starts awake and owes its first pass; the stopped one + // never takes one. + round: 0, + round_owed: (0..n).map(|cpu| scenario.stopped != Some(cpu)).collect(), + round_owed_count: n - usize::from(scenario.stopped.is_some()), + wc_idle: vec![None; n], + wc_round_seen: 0, + wc_rounds: 0, + rounds_to_run: 0, + levels: LevelCounts::default(), + probing: vec![false; n], next_irq, next_key: 1, next_spawn_cpu: 0, @@ -661,7 +707,7 @@ impl<'q> Vm<'q> { killed_at_park: 0, killed: BTreeMap::new(), unwind_left: BTreeMap::new(), - trace_cursor: 0, + step_trace: Vec::new(), retire_latency: 0, retire_bound: 0, scenario, @@ -671,6 +717,8 @@ impl<'q> Vm<'q> { vm.spawn(index, template); } } + // Spawned before any CPU took a step, so sent by none of them. + vm.levels.device_ipis += vm.hw.with(|s| std::mem::take(&mut s.kicked)).len() as u64; vm } @@ -727,10 +775,11 @@ impl<'q> Vm<'q> { .handles .place(CpuId(self.next_spawn_cpu as u32), self.clock) .0 as usize; - self.next_spawn_cpu = (least_loaded + 1) % self.scenario.cpus; + self.next_spawn_cpu = (least_loaded + 1) % self.scenario.cpus(); let dst = match self.scenario.placement { PlacementShape::LeastLoadedRotating => least_loaded, PlacementShape::AllOn(cpu) => cpu, + PlacementShape::Listed(cpus) => cpus[self.spawned.len()], }; let builder = TaskBuilder { key, @@ -798,14 +847,12 @@ impl<'q> Vm<'q> { // this the explorer could hold one CPU at its interrupt while another // ran for milliseconds, and invariant I4 would be measuring the // explorer's freedom rather than the protocol's latency. - let delivery_owed = (0..self.scenario.cpus) + let delivery_owed = (0..self.scenario.cpus()) .filter(|cpu| !self.is_stopped(*cpu)) .any(|cpu| { (pending_ipi[cpu] > 0 && self.ipi_due[cpu].is_some_and(|at| at <= self.clock)) || armed[cpu].is_some_and(|at| at <= self.clock) - || (need_resched[cpu] - && self.resched_at[cpu] - .is_some_and(|at| at.after(RUN_CHUNK_NS) <= self.clock)) + || self.resched_at[cpu].is_some_and(|at| at.after(RUN_CHUNK_NS) <= self.clock) }); // The same device one step further in: a CPU that has held an unwinding @@ -826,7 +873,7 @@ impl<'q> Vm<'q> { // one chunk of grace, the chunk that carries the clock past it, and the // chunk that discharges it. [`Vm::unwind_gate_ns`] is the instrument and // `the_unwind_gate_lasts_one_chunk_and_not_one_unwind` is the gate. - let unwind_owed = (0..self.scenario.cpus) + let unwind_owed = (0..self.scenario.cpus()) .filter_map(|cpu| { self.unwind_at[cpu] .filter(|at| at.after(RUN_CHUNK_NS) <= self.clock) @@ -835,7 +882,7 @@ impl<'q> Vm<'q> { .min() .map(|(_, cpu)| cpu); - for cpu in 0..self.scenario.cpus { + for cpu in 0..self.scenario.cpus() { // A stopped CPU is offered nothing at all — not its interrupts, not // its passes, not its execution steps. What it has already been // given stays where it is, which is the state this dimension exists @@ -898,7 +945,7 @@ impl<'q> Vm<'q> { // A CPU number, not a walk of `halted`: it indexes three containers // and names the `Step` this builds. #[allow(clippy::needless_range_loop)] - for thief in 0..self.scenario.cpus { + for thief in 0..self.scenario.cpus() { if halted[thief] { continue; } @@ -909,7 +956,7 @@ impl<'q> Vm<'q> { if self.cpus[thief].running().is_some() || !self.cpus[thief].rq().is_empty() { continue; } - for victim in 0..self.scenario.cpus { + for victim in 0..self.scenario.cpus() { if victim != thief && self.cpus[victim].rq().fair_len() > 0 { steps.push(Step::OldSteal { thief, victim }); } @@ -969,13 +1016,29 @@ impl<'q> Vm<'q> { self.ran.insert(key); } } + for cpu in 0..self.scenario.cpus() { + self.probing[cpu] = self.cpus[cpu].probe_outstanding(); + } + let actor = crate::explore::actor(&step); + // Only a CPU's own step takes a task out of its parked set. + let parked: Vec = + actor.map_or_else(Vec::new, |cpu| self.cpus[cpu].parked().map(|p| p.key()).collect()); self.execute_inner(step, choices); - let owed = self.hw.with(|s| s.need_resched.clone()); - // A CPU number: it reads `owed` and writes `self.resched_at` at the + self.step_trace = self.hw.with(|s| std::mem::take(&mut s.trace)); + self.note_levels(actor, parked); + let (resched, halted) = self.hw.with(|s| (s.need_resched.clone(), s.halted.clone())); + // A CPU number: it reads `resched` and writes `self.resched_at` at the // same index, which is the pairing this loop is about. #[allow(clippy::needless_range_loop)] - for cpu in 0..self.scenario.cpus { - match (owed[cpu], self.resched_at[cpu]) { + for cpu in 0..self.scenario.cpus() { + // Under the old protocol an idle CPU's idle loop is its steal, which + // the explorer schedules as `OldSteal` whenever it likes: that + // freedom is the defect `old_steal_port` reproduces. + let idle = self.scenario.protocol == Protocol::New + && !halted[cpu] + && self.cpus[cpu].running().is_none(); + let owed = resched[cpu] || idle; + match (owed, self.resched_at[cpu]) { (true, None) => self.resched_at[cpu] = Some(self.clock), (false, _) => self.resched_at[cpu] = None, (true, Some(_)) => {} @@ -1009,6 +1072,90 @@ impl<'q> Vm<'q> { } } self.note_probe_gaps(); + self.open_round(); + } + + /// A pass on `cpu` pays the pass it owed, and is its part of the open + /// round; the last one owed completes it. + fn note_pass(&mut self, cpu: usize) { + self.resched_at[cpu] = None; + if std::mem::take(&mut self.round_owed[cpu]) { + self.round_owed_count -= 1; + if self.round_owed_count == 0 { + self.round += 1; + } + } + } + + /// With no round open, open one on every CPU that owes a pass now. A + /// machine with every CPU halted and no IPI on its way opens none until one + /// wakes. + fn open_round(&mut self) { + if self.round_owed_count > 0 { + return; + } + let (halted, pending) = self.hw.with(|s| (s.halted.clone(), s.pending_ipi.clone())); + for cpu in 0..self.scenario.cpus() { + let owes = (!halted[cpu] || pending[cpu] > 0) && !self.is_stopped(cpu); + self.round_owed[cpu] = owes; + self.round_owed_count += usize::from(owes); + } + } + + /// Fold the step into [`Vm::levels`]: the level each kick, migration and + /// wake crossed, and each probe the step consumed with no task handed back, + /// from `actor`, the CPU whose step it was, and `parked`, its parked set + /// before the step. + fn note_levels(&mut self, actor: Option, parked: Vec) { + let topology = &self.scenario.topology; + let at = |a: usize, b: CpuId| crate::topology::slot(topology.level(a, b.0 as usize)); + for target in self.hw.with(|s| std::mem::take(&mut s.kicked)) { + match actor { + Some(from) => self.levels.ipis[at(from, target)] += 1, + None => self.levels.device_ipis += 1, + } + } + for event in &self.step_trace { + if let TraceKind::Migrate { to, .. } = event.kind { + self.levels.migrations[at(event.cpu.0 as usize, to)] += 1; + } + } + // A wake is a task leaving its CPU's parked set, whatever took it out — + // a message, its deadline or a retire — including one this step parked. + if let Some(home) = actor { + let parked_in_step = self.step_trace.iter().filter_map(|event| match event.kind { + TraceKind::ParkCommit { task, .. } => Some(task), + _ => None, + }); + let woken: Vec = parked + .into_iter() + .chain(parked_in_step) + .filter(|&task| self.cpus[home].parked_task(task).is_none()) + .collect(); + for task in woken { + let placed = match self.shared[&task].state() { + TaskState::Ready(cpu) | TaskState::Running(cpu) | TaskState::InTransit(cpu) => { + cpu + } + state => unreachable!("{task:?} left the parked set as {state:?}"), + }; + self.levels.wakes[at(home, placed)] += 1; + } + } + // A probe is consumed only by its victim's drain, so the step that + // consumed it is that victim's. + for thief in 0..self.scenario.cpus() { + if !self.probing[thief] || self.cpus[thief].probe_outstanding() { + continue; + } + let answered = self.step_trace.iter().any(|event| { + matches!(event.kind, TraceKind::Migrate { to, .. } if to.0 as usize == thief) + }); + if !answered { + let victim = actor.expect("a probe is consumed inside its victim's pass"); + self.levels.failed_steals[at(victim, CpuId(thief as u32))] += 1; + } + } } /// [`Vm::probe_gap_ns`]: how long a halted CPU has sat next to a surplus it @@ -1019,14 +1166,16 @@ impl<'q> Vm<'q> { /// with a gap standing still reports it. fn note_probe_gaps(&mut self) { let halted = self.hw.with(|s| s.halted.clone()); + let worth = |cpu: usize| self.handles.get(CpuId(cpu as u32)).surplus() >= PROBE_WORTH_IT; + // Counted once, so a step costs one walk of the machine and not one per + // CPU: at 768 CPUs that is the difference. + let surpluses = (0..self.scenario.cpus()).filter(|&cpu| worth(cpu)).count(); // A CPU number, for [`Vm::execute`]'s reason: it reads `halted`, asks // `self.cpus` and writes `self.probe_gap_since` at one index, and the // pairing across those three is what this loop is about. #[allow(clippy::needless_range_loop)] - for cpu in 0..self.scenario.cpus { - let surplus_next_door = (0..self.scenario.cpus).any(|victim| { - victim != cpu && self.handles.get(CpuId(victim as u32)).surplus() >= PROBE_WORTH_IT - }); + for cpu in 0..self.scenario.cpus() { + let surplus_next_door = surpluses > usize::from(worth(cpu)); let blind = halted[cpu] && surplus_next_door && !self.cpus[cpu].probe_outstanding(); match (blind, self.probe_gap_since[cpu]) { (true, None) => self.probe_gap_since[cpu] = Some(self.clock), @@ -1094,7 +1243,7 @@ impl<'q> Vm<'q> { ChargeShape::Honest => None, ChargeShape::Double { process } => self.scenario.process_index(process), }; - for cpu in 0..self.scenario.cpus { + for cpu in 0..self.scenario.cpus() { let Some(task) = self.cpus[cpu].running() else { continue; }; @@ -1150,7 +1299,7 @@ impl<'q> Vm<'q> { if self.hw.with(|s| s.kicks) == before { return; } - for cpu in 0..self.scenario.cpus { + for cpu in 0..self.scenario.cpus() { if self.hw.with(|s| s.pending_ipi[cpu]) > 0 { self.arm_ipi(cpu); } @@ -1280,7 +1429,11 @@ impl<'q> Vm<'q> { } self.apply(action); self.hw.leave_pass(); + if self.scenario.surplus == SurplusShape::Withheld { + self.handles.get(CpuId(cpu as u32)).publish_surplus(0); + } self.note_kicks(kicks_before); + self.note_pass(cpu); end } @@ -1609,7 +1762,7 @@ impl<'q> Vm<'q> { /// one that was going to happen anyway, so issuing it early perturbs the /// *timing* of the workload and not its token accounting. fn hoist_wake(&self, blocking_cpu: usize, queue: usize) -> Option { - for cpu in 0..self.scenario.cpus { + for cpu in 0..self.scenario.cpus() { if cpu == blocking_cpu { continue; } @@ -1724,7 +1877,7 @@ impl<'q> Vm<'q> { } fn scan_containers(&self, key: TaskKey) -> bool { - (0..self.scenario.cpus) + (0..self.scenario.cpus()) .any(|cpu| kernel::sched::invariants::residents(&self.cpus[cpu]).any(|(k, _)| k == key)) } diff --git a/kernel/sim/src/workload.rs b/kernel/sim/src/workload.rs index 91e49661a93..7299da38712 100644 --- a/kernel/sim/src/workload.rs +++ b/kernel/sim/src/workload.rs @@ -15,6 +15,8 @@ use kernel::sched::cpu::Balance; use kernel::sched::queue::FairOrder; use kernel::sched::task::WaitClass; +use crate::topology::Shape; + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Op { /// Consume CPU time. The VM splits it into chunks so that a quantum can @@ -250,6 +252,25 @@ pub enum PlacementShape { /// one is a measurement rather than a belief. See /// `scenarios::lopsided_placement`. AllOn(usize), + /// Spawn `i` onto the `i`th CPU listed, whatever the counters say: an + /// adversary like [`PlacementShape::AllOn`], for a machine no workload + /// stages. See `scenarios::group_imbalance`. + Listed(&'static [usize]), +} + +/// What a pass's published surplus says to a CPU choosing whom to probe. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SurplusShape { + /// The core's own: every pass publishes the fair band it could give away, + /// the number its push rang on. + Published, + /// Every pass's published surplus is overwritten with zero once it ends, + /// after its push rang a sleeper on the real one: the CPU it rang wakes, + /// finds nothing worth a probe and sleeps again. An adversary like + /// [`PlacementShape::AllOn`] — the defect two spellings of one inequality + /// write (`SchedPass::best_victim`), staged from outside the core, where a + /// rung CPU declines to probe. + Withheld, } /// Which teardown/balance algorithm the VM drives the core with. @@ -268,7 +289,8 @@ pub enum Protocol { #[derive(Clone, Debug)] pub struct Scenario { pub name: &'static str, - pub cpus: usize, + /// The machine: its CPUs, and the cache tree they sit in. + pub topology: Shape, pub queues: Vec, pub procs: Vec, pub irqs: Vec, @@ -303,6 +325,7 @@ pub struct Scenario { /// core implements, so the simulator drives the real code whichever one a /// scenario names. pub balance: Balance, + pub surplus: SurplusShape, /// How the fair band picks between two ready threads of one share. A /// scenario dimension for the same reason [`ShareShape`] is, and the type is /// the core's own rather than a parallel copy of it: the broken orderings @@ -331,6 +354,10 @@ pub struct Scenario { } impl Scenario { + pub fn cpus(&self) -> usize { + self.topology.cpus() + } + /// The longest preempt-off section any thread of this scenario runs. It /// is a *term* of invariant I4's RT latency bound, not an excuse for it: /// making the budget visible is what stops "the sim cannot see kernel @@ -355,8 +382,9 @@ impl Scenario { self } - pub fn with_cpus(mut self, cpus: usize) -> Self { - self.cpus = cpus; + /// The same workload on another machine. + pub fn with_topology(mut self, topology: Shape) -> Self { + self.topology = topology; self } @@ -400,12 +428,17 @@ impl Scenario { /// wrong, and a panic at the first spawn of a sweep would name the VM /// instead of the scenario that asked for it. pub fn with_placement(mut self, placement: PlacementShape) -> Self { - if let PlacementShape::AllOn(cpu) = placement { + let named: &[usize] = match &placement { + PlacementShape::LeastLoadedRotating => &[], + PlacementShape::AllOn(cpu) => std::slice::from_ref(cpu), + PlacementShape::Listed(cpus) => cpus, + }; + for cpu in named { assert!( - cpu < self.cpus, + *cpu < self.cpus(), "{}: placement names cpu{cpu} on a {}-cpu machine", self.name, - self.cpus, + self.cpus(), ); } self.placement = placement; @@ -417,10 +450,10 @@ impl Scenario { /// sweep is the wrong place to find out. pub fn with_stopped(mut self, cpu: usize) -> Self { assert!( - cpu < self.cpus, + cpu < self.cpus(), "{}: stopping cpu{cpu} on a {}-cpu machine", self.name, - self.cpus, + self.cpus(), ); self.stopped = Some(cpu); self @@ -431,6 +464,11 @@ impl Scenario { self } + pub fn with_surplus(mut self, surplus: SurplusShape) -> Self { + self.surplus = surplus; + self + } + pub fn with_order(mut self, order: FairOrder) -> Self { self.order = order; self diff --git a/kernel/sim/tests/policy.rs b/kernel/sim/tests/policy.rs index d6b3d855ca6..e45b7b8af92 100644 --- a/kernel/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -131,7 +131,7 @@ fn stream(scenario: &Scenario, seed: u64) -> ChoiceStream { if seed.is_multiple_of(2) { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) } } @@ -1339,7 +1339,7 @@ fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { // its own exit, and every one of them is on the one CPU. let threads: u64 = scenario.procs.iter().map(|p| p.initial.len() as u64).sum(); assert_eq!( - scenario.cpus, 1, + scenario.cpus(), 1, "{name}: the bound below is one run queue's" ); let bound = (threads + 1) * DISPATCH_NS; diff --git a/kernel/sim/tests/scenarios.rs b/kernel/sim/tests/scenarios.rs index dcac8327b5b..bf3a5ac0382 100644 --- a/kernel/sim/tests/scenarios.rs +++ b/kernel/sim/tests/scenarios.rs @@ -140,7 +140,7 @@ fn old_steal_port_is_caught() { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); if outcome.passed() { @@ -188,7 +188,7 @@ fn old_commit_before_pass_is_caught() { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); if outcome.passed() { @@ -242,7 +242,7 @@ fn old_migrate_keeping_the_corpse_is_caught() { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); if outcome.passed() { @@ -285,7 +285,7 @@ fn rt_starving_the_corpse_is_caught() { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); if outcome.passed() { @@ -321,7 +321,7 @@ fn a_retire_completes_inside_its_derived_bound() { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); assert!(outcome.passed(), "{}", outcome.report()); @@ -370,13 +370,13 @@ fn a_retire_completes_inside_its_derived_bound() { #[test] fn the_unwind_gate_lasts_one_chunk_and_not_one_unwind() { let scenario = scenarios::retire_under_balance(); - let bound = (2 + scenario.cpus as u64) * kernel_sim::vm::RUN_CHUNK_NS; + let bound = (2 + scenario.cpus() as u64) * kernel_sim::vm::RUN_CHUNK_NS; let mut worst = 0; for seed in 0..SEEDS { let mut choices = if seed % 2 == 0 { ChoiceStream::from_seed(seed) } else { - ChoiceStream::pct(seed, scenario.cpus, 3) + ChoiceStream::pct(seed, scenario.cpus(), 3) }; let outcome = run(scenario.clone(), &mut choices); assert!(outcome.passed(), "{}", outcome.report()); @@ -589,7 +589,7 @@ fn corpus_traces_still_do_what_they_were_committed_for() { #[test] fn a_run_replays_exactly() { for scenario in scenarios::all() { - let mut original = ChoiceStream::pct(42, scenario.cpus, 3); + let mut original = ChoiceStream::pct(42, scenario.cpus(), 3); let first = run(scenario.clone(), &mut original); let mut replay = ChoiceStream::replay(first.decisions.clone()); let again = run(scenario.clone(), &mut replay); diff --git a/kernel/sim/tests/scout/amd-bench.log b/kernel/sim/tests/scout/amd-bench.log new file mode 100644 index 00000000000..149d38e0457 --- /dev/null +++ b/kernel/sim/tests/scout/amd-bench.log @@ -0,0 +1,41 @@ +[---------- -------- --.--- cpu0 kernel] ACPI: MADT cpus=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +[2026-10-12 02:42:53 9.395 cpu0 kernel] scout-topo: cpu0 hw=0x0 AuthenticAMD max=0x10/0x80000020 1= 860f01:100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 0:100:0:0 +[2026-10-12 02:42:53 9.395 cpu0 kernel] scout-topo: cpu0 levels B=[ 1:2:100:0 7:10:201:0 0:0:2:0 ] +[2026-10-12 02:42:53 9.395 cpu0 kernel] scout-topo: cpu0 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.406 cpu1 kernel] scout-topo: cpu1 hw=0x1 AuthenticAMD max=0x10/0x80000020 1= 860f01:1100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 1:100:0:0 +[2026-10-12 02:42:53 9.406 cpu1 kernel] scout-topo: cpu1 levels B=[ 1:2:100:1 7:10:201:1 0:0:2:1 ] +[2026-10-12 02:42:53 9.406 cpu1 kernel] scout-topo: cpu1 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.407 cpu0 kernel] SMP: cpu1 tsc=16898130846 inside the BSP's 16880159772..16899922224 (19762452 cycles wide) +[2026-10-12 02:42:53 9.407 cpu0 kernel] SMP: AP cpu1 lapic=1 online +[2026-10-12 02:42:53 9.417 cpu2 kernel] scout-topo: cpu2 hw=0x2 AuthenticAMD max=0x10/0x80000020 1= 860f01:2100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 2:101:0:0 +[2026-10-12 02:42:53 9.417 cpu2 kernel] scout-topo: cpu2 levels B=[ 1:2:100:2 7:10:201:2 0:0:2:2 ] +[2026-10-12 02:42:53 9.417 cpu2 kernel] scout-topo: cpu2 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.418 cpu0 kernel] SMP: cpu2 tsc=16918013268 inside the BSP's 16900041240..16919804142 (19762902 cycles wide) +[2026-10-12 02:42:53 9.418 cpu0 kernel] SMP: AP cpu2 lapic=2 online +[2026-10-12 02:42:53 9.428 cpu3 kernel] scout-topo: cpu3 hw=0x3 AuthenticAMD max=0x10/0x80000020 1= 860f01:3100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 3:101:0:0 +[2026-10-12 02:42:53 9.428 cpu3 kernel] scout-topo: cpu3 levels B=[ 1:2:100:3 7:10:201:3 0:0:2:3 ] +[2026-10-12 02:42:53 9.428 cpu3 kernel] scout-topo: cpu3 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.429 cpu0 kernel] SMP: cpu3 tsc=16937897634 inside the BSP's 16919928234..16939690506 (19762272 cycles wide) +[2026-10-12 02:42:53 9.429 cpu0 kernel] SMP: AP cpu3 lapic=3 online +[2026-10-12 02:42:53 9.439 cpu4 kernel] scout-topo: cpu4 hw=0x4 AuthenticAMD max=0x10/0x80000020 1= 860f01:4100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 4:102:0:0 +[2026-10-12 02:42:53 9.439 cpu4 kernel] scout-topo: cpu4 levels B=[ 1:2:100:4 7:10:201:4 0:0:2:4 ] +[2026-10-12 02:42:53 9.439 cpu4 kernel] scout-topo: cpu4 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.440 cpu0 kernel] SMP: cpu4 tsc=16957797876 inside the BSP's 16939827864..16959590280 (19762416 cycles wide) +[2026-10-12 02:42:53 9.440 cpu0 kernel] SMP: AP cpu4 lapic=4 online +[2026-10-12 02:42:53 9.450 cpu5 kernel] scout-topo: cpu5 hw=0x5 AuthenticAMD max=0x10/0x80000020 1= 860f01:5100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 5:102:0:0 +[2026-10-12 02:42:53 9.450 cpu5 kernel] scout-topo: cpu5 levels B=[ 1:2:100:5 7:10:201:5 0:0:2:5 ] +[2026-10-12 02:42:53 9.450 cpu5 kernel] scout-topo: cpu5 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.451 cpu0 kernel] SMP: cpu5 tsc=16977694824 inside the BSP's 16959725388..16979487660 (19762272 cycles wide) +[2026-10-12 02:42:53 9.451 cpu0 kernel] SMP: AP cpu5 lapic=5 online +[2026-10-12 02:42:53 9.461 cpu6 kernel] scout-topo: cpu6 hw=0x6 AuthenticAMD max=0x10/0x80000020 1= 860f01:6100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 6:103:0:0 +[2026-10-12 02:42:53 9.461 cpu6 kernel] scout-topo: cpu6 levels B=[ 1:2:100:6 7:10:201:6 0:0:2:6 ] +[2026-10-12 02:42:53 9.461 cpu6 kernel] scout-topo: cpu6 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.462 cpu0 kernel] SMP: cpu6 tsc=16997585616 inside the BSP's 16979615568..16999377858 (19762290 cycles wide) +[2026-10-12 02:42:53 9.462 cpu0 kernel] SMP: AP cpu6 lapic=6 online +[2026-10-12 02:42:53 9.472 cpu7 kernel] scout-topo: cpu7 hw=0x7 AuthenticAMD max=0x10/0x80000020 1= 860f01:7100800:76d8320b:178bfbff 80000001= 860f01:0:75c237ff:2fd3fbff 80000008= 3030:90cf757:700f:10000 8000001E= 7:103:0:0 +[2026-10-12 02:42:53 9.472 cpu7 kernel] scout-topo: cpu7 levels B=[ 1:2:100:7 7:10:201:7 0:0:2:7 ] +[2026-10-12 02:42:53 9.472 cpu7 kernel] scout-topo: cpu7 caches 4=[ 0:0:0:0 ] 8000001D=[ 4121:1c0003f:3f:0 4122:1c0003f:3f:0 4143:1c0003f:3ff:2 1c163:3c0003f:fff:1 0:0:0:0 ] +[2026-10-12 02:42:53 9.473 cpu0 kernel] SMP: cpu7 tsc=17017476138 inside the BSP's 16999505820..17019268146 (19762326 cycles wide) +[2026-10-12 02:42:53 9.473 cpu0 kernel] SMP: AP cpu7 lapic=7 online +[2026-10-12 02:42:53 9.473 cpu0 kernel] SMP: roster full at 8 CPUs; ignoring further MADT entries +[2026-10-12 02:42:53 9.473 cpu0 kernel] SMP: 8 of 16 MADT cpus online, 7 of 7 APs inside the BSP's TSC bracket diff --git a/kernel/sim/tests/scout/t14.log b/kernel/sim/tests/scout/t14.log new file mode 100644 index 00000000000..8633dd33a16 --- /dev/null +++ b/kernel/sim/tests/scout/t14.log @@ -0,0 +1,40 @@ +[2026-10-12 06:42:47 11.460 cpu0 kernel] ACPI: MADT cpus=[0, 2, 4, 6, 1, 3, 5, 7] +[2026-10-12 06:42:47 11.558 cpu0 kernel] scout-topo: cpu0 hw=0x0 GenuineIntel max=0x1b/0x80000008 1= 806c1:100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.558 cpu0 kernel] scout-topo: cpu0 levels B=[ 1:2:100:0 4:8:201:0 0:0:2:0 ] +[2026-10-12 06:42:47 11.558 cpu0 kernel] scout-topo: cpu0 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.569 cpu1 kernel] scout-topo: cpu1 hw=0x2 GenuineIntel max=0x1b/0x80000008 1= 806c1:2100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.569 cpu1 kernel] scout-topo: cpu1 levels B=[ 1:2:100:2 4:8:201:2 0:0:2:2 ] +[2026-10-12 06:42:47 11.569 cpu1 kernel] scout-topo: cpu1 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.569 cpu0 kernel] SMP: cpu1 tsc=27986034133 inside the BSP's 27961797498..27989101253 (27303755 cycles wide) +[2026-10-12 06:42:47 11.569 cpu0 kernel] SMP: AP cpu1 lapic=2 online +[2026-10-12 06:42:47 11.580 cpu2 kernel] scout-topo: cpu2 hw=0x4 GenuineIntel max=0x1b/0x80000008 1= 806c1:4100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.580 cpu2 kernel] scout-topo: cpu2 levels B=[ 1:2:100:4 4:8:201:4 0:0:2:4 ] +[2026-10-12 06:42:47 11.580 cpu2 kernel] scout-topo: cpu2 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.580 cpu0 kernel] SMP: cpu2 tsc=28013386024 inside the BSP's 27989159128..28015779152 (26620024 cycles wide) +[2026-10-12 06:42:47 11.580 cpu0 kernel] SMP: AP cpu2 lapic=4 online +[2026-10-12 06:42:47 11.591 cpu3 kernel] scout-topo: cpu3 hw=0x6 GenuineIntel max=0x1b/0x80000008 1= 806c1:6100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.591 cpu3 kernel] scout-topo: cpu3 levels B=[ 1:2:100:6 4:8:201:6 0:0:2:6 ] +[2026-10-12 06:42:47 11.591 cpu3 kernel] scout-topo: cpu3 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.591 cpu0 kernel] SMP: cpu3 tsc=28040058334 inside the BSP's 28015827591..28042483655 (26656064 cycles wide) +[2026-10-12 06:42:47 11.591 cpu0 kernel] SMP: AP cpu3 lapic=6 online +[2026-10-12 06:42:47 11.602 cpu4 kernel] scout-topo: cpu4 hw=0x1 GenuineIntel max=0x1b/0x80000008 1= 806c1:1100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.602 cpu4 kernel] scout-topo: cpu4 levels B=[ 1:2:100:1 4:8:201:1 0:0:2:1 ] +[2026-10-12 06:42:47 11.602 cpu4 kernel] scout-topo: cpu4 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.602 cpu0 kernel] SMP: cpu4 tsc=28066737651 inside the BSP's 28042537058..28069162155 (26625097 cycles wide) +[2026-10-12 06:42:47 11.602 cpu0 kernel] SMP: AP cpu4 lapic=1 online +[2026-10-12 06:42:47 11.613 cpu5 kernel] scout-topo: cpu5 hw=0x3 GenuineIntel max=0x1b/0x80000008 1= 806c1:3100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.613 cpu5 kernel] scout-topo: cpu5 levels B=[ 1:2:100:3 4:8:201:3 0:0:2:3 ] +[2026-10-12 06:42:47 11.613 cpu5 kernel] scout-topo: cpu5 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.613 cpu0 kernel] SMP: cpu5 tsc=28093417832 inside the BSP's 28069213915..28096506772 (27292857 cycles wide) +[2026-10-12 06:42:47 11.613 cpu0 kernel] SMP: AP cpu5 lapic=3 online +[2026-10-12 06:42:47 11.625 cpu6 kernel] scout-topo: cpu6 hw=0x5 GenuineIntel max=0x1b/0x80000008 1= 806c1:5100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.625 cpu6 kernel] scout-topo: cpu6 levels B=[ 1:2:100:5 4:8:201:5 0:0:2:5 ] +[2026-10-12 06:42:47 11.625 cpu6 kernel] scout-topo: cpu6 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.625 cpu0 kernel] SMP: cpu6 tsc=28120768891 inside the BSP's 28096564259..28123708707 (27144448 cycles wide) +[2026-10-12 06:42:47 11.625 cpu0 kernel] SMP: AP cpu6 lapic=5 online +[2026-10-12 06:42:47 11.636 cpu7 kernel] scout-topo: cpu7 hw=0x7 GenuineIntel max=0x1b/0x80000008 1= 806c1:7100800:77fafbbf:bfebfbff 1A= 0:0:0:0 80000001= 0:0:121:2c100800 80000008= 3027:0:0:0 +[2026-10-12 06:42:47 11.636 cpu7 kernel] scout-topo: cpu7 levels B=[ 1:2:100:7 4:8:201:7 0:0:2:7 ] +[2026-10-12 06:42:47 11.636 cpu7 kernel] scout-topo: cpu7 caches 4=[ 1c004121:2c0003f:3f:0 1c004122:1c0003f:3f:0 1c004143:4c0003f:3ff:0 1c03c163:1c0003f:3fff:4 0:0:0:0 ] +[2026-10-12 06:42:47 11.636 cpu0 kernel] SMP: cpu7 tsc=28147963000 inside the BSP's 28123759514..28151135967 (27376453 cycles wide) +[2026-10-12 06:42:47 11.636 cpu0 kernel] SMP: AP cpu7 lapic=7 online +[2026-10-12 06:42:47 11.636 cpu0 kernel] SMP: 8 of 8 MADT cpus online, 7 of 7 APs inside the BSP's TSC bracket diff --git a/kernel/sim/tests/topology.rs b/kernel/sim/tests/topology.rs new file mode 100644 index 00000000000..c2888ce2cee --- /dev/null +++ b/kernel/sim/tests/topology.rs @@ -0,0 +1,271 @@ +//! The scheduler at the shapes of real machines, and work conservation. +//! +//! Every scenario runs at every [`topology::shapes`] machine — the T14 and the +//! AMD bench as their scouts read them, and two permuted synthetic trees up to +//! 768 CPUs — with every invariant walked after every step, work conservation +//! (I15) among them. What each run did at each level of the tree is printed as +//! counts: +//! +//! ```text +//! cargo test -p kernel-sim --test topology -- --nocapture +//! ``` + +use kernel::sched::cpu::{Balance, PUSH_THRESHOLD}; +use kernel_sim::choice::ChoiceStream; +use kernel_sim::explore::{run, Outcome}; +use kernel_sim::scenarios; +use kernel_sim::topology::{self, Keys, LevelCounts, Shape}; +use kernel_sim::workload::{Scenario, SurplusShape}; + +const MS: u64 = 1_000_000; + +fn stream(scenario: &Scenario, seed: u64) -> ChoiceStream { + if seed.is_multiple_of(2) { + ChoiceStream::from_seed(seed) + } else { + ChoiceStream::pct(seed, scenario.cpus(), 3) + } +} + +fn sweep(scenario: &Scenario, seeds: u64) -> Vec { + (0..seeds).map(|seed| run(scenario.clone(), &mut stream(scenario, seed))).collect() +} + +/// `eax:ebx:ecx:edx` quads in hex, from the first `[` after `label` in `line` +/// to its `]`. +fn quads(line: &str, label: &str) -> Vec<[u32; 4]> { + let after = &line[line.find(label).expect("the label is on the line") + label.len()..]; + let inside = &after[after.find('[').expect("a list opens") + 1..after.find(']').expect("and closes")]; + inside + .split_whitespace() + .map(|quad| { + let words: Vec = quad + .split(':') + .map(|word| u32::from_str_radix(word, 16).expect("a hex word")) + .collect(); + [words[0], words[1], words[2], words[3]] + }) + .collect() +} + +/// What a bench's scout logged, decoded the way the SDM and the APM define the +/// fields: each listed CPU's x2APIC id from the MADT, and from every CPU that +/// came up its leaf 0xB SMT and package widths (the subleaves of level type 1 +/// and 2, EAX bits 4:0) and how many ids share its highest data or unified +/// cache (the deterministic cache leaf, 4 on Intel and 0x8000001D on AMD, EAX +/// bits 25:14 plus one). Those three are identical on every CPU that ran, or +/// the shape would not be one tree. +fn scouted(log: &str) -> Vec { + let madt: Vec = log + .lines() + .find_map(|line| line.split("MADT cpus=[").nth(1)) + .expect("the log lists the MADT") + .trim_end_matches(']') + .split(", ") + .map(|id| id.parse().expect("a decimal id")) + .collect(); + let mut widths = Vec::new(); + for line in log.lines().filter(|line| line.contains(" levels B=")) { + let cpu: usize = line + .split("scout-topo: cpu") + .nth(1) + .and_then(|rest| rest.split(' ').next()) + .and_then(|cpu| cpu.parse().ok()) + .expect("the line names its cpu"); + let levels = quads(line, "B="); + assert_eq!(levels[0][3], madt[cpu], "cpu{cpu}'s own x2APIC id is the MADT's"); + let shift = |kind: u32| { + levels.iter().find(|q| (q[2] >> 8) & 0xff == kind).expect("the level is listed")[0] & 0x1f + }; + widths.push((shift(1), shift(2))); + } + let mut sharing = Vec::new(); + for line in log.lines().filter(|line| line.contains(" caches ")) { + let leaf = if line.contains("8000001D=") { "8000001D=" } else { "4=" }; + let (_, shared) = quads(line, leaf) + .into_iter() + .take_while(|q| q[0] & 0x1f != 0) + .filter(|q| matches!(q[0] & 0x1f, 1 | 3)) + .map(|q| ((q[0] >> 5) & 7, ((q[0] >> 14) & 0xfff) + 1)) + .max() + .expect("the CPU has a data cache"); + sharing.push(shared); + } + widths.dedup(); + sharing.dedup(); + let ([(smt, package)], [shared]) = (widths.as_slice(), sharing.as_slice()) else { + panic!("the CPUs that ran disagree: widths {widths:?}, sharing {sharing:?}"); + }; + let llc = shared.next_power_of_two().trailing_zeros(); + madt.iter() + .map(|&id| Keys { core: id >> smt, llc: id >> llc, package: id >> package }) + .collect() +} + +/// **The oracle: each bench's shape is what its own CPUs reported.** The logs +/// are the scouts' boots as posted on the topology measurement +/// (https://github.com/ToyOSOrg/ToyOS/pull/865#issuecomment-6119427274), staged +/// from `042015035`. The AMD bench brought up 8 of its 16 CPUs, so the other +/// eight are read as its MADT lists them under the widths the eight that ran +/// reported — the MADT's ids being APIC ids, as they are for every CPU that +/// came up. +#[test] +fn a_benchs_shape_is_what_its_scout_read() { + for (shape, log) in [ + (Shape::t14(), include_str!("scout/t14.log")), + (Shape::amd_bench(), include_str!("scout/amd-bench.log")), + ] { + let read = scouted(log); + assert_eq!(shape.cpus(), read.len(), "{}: the MADT lists every CPU", shape.name); + for (cpu, keys) in read.into_iter().enumerate() { + assert_eq!(shape.keys(cpu), keys, "{}: cpu{cpu}", shape.name); + } + } +} + +/// Seeds per scenario at a shape: the in-test count for the benches' widths, +/// fewer as the machine widens and every step walks more CPUs. +fn seeds(shape: &Shape) -> u64 { + match shape.cpus() { + 0..=16 => 100, + 17..=64 => 20, + _ => 2, + } +} + +#[test] +fn every_scenario_holds_at_every_shape() { + let mut failures = Vec::new(); + for shape in topology::shapes() { + let mut levels = LevelCounts::default(); + let (mut wc_rounds, mut rounds_to_run, mut runs) = (0, 0, 0); + for scenario in scenarios::all() { + let scenario = scenario.with_topology(shape.clone()); + for outcome in sweep(&scenario, seeds(&shape)) { + if !outcome.passed() { + failures.push(format!("{}: {}", shape.name, outcome.report())); + } + levels.add(&outcome.levels); + wc_rounds = wc_rounds.max(outcome.wc_rounds); + rounds_to_run = rounds_to_run.max(outcome.rounds_to_run); + runs += 1; + } + } + println!( + "{} over {runs} runs: worst idle beside stealable work {wc_rounds} rounds, worst \ + wait to run {rounds_to_run} rounds\n{}", + shape.describe(), + levels.table(), + ); + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// Seeds for the work-conservation cases. +const WC_SEEDS: u64 = 20; + +/// Every thread of a machine-filling workload spawned onto cpu0 of a bench: +/// one victim, every other CPU idle, and only the balance path to fix it. +fn lopsided(shape: Shape, threads: usize) -> Scenario { + scenarios::lopsided_placement(shape.cpus(), threads, 60 * MS).with_topology(shape) +} + +/// **The shipped push keeps every idle CPU inside I15's bound**, and the bound +/// constrains: on the lopsided machine the last CPU its cursor reaches waits +/// out the cursor's walk, up to `cpus` rounds, against `cpus + 2`. +#[test] +fn the_push_keeps_an_idle_cpu_beside_work_inside_its_bound() { + for shape in [Shape::t14(), Shape::amd_bench()] { + let bound = shape.cpus() as u64 + 2; + let scenario = lopsided(shape.clone(), 2 * shape.cpus()); + let mut worst = 0; + for outcome in sweep(&scenario, WC_SEEDS) { + assert!(outcome.passed(), "{}: {}", shape.name, outcome.report()); + worst = worst.max(outcome.wc_rounds); + } + println!("{}: worst idle beside stealable work {worst} rounds, bound {bound}", shape.name); + assert!( + worst * 2 > bound, + "{}: the worst stretch is {worst} rounds against {bound}, more than twice under it — \ + the bound has stopped constraining anything", + shape.name, + ); + } +} + +/// **An idle CPU whose whole CCX looks busy is still reached**: the machine +/// [`scenarios::group_imbalance`] stages, under the shipped balance path. It +/// must hold, and the idle CPU must have been seen beside the work, or the +/// staging put nobody there. +#[test] +fn an_idle_cpu_whose_ccx_looks_busy_is_reached() { + let scenario = scenarios::group_imbalance(); + let mut worst = 0; + for outcome in sweep(&scenario, WC_SEEDS) { + assert!(outcome.passed(), "{}", outcome.report()); + worst = worst.max(outcome.wc_rounds); + } + assert!(worst > 0, "no CPU of the staged machine was ever idle beside stealable work"); +} + +/// **The negative gate: a push that waits for more surplus than a probe takes.** +/// `PUSH_THRESHOLD`'s own doc names the drift: one above the probe's threshold +/// leaves a victim holding one stealable task unannounced, so a CPU that halted +/// before the task was queued is never rung. Three threads on cpu0 is exactly +/// that victim. It must fail, on I15 alone; the shipped threshold on the same +/// machine is the control and must hold. +#[test] +fn a_push_that_waits_for_more_than_a_probe_takes_is_caught() { + for shape in [Shape::t14(), Shape::amd_bench()] { + let shipped = lopsided(shape.clone(), 3); + let late = shipped.clone().with_balance(Balance::PushOnSurplus { + threshold: PUSH_THRESHOLD + 1, + }); + let caught: Vec = sweep(&late, WC_SEEDS).into_iter().filter(|o| !o.passed()).collect(); + assert!( + !caught.is_empty(), + "{}: a push one above the probe's threshold went undetected in {WC_SEEDS} schedules", + shape.name, + ); + for outcome in &caught { + assert!( + outcome.violations.iter().all(|v| v.starts_with("I15")), + "{}: caught by something other than I15: {:?}", + shape.name, + outcome.violations, + ); + } + println!("{}: caught in {} of {WC_SEEDS} schedules", shape.name, caught.len()); + for outcome in sweep(&shipped, WC_SEEDS) { + assert!(outcome.passed(), "{}: {}", shape.name, outcome.report()); + } + } +} + +/// **The negative gate for a CPU that is rung and declines to probe**, on every +/// machine down to two CPUs, where the rung CPU is the only sleeper and the +/// victim the only CPU a round waits on, ringing it in the very pass that +/// completes the round. Every pass's surplus is withheld from the probe after +/// its push rang on it ([`SurplusShape::Withheld`]), so an idle CPU wakes, reads +/// nothing worth a probe and sleeps again for as long as the work lasts. Every +/// schedule must fail, on I15 alone; the published surplus on the same machine +/// is the control and must hold. +#[test] +fn a_cpu_that_is_rung_and_declines_to_probe_is_caught() { + for shape in [Shape::flat(2), Shape::t14(), Shape::amd_bench()] { + let shipped = lopsided(shape.clone(), 3); + let declined = shipped.clone().with_surplus(SurplusShape::Withheld); + for outcome in sweep(&declined, WC_SEEDS) { + assert!(!outcome.passed(), "{}: a declined probe went undetected", shape.name); + assert!( + outcome.violations.iter().all(|v| v.starts_with("I15")), + "{}: caught by something other than I15: {:?}", + shape.name, + outcome.violations, + ); + } + for outcome in sweep(&shipped, WC_SEEDS) { + assert!(outcome.passed(), "{}: {}", shape.name, outcome.report()); + } + } +}