x86-64 kernel leaves are global under a declared CR4.PGE, and only a kernel-half change drops them

kernel/loom/Cargo.toml

@@ -322,4 +322,4 @@ test = false
 # library's tests and the simulator run. A `cfg` name and not a feature, since a
 # feature here is a model control.
 [lints.rust]
-unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators", "mask-windows", "sched-check", "sched-tripwire", "protocol-port", "placement-ignores-staleness"))'] }
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators", "test-actuators", "mask-windows", "sched-check", "sched-tripwire", "protocol-port", "placement-ignores-staleness"))'] }

kernel/loom/tests/counters_round.rs

@@ -20,7 +20,7 @@
 use std::sync::atomic::{AtomicBool, Ordering::SeqCst};
 
 use kernel_loom::seqlock::Published;
-use kernel_loom::shootdown::Shootdown;
+use kernel_loom::shootdown::{Reach, Shootdown};
 use loom::sync::Arc;
 
 /// What the CPU's block held before the round, and what it answers the round with.
@@ -44,11 +44,11 @@ fn a_cpu_read_as_answered_is_read_with_its_answer() {
         let rounds = Arc::new(Shootdown::new());
         let block = Arc::new(Published::new());
         block.publish(BEFORE);
-        let generation = rounds.issue();
+        let generation = rounds.issue(Reach::Narrow);
 
         let cpu1 = {
             let (rounds, block) = (rounds.clone(), block.clone());
-            loom::thread::spawn(move || rounds.serve(1, || block.publish(ANSWER)))
+            loom::thread::spawn(move || rounds.serve(1, |_| block.publish(ANSWER)))
         };
 
         for _ in 0..POLLS {

kernel/loom/tests/tlb_shootdown.rs

@@ -10,6 +10,8 @@
 //! to mean and the only one expressible here. So `UNMAPPED` after an
 //! acknowledged shootdown is "cpu 1 can no longer reach the page the initiator
 //! is about to free", and `MAPPED` is the use-after-free this stage removes.
+//! A kernel-half entry is a second cell, re-walked only by a wide answer, as a
+//! global translation is dropped only by the flush that drops global ones.
 //!
 //! The edge the property rests on is `serve`'s load of what it owes: it
 //! synchronizes with `issue`'s release, and that is what puts the initiator's
@@ -39,7 +41,7 @@
 
 use std::sync::atomic::{AtomicBool, Ordering::SeqCst};
 
-use kernel_loom::shootdown::Shootdown;
+use kernel_loom::shootdown::{Reach, Shootdown};
 use loom::sync::Arc;
 use loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
 
@@ -85,10 +87,14 @@ static ACKED: AtomicBool = AtomicBool::new(false);
 
 struct Machine {
     shootdown: Shootdown,
-    /// The page-table entry the initiator unmaps.
+    /// The user-half page-table entry the initiator unmaps.
     pte: AtomicU64,
-    /// What cpu 1's last page walk found. A flush re-walks.
+    /// What cpu 1's last page walk of it found. Every flush re-walks.
     tlb: AtomicU64,
+    /// The kernel-half entry, whose translation is global.
+    kernel_pte: AtomicU64,
+    /// What cpu 1's last walk of it found. Only a wide flush re-walks.
+    kernel_tlb: AtomicU64,
 }
 
 impl Machine {
@@ -97,28 +103,47 @@ impl Machine {
             shootdown: Shootdown::new(),
             pte: AtomicU64::new(MAPPED),
             tlb: AtomicU64::new(MAPPED),
+            kernel_pte: AtomicU64::new(MAPPED),
+            kernel_tlb: AtomicU64::new(MAPPED),
         })
     }
 
+    /// cpu 1's flush, as far as `reach` asks.
+    fn flush(&self, reach: Reach) {
+        self.tlb.store(self.pte.load(Relaxed), Relaxed);
+        if reach == Reach::Wide {
+            self.kernel_tlb.store(self.kernel_pte.load(Relaxed), Relaxed);
+        }
+    }
+
+    /// One serve of vector 0xFE's body on cpu 1.
+    fn serve(&self) {
+        self.shootdown.serve(1, |reach| self.flush(reach));
+    }
+
     /// cpu 1's side: take the interrupt and run vector 0xFE's body.
     fn take_interrupts(&self) {
         for _ in 0..SERVES {
             loom::thread::yield_now();
-            self.shootdown
-                .serve(1, || self.tlb.store(self.pte.load(Relaxed), Relaxed));
+            self.serve();
         }
     }
 
-    /// The initiator's side, up to the point where it would free the pages.
-    /// `None` when the bounded poll ran out, which is a schedule this model
-    /// says nothing about.
-    fn unmap_and_wait(&self) -> Option<u64> {
-        self.pte.store(UNMAPPED, Relaxed);
-        let generation = self.shootdown.issue();
+    /// The initiator's side, up to the point where it would free the pages:
+    /// `pte` written for a user-half change, `kernel_pte` for a kernel-half
+    /// one, and what cpu 1 holds of it once it has answered. `None` when the
+    /// bounded poll ran out, which is a schedule this model says nothing about.
+    fn unmap_and_wait(&self, reach: Reach) -> Option<u64> {
+        let (pte, tlb) = match reach {
+            Reach::Narrow => (&self.pte, &self.tlb),
+            Reach::Wide => (&self.kernel_pte, &self.kernel_tlb),
+        };
+        pte.store(UNMAPPED, Relaxed);
+        let generation = self.shootdown.issue(reach);
         for _ in 0..POLLS {
             if self.shootdown.served(1, generation) {
                 ACKED.store(true, SeqCst);
-                return Some(self.tlb.load(Relaxed));
+                return Some(tlb.load(Relaxed));
             }
             loom::thread::yield_now();
         }
@@ -138,7 +163,7 @@ fn an_acknowledged_flush_postdates_the_page_table_write() {
             loom::thread::spawn(move || m.take_interrupts())
         };
 
-        if let Some(reachable) = m.unmap_and_wait() {
+        if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) {
             assert_eq!(
                 reachable, UNMAPPED,
                 "cpu 1 acknowledged the shootdown while still holding a \
@@ -174,10 +199,10 @@ fn one_serve_answers_two_concurrent_shootdowns() {
 
         let second = {
             let m = m.clone();
-            loom::thread::spawn(move || m.unmap_and_wait())
+            loom::thread::spawn(move || m.unmap_and_wait(Reach::Narrow))
         };
 
-        if let Some(reachable) = m.unmap_and_wait() {
+        if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) {
             assert_eq!(reachable, UNMAPPED, "the first initiator freed a page cpu 1 could still reach");
         }
         if let Some(reachable) = second.join().unwrap() {
@@ -213,25 +238,25 @@ fn an_initiator_answers_while_it_waits() {
 
         // cpu 0 unmaps and answers for itself; cpu 1 has not issued yet, so the
         // generation cpu 0 publishes is its own.
-        let g0 = s.issue();
-        s.serve(0, || {});
+        let g0 = s.issue(Reach::Narrow);
+        s.serve(0, |_| {});
 
         // cpu 1 unmaps while cpu 0 is between its own `serve` and its wait.
-        let g1 = s.issue();
+        let g1 = s.issue(Reach::Narrow);
 
         // cpu 0's wait is not satisfied — cpu 1 has not flushed — and that turn
         // is nevertheless where cpu 1's answer comes from.
-        assert!(!s.wait_turn(0, 1, g0, || {}));
+        assert!(!s.wait_turn(0, 1, g0, |_| {}));
         assert!(
             s.served(0, g1),
             "cpu 0 is waiting for cpu 1 and has stopped answering it, which is \
              a wait neither CPU can leave",
         );
 
         // And cpu 1's own wait then has an answer to find.
-        s.serve(1, || {});
-        assert!(s.wait_turn(1, 0, g1, || {}));
-        assert!(s.wait_turn(0, 1, g0, || {}));
+        s.serve(1, |_| {});
+        assert!(s.wait_turn(1, 0, g1, |_| {}));
+        assert!(s.wait_turn(0, 1, g0, |_| {}));
     });
 }
 
@@ -245,16 +270,16 @@ fn an_initiator_answers_while_it_waits() {
 fn a_nested_serve_is_not_undone_by_the_one_it_interrupted() {
     model(|| {
         let s = Arc::new(Shootdown::new());
-        let first = s.issue();
+        let first = s.issue(Reach::Narrow);
         let nested = {
             let s = Arc::clone(&s);
             loom::thread::spawn(move || {
-                let later = s.issue();
-                s.serve(1, || {});
+                let later = s.issue(Reach::Narrow);
+                s.serve(1, |_| {});
                 later
             })
         };
-        s.serve(1, || {});
+        s.serve(1, |_| {});
         let later = nested.join().unwrap();
         assert!(s.served(1, first));
         assert!(
@@ -264,3 +289,140 @@ fn a_nested_serve_is_not_undone_by_the_one_it_interrupted() {
         );
     });
 }
+
+/// A kernel-half shootdown acknowledged means cpu 1 has dropped its global
+/// translation since the write.
+///
+/// The edge is `issue`'s count of wide generations, raised before the
+/// generation exists: a serve that owes the generation reads the count, so its
+/// answer is wide. Reds when the count is raised after the generation is
+/// issued, where a serve can owe it and still answer narrow.
+#[test]
+fn an_acknowledged_kernel_half_shootdown_dropped_the_global_translation() {
+    ACKED.store(false, SeqCst);
+    model(|| {
+        let m = Machine::new();
+
+        let target = {
+            let m = m.clone();
+            loom::thread::spawn(move || m.take_interrupts())
+        };
+
+        if let Some(reachable) = m.unmap_and_wait(Reach::Wide) {
+            assert_eq!(
+                reachable, UNMAPPED,
+                "cpu 1 acknowledged a kernel-half shootdown while still holding the \
+                 global translation the initiator changed",
+            );
+        }
+
+        target.join().unwrap();
+    });
+    assert!(
+        ACKED.load(SeqCst),
+        "no interleaving completed the wait, so the assertion never ran",
+    );
+}
+
+/// One serve may answer a user-half and a kernel-half shootdown at once, and
+/// its answer is then the wide one: the user-half initiator's page is gone
+/// either way, and the kernel-half initiator's global translation with it.
+#[test]
+fn one_serve_answering_a_user_half_and_a_kernel_half_shootdown_answers_wide() {
+    ACKED.store(false, SeqCst);
+    model(|| {
+        let m = Machine::new();
+
+        let target = {
+            let m = m.clone();
+            loom::thread::spawn(move || m.take_interrupts())
+        };
+
+        let kernel = {
+            let m = m.clone();
+            loom::thread::spawn(move || m.unmap_and_wait(Reach::Wide))
+        };
+
+        if let Some(reachable) = m.unmap_and_wait(Reach::Narrow) {
+            assert_eq!(reachable, UNMAPPED, "the user-half initiator freed a page cpu 1 could still reach");
+        }
+        if let Some(reachable) = kernel.join().unwrap() {
+            assert_eq!(reachable, UNMAPPED, "the kernel-half initiator's global translation outlived cpu 1's answer");
+        }
+
+        target.join().unwrap();
+    });
+    assert!(
+        ACKED.load(SeqCst),
+        "no interleaving completed a wait, so the assertions never ran",
+    );
+}
+
+/// A serve nested inside a wide one that has not yet flushed answers wide
+/// itself: the record of a wide answer is raised only once its flush is done.
+///
+/// The two serves are two threads on cpu 1, so loom places the nested one at
+/// every point of the outer one. Reds when the record is raised before the
+/// flush, where the nested serve reads it, answers narrow, and publishes the
+/// generation while the global translation is still held.
+#[test]
+fn a_serve_nested_in_an_unfinished_wide_one_answers_wide() {
+    ACKED.store(false, SeqCst);
+    model(|| {
+        let m = Machine::new();
+
+        let outer = {
+            let m = m.clone();
+            loom::thread::spawn(move || m.serve())
+        };
+        let nested = {
+            let m = m.clone();
+            loom::thread::spawn(move || m.serve())
+        };
+
+        if let Some(reachable) = m.unmap_and_wait(Reach::Wide) {
+            assert_eq!(
+                reachable, UNMAPPED,
+                "a nested serve published the kernel-half shootdown while the wide \
+                 answer it nested in had not yet dropped the global translation",
+            );
+        }
+
+        outer.join().unwrap();
+        nested.join().unwrap();
+    });
+    assert!(
+        ACKED.load(SeqCst),
+        "no interleaving completed the wait, so the assertion never ran",
+    );
+}
+
+/// A serve owing only user-half shootdowns answers narrow, before and after a
+/// kernel-half one, which it answers wide once: a user-half change keeps the
+/// kernel's global translations.
+///
+/// **Not an interleaving question**, as `an_initiator_answers_while_it_waits`
+/// is not; reds when a serve answers wide with no wide generation owed.
+#[test]
+fn only_a_kernel_half_shootdown_asks_a_wide_answer() {
+    model(|| {
+        let s = Shootdown::new();
+        let reach_of = |s: &Shootdown| {
+            let mut answered = None;
+            s.serve(1, |reach| answered = Some(reach));
+            answered.expect("serve runs its flush")
+        };
+
+        s.issue(Reach::Narrow);
+        assert_eq!(reach_of(&s), Reach::Narrow, "a user-half shootdown was answered wide");
+        s.issue(Reach::Wide);
+        s.issue(Reach::Narrow);
+        assert_eq!(reach_of(&s), Reach::Wide, "a kernel-half shootdown was answered narrow");
+        s.issue(Reach::Narrow);
+        assert_eq!(
+            reach_of(&s),
+            Reach::Narrow,
+            "a user-half shootdown after an answered kernel-half one was answered wide",
+        );
+    });
+}

kernel/src/arch/aarch64/paging.rs

@@ -903,3 +903,10 @@ pub fn scanout_whole_pages(phys: u64, size: u64) -> Result<(), &'static str> {
     }
     Ok(())
 }
+
+/// x86-64's stages a global kernel leaf going stale, and this architecture's
+/// kernel leaves are not judged by it: refused.
+#[cfg(feature = "test-actuators")]
+pub fn debug_global_leaf(_how: u64) -> u64 {
+    toyos_abi::syscall::SyscallError::NotSupported.to_u64()
+}