diff --git a/issues/ci-runs-the-whole-guest-suite.md b/issues/ci-runs-the-whole-guest-suite.md index 3de92351150..9433460b4fa 100644 --- a/issues/ci-runs-the-whole-guest-suite.md +++ b/issues/ci-runs-the-whole-guest-suite.md @@ -17,4 +17,6 @@ dispatches the reshape. **Exit condition**: `--ci guest` runs a declared list in `tests/toyos.rs` and nothing else, the rest of the suite runs on demand, and root `CLAUDE.md`'s "The guest suite is their `guest / -suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff. +suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff. The same diff deletes +`ON_DEMAND` in `tests/toyos.rs`, the stopgap that keeps a test out of an unfiltered run until then: +once `--ci guest` runs a declared list, everything outside it is on demand. diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index e18419ff407..2556eb2bbd5 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -119,17 +119,25 @@ memory beyond what 2 MiB process pages allow **No linker runs inside ToyOS** (owner, 2026-10-04: "Just get rid of toyos ld"). `/system/bin/toyos-ld` went with its crate, its `[programs]` row and its -host tests, and nothing replaced it: a ToyOS process cannot link an object, and +host tests, and no image ships a linker: `ld.lld` links inside ToyOS only +where `hosted_clang_hello` stages it, and the ToyOS-hosted rustc names no linker the guest has (`issues/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). *Exit*: the commit that lands M2's compile-and-run test, which links inside the guest. -**What stops M2: clang and lld running inside ToyOS.** `cargo run -- +**What stops M2: the driver's link and the install.** `cargo run -- --hosted-clang` makes a clang and an `ld.lld` for `x86_64-unknown-toyos`, static PIEs that name no library (`src/hostedclang.rs`): 147.5 MB and -85.6 MB, 121.9 MB and 70.9 MB stripped. No guest has run either. -- Run: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` lists, and - `issues/libc-has-no-pread-or-pwrite.md`. +85.6 MB, 121.9 MB and 70.9 MB stripped. Both run in a guest: +`hosted_clang_hello` (`tests/common/hostedclang.rs`, run on demand) stages them +on ROOT, compiles `hello.c` with `clang -c`, links it by running `ld.lld` on the +line the driver gives it, runs the program, and both products are the host's +bytes. +- Link through the driver: `clang hello.c` starts LLD as a child, which libc + cannot (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`). +- Install: `pkg` installs neither (**To build**, "`pkg` installs a toolchain"). +- Run past `hello.c`: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` + lists, and `issues/libc-has-no-pread-or-pwrite.md`. **What M3 adds: a rustc that carries that LLVM**, after all of M2's. - Build: `issues/rustc-llvm-cannot-build-for-a-toyos-host.md`, and diff --git a/src/build.rs b/src/build.rs index 984b49a4d48..1dfa648b923 100644 --- a/src/build.rs +++ b/src/build.rs @@ -2954,6 +2954,7 @@ mod tests { "tests/acpicase/system.toml", "tests/blockgrantcase/system.toml", "tests/consentcase/system.toml", + "tests/hostedclangcase/system.toml", "tests/jobcase/system.toml", "tests/latencycase/system.toml", "tests/logstallcase/system.toml", diff --git a/src/hostedclang.rs b/src/hostedclang.rs index a4480cf4b6d..ea41a79f106 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -5,7 +5,8 @@ //! host makes both //! (`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`). //! -//! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is +//! **Made only when asked for**, by `cargo run -- --hosted-clang` and by the +//! guest test that runs it (`tests/common/hostedclang.rs`): its build is //! LLVM's, and its key moves with every sysroot's, so no other build makes it. //! //! **A function of its key** ([`key_of`]): the host LLVM's key, which names the @@ -97,12 +98,7 @@ pub struct HostedClang { /// `cargo run -- --hosted-clang`: make this tree's, and name each binary with /// its size. pub fn dispatch(root: &Path) { - let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); - let sysroot = crate::toolchain::ensure(root, &mut lock); - let fork = crate::sysroot::fork_checkout(root, &mut lock); - let store = keystore::host(); - let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock); - let hosted = resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)); + let hosted = ensure(root); for (_, kept) in BINARIES { let binary = hosted.dir.join("bin").join(kept); let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); @@ -110,10 +106,20 @@ pub fn dispatch(root: &Path) { } } +/// This tree's, made if nobody on this host has made it. +pub fn ensure(root: &Path) -> HostedClang { + let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); + let sysroot = crate::toolchain::ensure(root, &mut lock); + let fork = crate::sysroot::fork_checkout(root, &mut lock); + let store = keystore::host(); + let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock); + resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)) +} + /// The product of the LLVM at `llvm`, the one the fork `fork` names, and the /// sysroot at `sysroot`, both held in use in `store`: made if nobody on this /// host has made it, under `ninja`. -pub fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { +fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { let key = key_of(&stored(llvm), &stored(sysroot), &options()); let dir = Keyed::HostedClang.store(store).join(&key); let make = || place(fork, llvm, sysroot, ninja, &key, &dir); diff --git a/tests/checks.rs b/tests/checks.rs index 2cab3761f6a..d8149a545bc 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -780,9 +780,10 @@ mod checks { .collect() }; let names = |of: &[&str]| -> BTreeSet { of.iter().map(|n| n.to_string()).collect() }; - let every: BTreeSet = declared().map(String::from).collect(); - let cases: [(&[&str], _); 6] = [ + let every: BTreeSet = declared().filter(|n| *n != "hosted_clang_hello").map(String::from).collect(); + let cases: [(&[&str], _); 7] = [ (&[], every), + (&["hosted_clang_hello"], names(&["hosted_clang_hello"])), (&["virt_el2"], names(&["virt_el2_drop"])), (&["el2_drop"], names(&["virt_el2_drop"])), (&["nested_nmi"], names(&["nested_nmi_is_loud"])), diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs new file mode 100644 index 00000000000..ebeb96d801f --- /dev/null +++ b/tests/common/hostedclang.rs @@ -0,0 +1,130 @@ +//! clang and LLD built for ToyOS (`src/hostedclang.rs`), run inside a guest: +//! M2 of `issues/toyos-builds-itself.md`. +//! +//! The binaries, clang's resource headers and the C sysroot's headers and +//! `libtoyos_c.a` go on the ROOT of `tests/hostedclangcase`, as fixtures +//! beside the image's own files: ROOT is memory the loader filled, so the +//! guest reads them with no disk and no server behind it. The job compiles and +//! links `hello.c` there, runs it, and compares each product with what the +//! host's clang and LLD, of the same LLVM commit, made of the same input on the +//! same command line. + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + + +use super::{compile, qemu}; +use qemu::{BootOptions, QemuInstance}; + +/// The job, and the guest binary it runs. +const JOB: &str = "hosted_clang_hello"; + +/// Where on ROOT the job's directory is, and so where the guest reads it. +const STAGED: &str = "share/hosted-clang"; + +/// What `hello.c` prints. +const HELLO: &str = "hello from a C program compiled and linked inside ToyOS"; + +/// The host's half: its clang and LLD on `hello.c`, the guest's on the same +/// file with the same arguments, and the guest's verdict on their bytes. +pub fn hosted_clang_hello() -> Result<(), String> { + let hosted = toyos_build::hostedclang::ensure(&compile::repo_root()); + let c = compile::c_sysroot(); + let case = compile::repo_root().join("tests/hostedclangcase"); + + let scratch = super::lane::dir().join(JOB); + fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; + let (object, linked) = (scratch.join("hello.o"), scratch.join("hello")); + let mut cc = Command::new(&c.clang); + cc.args(c.args()).args(["-c", "hello.c", "-o"]).arg(&object).current_dir(&case); + host_run(cc)?; + // The line the ToyOS driver hands LLD (`clang -###`), as the guest runs it. + let lld = c.clang.with_file_name("ld.lld"); + let mut ld = Command::new(&lld); + ld.arg(format!("--sysroot={}", c.dir.display())) + .args(["-pie", "--eh-frame-hdr", "-o"]) + .arg(&linked) + .arg(format!("-L{}", c.dir.join("lib").display())) + .arg(&object) + .arg("-ltoyos_c"); + host_run(ld)?; + + let mut staged: Vec<(String, Vec)> = Vec::new(); + let mut stage = |rel: &str, from: &Path| { + let bytes = fs::read(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())); + staged.push((format!("{STAGED}/{rel}"), bytes)); + }; + stage("bin/clang", &hosted.dir.join("bin/clang")); + stage("bin/ld.lld", &hosted.dir.join("bin/ld.lld")); + stage("sysroot/lib/libtoyos_c.a", &c.dir.join("lib/libtoyos_c.a")); + stage("src/hello.c", &case.join("hello.c")); + for (name, product) in [("hello.o", &object), ("hello", &linked)] { + let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?; + let sha256: String = toyos_sha2_hw::sha256_digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); + eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {sha256}", bytes.len()); + stage(&format!("host/{name}"), product); + } + // clang finds its resource headers beside itself, `../lib/clang`. + for (rel, from) in tree(&hosted.dir.join("lib/clang")) { + stage(&format!("lib/clang/{rel}"), &from); + } + // The C headers alone: `include/c++` is libc++'s, which no C compile reads. + for (rel, from) in tree(&c.dir.join("include")).into_iter().filter(|(rel, _)| !rel.starts_with("c++/")) { + stage(&format!("sysroot/include/{rel}"), &from); + } + fs::remove_dir_all(&scratch).map_err(|e| format!("remove {}: {e}", scratch.display()))?; + + let options = BootOptions { extra_root_files: staged, ..Default::default() }; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/toyos-rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(&case, &[], &[(JOB.to_string(), bin)], options); + let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED}"), Duration::from_secs(600)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + for said in [&format!("{JOB}: hello said: {HELLO}"), &format!("{JOB}: ok")] { + if !result.stdout.lines().any(|l| l.trim_end().ends_with(said.as_str())) { + return Err(format!("the job never said `{said}`:\n{}", result.stdout)); + } + } + for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: "))) { + eprintln!(" [{JOB}] {}", line.trim()); + } + Ok(()) +} + +/// Run a host tool, and refuse its failure with what it said. +fn host_run(mut command: Command) -> Result<(), String> { + let done = command.output().map_err(|e| format!("run {command:?}: {e}"))?; + if !done.status.success() { + return Err(format!( + "{command:?} ended {}:\n{}{}", + done.status, + String::from_utf8_lossy(&done.stdout), + String::from_utf8_lossy(&done.stderr) + )); + } + Ok(()) +} + +/// Every file under `dir`, by its path relative to `dir`. +fn tree(dir: &Path) -> Vec<(String, std::path::PathBuf)> { + let mut files = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(at) = pending.pop() { + for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); + if path.is_dir() { + pending.push(path); + } else { + let rel = path.strip_prefix(dir).expect("under the walked directory"); + files.push((rel.to_string_lossy().into_owned(), path)); + } + } + } + files +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 1a4e52d7ea3..4f638ff5624 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,6 +7,7 @@ pub mod compile; /// The device boot: what `tests/metaldevicecase` measures. pub mod devices; pub mod faults; +pub mod hostedclang; /// The HTTPS server a guest's client fetches from. pub mod https; pub mod iommu; diff --git a/tests/hostedclangcase/hello.c b/tests/hostedclangcase/hello.c new file mode 100644 index 00000000000..12df55322cc --- /dev/null +++ b/tests/hostedclangcase/hello.c @@ -0,0 +1,6 @@ +#include + +int main(void) { + printf("hello from a C program compiled and linked inside ToyOS\n"); + return 0; +} diff --git a/tests/hostedclangcase/system.toml b/tests/hostedclangcase/system.toml new file mode 100644 index 00000000000..95de68ebdc8 --- /dev/null +++ b/tests/hostedclangcase/system.toml @@ -0,0 +1,11 @@ +# The boot `hosted_clang_hello` stages ToyOS's own clang and LLD on, with the C +# sysroot they read: a ROOT of its own, since they are a quarter of a gigabyte +# and no other test reads them. +[boot] +start = ["logkeeper", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] + +[programs.test-runner] diff --git a/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs b/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs new file mode 100644 index 00000000000..e2dd9ec9d74 --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs @@ -0,0 +1,91 @@ +//! clang and `ld.lld` built for ToyOS compile and link a C program here, the +//! program runs, and what each wrote is the host's clang's and LLD's bytes. +//! +//! Its one argument is the directory the harness staged on ROOT: `bin/clang` +//! and `bin/ld.lld` with clang's resource headers at `lib/clang`, the C +//! sysroot at `sysroot`, the program at `src/hello.c`, and what the host's +//! clang and LLD made of it at `host/hello.o` and `host/hello`. LLD is run +//! directly, on the line the ToyOS driver gives it, because a driver that links +//! starts LLD as a child and libc starts none. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Where this job writes: `/tmp` is the guest's one writable tree. +const OUT: &str = "/tmp/hosted_clang_hello"; + +fn main() { + let staged = PathBuf::from(std::env::args().nth(1).expect("the staged directory, as the one argument")); + let sysroot = staged.join("sysroot"); + let out = Path::new(OUT); + std::fs::create_dir_all(out).unwrap_or_else(|e| panic!("create {OUT}: {e}")); + let (object, linked) = (out.join("hello.o"), out.join("hello")); + + // The arguments are the host's, so the two products can be compared. + let mut compile = Command::new(staged.join("bin/clang")); + compile + .arg("--target=x86_64-unknown-toyos") + .arg(format!("--sysroot={}", sysroot.display())) + .args(["-c", "hello.c", "-o"]) + .arg(&object) + .current_dir(staged.join("src")); + run(compile, "clang -c"); + + let mut link = Command::new(staged.join("bin/ld.lld")); + link.arg(format!("--sysroot={}", sysroot.display())) + .args(["-pie", "--eh-frame-hdr", "-o"]) + .arg(&linked) + .arg(format!("-L{}", sysroot.join("lib").display())) + .arg(&object) + .arg("-ltoyos_c"); + run(link, "ld.lld"); + + let ran = Command::new(&linked).output().unwrap_or_else(|e| panic!("run {}: {e}", linked.display())); + assert!(ran.status.success(), "{} ended {:?}: {}", linked.display(), ran.status, String::from_utf8_lossy(&ran.stderr)); + for line in String::from_utf8_lossy(&ran.stdout).lines() { + println!("hosted_clang_hello: hello said: {line}"); + } + + let mut differ = false; + for (made, by) in [(&object, "clang"), (&linked, "ld.lld")] { + let name = made.file_name().expect("a file name"); + differ |= !same(made, &staged.join("host").join(name), by); + } + assert!(!differ, "the guest's products are not the host's bytes"); + println!("hosted_clang_hello: ok"); +} + +/// Run `command`, and refuse its failure with what it said. +fn run(mut command: Command, what: &str) { + let done = command.output().unwrap_or_else(|e| panic!("{what} did not start: {e}")); + assert!( + done.status.success(), + "{what} ended {:?}:\n{}{}", + done.status, + String::from_utf8_lossy(&done.stdout), + String::from_utf8_lossy(&done.stderr) + ); + println!("hosted_clang_hello: {what} ended 0"); +} + +/// Whether `made` holds the bytes of `host`, saying so, and where they first +/// part if they do not. +fn same(made: &Path, host: &Path, by: &str) -> bool { + let read = |p: &Path| std::fs::read(p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())); + let (ours, theirs) = (read(made), read(host)); + if ours == theirs { + println!("hosted_clang_hello: {by}'s {} is the host's {} bytes", made.display(), ours.len()); + return true; + } + let at = ours.iter().zip(&theirs).position(|(a, b)| a != b).unwrap_or(ours.len().min(theirs.len())); + let window = |bytes: &[u8]| bytes[at..bytes.len().min(at + 32)].iter().map(|b| format!("{b:02x}")).collect::(); + println!( + "hosted_clang_hello: {by}'s {} is {} bytes and the host's {}; they first part at {at:#x}: here {}, the host's {}", + made.display(), + ours.len(), + theirs.len(), + window(&ours), + window(&theirs) + ); + false +} diff --git a/tests/toyos.rs b/tests/toyos.rs index cdd8bb8fb04..d28a9df3523 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -193,6 +193,9 @@ const RUST_SKIP: &[&str] = &[ // It claims QEMU's virtio NIC, which the T14 has none of: `bar_map_again` // runs it. "bar_map_again", + // It needs ToyOS's own clang and LLD on ROOT, which only its boot stages: + // `hosted_clang_hello` runs it on tests/hostedclangcase. + "hosted_clang_hello", // It claims the NVMe controller a boot off that disk starts no server // for, and reads the inventory: `block_grants_reach_their_partitions` // runs it on tests/blockgrantcase. @@ -420,6 +423,10 @@ const MACHINE_TESTS: &[&str] = &[ // reads it have no host build, and the T14 boots from a stick beside an // NVMe disk that is another system's. "nvme_disk_keeps_log_and_home", + // clang and LLD built for ToyOS, compiling and linking inside it: the + // claim is that they run on ToyOS, which no host test can make, and the + // T14 would read nothing a guest does not. On demand: [`ON_DEMAND`]. + "hosted_clang_hello", // A badge the kernel stamps on a connection to diskserver, judged by // diskserver on the disk it drives: the grant's decisions are host-tested // in diskserver and toyos-blockring, but whether the badge diskserver @@ -3916,6 +3923,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "console_image_boots" => console_image_boots(), "desktop_without_x2apic" => desktop_without_x2apic(), "nvme_disk_keeps_log_and_home" => nvme_disk_keeps_log_and_home(test_config), + "hosted_clang_hello" => common::hostedclang::hosted_clang_hello(), "block_grants_reach_their_partitions" => block_grants_reach_their_partitions(), "update_writes_the_idle_slot_through_the_block_service" => update_writes_the_idle_slot_through_the_block_service(), "usbd_drives_the_spare" => usbd_drives_the_spare(test_config), @@ -6747,9 +6755,15 @@ fn build_tasks( machine.chain(screen).collect() } -/// Whether a run takes `name`: it has no filter, or one that matches it. +/// Machine tests a run boots only when a filter names them, so no unfiltered +/// run, CI's included, boots one: each needs a product only +/// `cargo run -- --hosted-clang` builds, which no CI job holds. +const ON_DEMAND: &[&str] = &["hosted_clang_hello"]; + +/// Whether a run takes `name`: a filter matches it, or there is none and it is +/// not [`ON_DEMAND`]. fn kept(filters: &[&str], name: &str) -> bool { - filters.is_empty() || filters.iter().any(|f| name.contains(f)) + filters.iter().any(|f| name.contains(f)) || (filters.is_empty() && !ON_DEMAND.contains(&name)) } /// The machine tests and the screen tests a run boots.