From 6b2edd91c1a114a8c87c992553d4a0ddcf944e35 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 10:55:19 +0200 Subject: [PATCH 1/8] hosted_clang_hello runs ToyOS's own clang and ld.lld in a guest; the loader refuses both at spawn M2a of issues/toyos-builds-itself.md. A machine test stages the clang and ld.lld that --hosted-clang builds, clang's resource headers, the C sysroot's C headers and libtoyos_c.a, hello.c, and what the host's clang and LLD (same LLVM commit) made of it, on the ROOT of a boot of its own (tests/hostedclangcase). ROOT, because the loader reads it into memory whole and the guest reads it with no disk or server behind it; 265 MB here (0xfd00000), in a 4 GiB guest. The guest job runs clang -c with the host's arguments, ld.lld directly on the line the ToyOS driver gives it (clang -### on the host), runs the result, and compares both products with the host's byte for byte. It is red: the kernel's loader refuses both binaries at spawn, each with a table past one kernel allocation (MAX_HEAP_ALLOC, 2,093,056 bytes): spawn: /system/share/hosted-clang/bin/clang: DT_RELASZ declares 5044488 bytes, past one kernel allocation spawn: /system/share/hosted-clang/bin/ld.lld: DT_STRSZ declares 2813680 bytes, past one kernel allocation issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md records it with what each binary declares. hostedclang::ensure is what dispatch did, so the harness makes the product the way --hosted-clang does; resolve is private now. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...e-elf-tables-pass-one-kernel-allocation.md | 45 ++++++ src/build.rs | 1 + src/hostedclang.rs | 22 +-- tests/common/hostedclang.rs | 130 ++++++++++++++++++ tests/common/mod.rs | 1 + tests/hostedclangcase/hello.c | 6 + tests/hostedclangcase/system.toml | 11 ++ .../src/bin/hosted_clang_hello.rs | 91 ++++++++++++ tests/toyos.rs | 8 ++ 9 files changed, 307 insertions(+), 8 deletions(-) create mode 100644 issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md create mode 100644 tests/common/hostedclang.rs create mode 100644 tests/hostedclangcase/hello.c create mode 100644 tests/hostedclangcase/system.toml create mode 100644 tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs diff --git a/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md b/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md new file mode 100644 index 00000000000..05d5fa9ed71 --- /dev/null +++ b/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md @@ -0,0 +1,45 @@ +--- +status: open +kind: defect +opened: 2026-10-10 +--- + +# The program loader refuses clang and LLD, whose ELF tables pass one kernel allocation + +A spawn reads each table the executable's dynamic section names into one +kernel heap allocation, and refuses the binary with `ResourceExhausted` when a +table passes `mm::MAX_HEAP_ALLOC`, 2 MiB less a page, 2,093,056 bytes +(`kernel/src/loader/mod.rs`, `read_elf_table`); the relocation index it builds +is held to the same bound (`kernel/src/elf/index.rs`, +`RelocationIndex::with_capacity`). std reports the refusal as "out of memory". + +The clang and `ld.lld` that `cargo run -- --hosted-clang` builds for +`x86_64-unknown-toyos` (`src/hostedclang.rs`, product `6079d1a299fd8083`) are +both refused at spawn by `hosted_clang_hello` (`tests/common/hostedclang.rs`), +on the kernel's lines: + +``` +spawn: /system/share/hosted-clang/bin/clang: DT_RELASZ declares 5044488 bytes, past one kernel allocation +spawn: /system/share/hosted-clang/bin/ld.lld: DT_STRSZ declares 2813680 bytes, past one kernel allocation +``` + +The loader logs only the first table it refuses. What each binary declares, +read with `readelf -d -S`: + +| | clang | ld.lld | +|---|---|---| +| `DT_RELASZ` (all `R_X86_64_RELATIVE`) | 5,044,488 | 1,958,616 | +| `.dynstr` (`DT_STRSZ`) | 5,737,227 | 2,813,680 | +| `.dynsym` | 1,674,696 | 912,072 | +| relocation index, 16 bytes a `RELATIVE` entry | 3,362,992 | 1,305,744 | + +Not measured: how far `LLVM_ENABLE_PLUGINS=OFF`, which stops clang exporting +its symbols, shrinks `.dynstr` and `.dynsym`; it cannot shrink clang's +`DT_RELASZ` or its index, which count relative relocations. Packing them as +`DT_RELR` would, and the loader reads no `DT_RELR` +(`issues/the-program-loader-runs-a-binary-whose-relr-table-it-never-reads.md`). + +Owner: M2 of `issues/toyos-builds-itself.md`. + +Exit: `hosted_clang_hello` gets past both spawns, and a table size a file +declares still cannot make the kernel allocate past a bound the loader states. diff --git a/src/build.rs b/src/build.rs index b383792baab..61a8ae4499a 100644 --- a/src/build.rs +++ b/src/build.rs @@ -2880,6 +2880,7 @@ mod tests { "diag/system.toml", "console/system.toml", "tests/acpicase/system.toml", + "tests/hostedclangcase/system.toml", "tests/jobcase/system.toml", "tests/latencycase/system.toml", "tests/logstallcase/system.toml", diff --git a/src/hostedclang.rs b/src/hostedclang.rs index a4480cf4b6d..ea41a79f106 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -5,7 +5,8 @@ //! host makes both //! (`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`). //! -//! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is +//! **Made only when asked for**, by `cargo run -- --hosted-clang` and by the +//! guest test that runs it (`tests/common/hostedclang.rs`): its build is //! LLVM's, and its key moves with every sysroot's, so no other build makes it. //! //! **A function of its key** ([`key_of`]): the host LLVM's key, which names the @@ -97,12 +98,7 @@ pub struct HostedClang { /// `cargo run -- --hosted-clang`: make this tree's, and name each binary with /// its size. pub fn dispatch(root: &Path) { - let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); - let sysroot = crate::toolchain::ensure(root, &mut lock); - let fork = crate::sysroot::fork_checkout(root, &mut lock); - let store = keystore::host(); - let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock); - let hosted = resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)); + let hosted = ensure(root); for (_, kept) in BINARIES { let binary = hosted.dir.join("bin").join(kept); let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); @@ -110,10 +106,20 @@ pub fn dispatch(root: &Path) { } } +/// This tree's, made if nobody on this host has made it. +pub fn ensure(root: &Path) -> HostedClang { + let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); + let sysroot = crate::toolchain::ensure(root, &mut lock); + let fork = crate::sysroot::fork_checkout(root, &mut lock); + let store = keystore::host(); + let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock); + resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)) +} + /// The product of the LLVM at `llvm`, the one the fork `fork` names, and the /// sysroot at `sysroot`, both held in use in `store`: made if nobody on this /// host has made it, under `ninja`. -pub fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { +fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { let key = key_of(&stored(llvm), &stored(sysroot), &options()); let dir = Keyed::HostedClang.store(store).join(&key); let make = || place(fork, llvm, sysroot, ninja, &key, &dir); diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs new file mode 100644 index 00000000000..8c023cb3a91 --- /dev/null +++ b/tests/common/hostedclang.rs @@ -0,0 +1,130 @@ +//! clang and LLD built for ToyOS (`src/hostedclang.rs`), run inside a guest: +//! M2a of `issues/toyos-builds-itself.md`. +//! +//! The binaries, clang's resource headers and the C sysroot's headers and +//! `libtoyos_c.a` go on the ROOT of `tests/hostedclangcase`, as fixtures +//! beside the image's own files: ROOT is memory the loader filled, so the +//! guest reads them with no disk and no server behind it. The job compiles and +//! links `hello.c` there, runs it, and compares each product with what the +//! host's clang and LLD, of the same LLVM commit, made of the same input on the +//! same command line. + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use sha2::{Digest, Sha256}; + +use super::{compile, qemu}; +use qemu::{BootOptions, QemuInstance}; + +/// The job, and the guest binary it runs. +const JOB: &str = "hosted_clang_hello"; + +/// Where on ROOT the job's directory is, and so where the guest reads it. +const STAGED: &str = "share/hosted-clang"; + +/// What `hello.c` prints. +const HELLO: &str = "hello from a C program compiled and linked inside ToyOS"; + +/// The host's half: its clang and LLD on `hello.c`, the guest's on the same +/// file with the same arguments, and the guest's verdict on their bytes. +pub fn hosted_clang_hello() -> Result<(), String> { + let hosted = toyos_build::hostedclang::ensure(&compile::repo_root()); + let c = compile::c_sysroot(); + let case = compile::repo_root().join("tests/hostedclangcase"); + + let scratch = super::lane::dir().join(JOB); + fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; + let (object, linked) = (scratch.join("hello.o"), scratch.join("hello")); + let mut cc = Command::new(&c.clang); + cc.args(c.args()).args(["-c", "hello.c", "-o"]).arg(&object).current_dir(&case); + host_run(cc)?; + // The line the ToyOS driver hands LLD (`clang -###`), as the guest runs it. + let lld = c.clang.with_file_name("ld.lld"); + let mut ld = Command::new(&lld); + ld.arg(format!("--sysroot={}", c.dir.display())) + .args(["-pie", "--eh-frame-hdr", "-o"]) + .arg(&linked) + .arg(format!("-L{}", c.dir.join("lib").display())) + .arg(&object) + .arg("-ltoyos_c"); + host_run(ld)?; + + let mut staged: Vec<(String, Vec)> = Vec::new(); + let mut stage = |rel: &str, from: &Path| { + let bytes = fs::read(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())); + staged.push((format!("{STAGED}/{rel}"), bytes)); + }; + stage("bin/clang", &hosted.dir.join("bin/clang")); + stage("bin/ld.lld", &hosted.dir.join("bin/ld.lld")); + stage("sysroot/lib/libtoyos_c.a", &c.dir.join("lib/libtoyos_c.a")); + stage("src/hello.c", &case.join("hello.c")); + for (name, product) in [("hello.o", &object), ("hello", &linked)] { + let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?; + eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {:x}", bytes.len(), Sha256::digest(&bytes)); + stage(&format!("host/{name}"), product); + } + // clang finds its resource headers beside itself, `../lib/clang`. + for (rel, from) in tree(&hosted.dir.join("lib/clang")) { + stage(&format!("lib/clang/{rel}"), &from); + } + // The C headers alone: `include/c++` is libc++'s, which no C compile reads. + for (rel, from) in tree(&c.dir.join("include")).into_iter().filter(|(rel, _)| !rel.starts_with("c++/")) { + stage(&format!("sysroot/include/{rel}"), &from); + } + fs::remove_dir_all(&scratch).map_err(|e| format!("remove {}: {e}", scratch.display()))?; + + let options = BootOptions { extra_root_files: staged, ..Default::default() }; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/toyos-rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(&case, &[], &[(JOB.to_string(), bin)], options); + let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED}"), Duration::from_secs(600)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + for said in [&format!("{JOB}: hello said: {HELLO}"), &format!("{JOB}: ok")] { + if !result.stdout.lines().any(|l| l.trim_end().ends_with(said.as_str())) { + return Err(format!("the job never said `{said}`:\n{}", result.stdout)); + } + } + for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: "))) { + eprintln!(" [{JOB}] {}", line.trim()); + } + Ok(()) +} + +/// Run a host tool, and refuse its failure with what it said. +fn host_run(mut command: Command) -> Result<(), String> { + let done = command.output().map_err(|e| format!("run {command:?}: {e}"))?; + if !done.status.success() { + return Err(format!( + "{command:?} ended {}:\n{}{}", + done.status, + String::from_utf8_lossy(&done.stdout), + String::from_utf8_lossy(&done.stderr) + )); + } + Ok(()) +} + +/// Every file under `dir`, by its path relative to `dir`. +fn tree(dir: &Path) -> Vec<(String, std::path::PathBuf)> { + let mut files = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(at) = pending.pop() { + for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); + if path.is_dir() { + pending.push(path); + } else { + let rel = path.strip_prefix(dir).expect("under the walked directory"); + files.push((rel.to_string_lossy().into_owned(), path)); + } + } + } + files +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 291983fd0b6..98f80076e01 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,6 +7,7 @@ pub mod compile; /// The device boot: what `tests/metaldevicecase` measures. pub mod devices; pub mod faults; +pub mod hostedclang; pub mod iommu; pub mod irqcensus; /// The `isa` claim's rows on the T14. diff --git a/tests/hostedclangcase/hello.c b/tests/hostedclangcase/hello.c new file mode 100644 index 00000000000..12df55322cc --- /dev/null +++ b/tests/hostedclangcase/hello.c @@ -0,0 +1,6 @@ +#include + +int main(void) { + printf("hello from a C program compiled and linked inside ToyOS\n"); + return 0; +} diff --git a/tests/hostedclangcase/system.toml b/tests/hostedclangcase/system.toml new file mode 100644 index 00000000000..95de68ebdc8 --- /dev/null +++ b/tests/hostedclangcase/system.toml @@ -0,0 +1,11 @@ +# The boot `hosted_clang_hello` stages ToyOS's own clang and LLD on, with the C +# sysroot they read: a ROOT of its own, since they are a quarter of a gigabyte +# and no other test reads them. +[boot] +start = ["logkeeper", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] + +[programs.test-runner] diff --git a/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs b/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs new file mode 100644 index 00000000000..e2dd9ec9d74 --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs @@ -0,0 +1,91 @@ +//! clang and `ld.lld` built for ToyOS compile and link a C program here, the +//! program runs, and what each wrote is the host's clang's and LLD's bytes. +//! +//! Its one argument is the directory the harness staged on ROOT: `bin/clang` +//! and `bin/ld.lld` with clang's resource headers at `lib/clang`, the C +//! sysroot at `sysroot`, the program at `src/hello.c`, and what the host's +//! clang and LLD made of it at `host/hello.o` and `host/hello`. LLD is run +//! directly, on the line the ToyOS driver gives it, because a driver that links +//! starts LLD as a child and libc starts none. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Where this job writes: `/tmp` is the guest's one writable tree. +const OUT: &str = "/tmp/hosted_clang_hello"; + +fn main() { + let staged = PathBuf::from(std::env::args().nth(1).expect("the staged directory, as the one argument")); + let sysroot = staged.join("sysroot"); + let out = Path::new(OUT); + std::fs::create_dir_all(out).unwrap_or_else(|e| panic!("create {OUT}: {e}")); + let (object, linked) = (out.join("hello.o"), out.join("hello")); + + // The arguments are the host's, so the two products can be compared. + let mut compile = Command::new(staged.join("bin/clang")); + compile + .arg("--target=x86_64-unknown-toyos") + .arg(format!("--sysroot={}", sysroot.display())) + .args(["-c", "hello.c", "-o"]) + .arg(&object) + .current_dir(staged.join("src")); + run(compile, "clang -c"); + + let mut link = Command::new(staged.join("bin/ld.lld")); + link.arg(format!("--sysroot={}", sysroot.display())) + .args(["-pie", "--eh-frame-hdr", "-o"]) + .arg(&linked) + .arg(format!("-L{}", sysroot.join("lib").display())) + .arg(&object) + .arg("-ltoyos_c"); + run(link, "ld.lld"); + + let ran = Command::new(&linked).output().unwrap_or_else(|e| panic!("run {}: {e}", linked.display())); + assert!(ran.status.success(), "{} ended {:?}: {}", linked.display(), ran.status, String::from_utf8_lossy(&ran.stderr)); + for line in String::from_utf8_lossy(&ran.stdout).lines() { + println!("hosted_clang_hello: hello said: {line}"); + } + + let mut differ = false; + for (made, by) in [(&object, "clang"), (&linked, "ld.lld")] { + let name = made.file_name().expect("a file name"); + differ |= !same(made, &staged.join("host").join(name), by); + } + assert!(!differ, "the guest's products are not the host's bytes"); + println!("hosted_clang_hello: ok"); +} + +/// Run `command`, and refuse its failure with what it said. +fn run(mut command: Command, what: &str) { + let done = command.output().unwrap_or_else(|e| panic!("{what} did not start: {e}")); + assert!( + done.status.success(), + "{what} ended {:?}:\n{}{}", + done.status, + String::from_utf8_lossy(&done.stdout), + String::from_utf8_lossy(&done.stderr) + ); + println!("hosted_clang_hello: {what} ended 0"); +} + +/// Whether `made` holds the bytes of `host`, saying so, and where they first +/// part if they do not. +fn same(made: &Path, host: &Path, by: &str) -> bool { + let read = |p: &Path| std::fs::read(p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())); + let (ours, theirs) = (read(made), read(host)); + if ours == theirs { + println!("hosted_clang_hello: {by}'s {} is the host's {} bytes", made.display(), ours.len()); + return true; + } + let at = ours.iter().zip(&theirs).position(|(a, b)| a != b).unwrap_or(ours.len().min(theirs.len())); + let window = |bytes: &[u8]| bytes[at..bytes.len().min(at + 32)].iter().map(|b| format!("{b:02x}")).collect::(); + println!( + "hosted_clang_hello: {by}'s {} is {} bytes and the host's {}; they first part at {at:#x}: here {}, the host's {}", + made.display(), + ours.len(), + theirs.len(), + window(&ours), + window(&theirs) + ); + false +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 008c522eced..891e8c10df1 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -175,6 +175,9 @@ const RUST_SKIP: &[&str] = &[ // It claims QEMU's virtio NIC, which the T14 has none of: `bar_map_again` // runs it. "bar_map_again", + // It needs ToyOS's own clang and LLD on ROOT, which only its boot stages: + // `hosted_clang_hello` runs it on tests/hostedclangcase. + "hosted_clang_hello", ]; /// The shared boot's last members, in this order: each fills a bound of its @@ -334,6 +337,10 @@ const MACHINE_TESTS: &[&str] = &[ // reads it have no host build, and the T14 boots from a stick beside an // NVMe disk that is another system's. "nvme_disk_keeps_log_and_home", + // clang and LLD built for ToyOS, compiling and linking inside it: the + // claim is that they run on ToyOS, which no host test can make, and the + // T14 would read nothing a guest does not. + "hosted_clang_hello", ]; /// **The metal profile**: which registrations run on the ThinkPad T14, what @@ -3200,6 +3207,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "bar_map_again" => bar_map_again(test_config), "console_image_boots" => console_image_boots(), "nvme_disk_keeps_log_and_home" => nvme_disk_keeps_log_and_home(test_config), + "hosted_clang_hello" => common::hostedclang::hosted_clang_hello(), other => Err(format!("unknown machine test {other}")), } } From 46a27b9251ad6a46d7e2494597f6962a10ea10e3 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 10 Oct 2026 21:19:55 +0200 Subject: [PATCH 2/8] hosted_clang_hello hashes the host's products with toyos-sha2-hw main took sha2 out of the build crate; the digest is toyos_sha2_hw's, printed as hex the way tests/toyos.rs prints one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/common/hostedclang.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs index 8c023cb3a91..baf731ccf19 100644 --- a/tests/common/hostedclang.rs +++ b/tests/common/hostedclang.rs @@ -14,7 +14,6 @@ use std::path::Path; use std::process::Command; use std::time::Duration; -use sha2::{Digest, Sha256}; use super::{compile, qemu}; use qemu::{BootOptions, QemuInstance}; @@ -63,7 +62,8 @@ pub fn hosted_clang_hello() -> Result<(), String> { stage("src/hello.c", &case.join("hello.c")); for (name, product) in [("hello.o", &object), ("hello", &linked)] { let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?; - eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {:x}", bytes.len(), Sha256::digest(&bytes)); + let sha256: String = toyos_sha2_hw::sha256_digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); + eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {sha256}", bytes.len()); stage(&format!("host/{name}"), product); } // clang finds its resource headers beside itself, `../lib/clang`. From 90a594e4b450893c7fe042852865d4a269aacf54 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:15:32 +0200 Subject: [PATCH 3/8] hosted_clang_hello runs on demand, only when a filter is its whole name The owner's ruling: it is an on-demand guest test, not CI's. It needs the hosted clang, which only `cargo run -- --hosted-clang` builds and no CI job holds, so no unfiltered run boots it, and a substring filter that happens to match it does not either. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/toyos.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index 9052163ae3d..ab0a9afb0f0 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -425,7 +425,7 @@ const MACHINE_TESTS: &[&str] = &[ "nvme_disk_keeps_log_and_home", // clang and LLD built for ToyOS, compiling and linking inside it: the // claim is that they run on ToyOS, which no host test can make, and the - // T14 would read nothing a guest does not. + // T14 would read nothing a guest does not. On demand: [`ON_DEMAND`]. "hosted_clang_hello", // A badge the kernel stamps on a connection to diskserver, judged by // diskserver on the disk it drives: the grant's decisions are host-tested @@ -6755,8 +6755,17 @@ fn build_tasks( machine.chain(screen).collect() } -/// Whether a run takes `name`: it has no filter, or one that matches it. +/// Machine tests a run boots only when a filter is the whole name, so no +/// unfiltered run, CI's included, boots one: each needs a product only +/// `cargo run -- --hosted-clang` builds, which no CI job holds. +const ON_DEMAND: &[&str] = &["hosted_clang_hello"]; + +/// Whether a run takes `name`: it has no filter, or one that matches it; one +/// [`ON_DEMAND`] is taken only by its whole name. fn kept(filters: &[&str], name: &str) -> bool { + if ON_DEMAND.contains(&name) { + return filters.contains(&name); + } filters.is_empty() || filters.iter().any(|f| name.contains(f)) } From bcf0800cd9aeaf5692fb9044dae2d7826fb3aa4d Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:16:14 +0200 Subject: [PATCH 4/8] The track says clang and LLD run in a guest, and the loader's refusal of them is closed #836 relocates an executable in userland, so the kernel reads no table its dynamic section names and allocates nothing a file declares: the loader issue's second half holds, and hosted_clang_hello gets past both spawns, its first. That issue was this branch's own and is cited nowhere else. toyos-builds-itself.md's M2 paragraph said no guest had run either binary. It now says what hosted_clang_hello shows, and what M2's exit still waits on: the driver's link, which starts LLD as a child, and pkg's install. The test's header cited an "M2a" the track never named; it cites M2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...e-elf-tables-pass-one-kernel-allocation.md | 45 ------------------- issues/toyos-builds-itself.md | 18 +++++--- tests/common/hostedclang.rs | 2 +- 3 files changed, 14 insertions(+), 51 deletions(-) delete mode 100644 issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md diff --git a/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md b/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md deleted file mode 100644 index 05d5fa9ed71..00000000000 --- a/issues/the-program-loader-refuses-clang-and-lld-whose-elf-tables-pass-one-kernel-allocation.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-10 ---- - -# The program loader refuses clang and LLD, whose ELF tables pass one kernel allocation - -A spawn reads each table the executable's dynamic section names into one -kernel heap allocation, and refuses the binary with `ResourceExhausted` when a -table passes `mm::MAX_HEAP_ALLOC`, 2 MiB less a page, 2,093,056 bytes -(`kernel/src/loader/mod.rs`, `read_elf_table`); the relocation index it builds -is held to the same bound (`kernel/src/elf/index.rs`, -`RelocationIndex::with_capacity`). std reports the refusal as "out of memory". - -The clang and `ld.lld` that `cargo run -- --hosted-clang` builds for -`x86_64-unknown-toyos` (`src/hostedclang.rs`, product `6079d1a299fd8083`) are -both refused at spawn by `hosted_clang_hello` (`tests/common/hostedclang.rs`), -on the kernel's lines: - -``` -spawn: /system/share/hosted-clang/bin/clang: DT_RELASZ declares 5044488 bytes, past one kernel allocation -spawn: /system/share/hosted-clang/bin/ld.lld: DT_STRSZ declares 2813680 bytes, past one kernel allocation -``` - -The loader logs only the first table it refuses. What each binary declares, -read with `readelf -d -S`: - -| | clang | ld.lld | -|---|---|---| -| `DT_RELASZ` (all `R_X86_64_RELATIVE`) | 5,044,488 | 1,958,616 | -| `.dynstr` (`DT_STRSZ`) | 5,737,227 | 2,813,680 | -| `.dynsym` | 1,674,696 | 912,072 | -| relocation index, 16 bytes a `RELATIVE` entry | 3,362,992 | 1,305,744 | - -Not measured: how far `LLVM_ENABLE_PLUGINS=OFF`, which stops clang exporting -its symbols, shrinks `.dynstr` and `.dynsym`; it cannot shrink clang's -`DT_RELASZ` or its index, which count relative relocations. Packing them as -`DT_RELR` would, and the loader reads no `DT_RELR` -(`issues/the-program-loader-runs-a-binary-whose-relr-table-it-never-reads.md`). - -Owner: M2 of `issues/toyos-builds-itself.md`. - -Exit: `hosted_clang_hello` gets past both spawns, and a table size a file -declares still cannot make the kernel allocate past a bound the loader states. diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index e18419ff407..2556eb2bbd5 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -119,17 +119,25 @@ memory beyond what 2 MiB process pages allow **No linker runs inside ToyOS** (owner, 2026-10-04: "Just get rid of toyos ld"). `/system/bin/toyos-ld` went with its crate, its `[programs]` row and its -host tests, and nothing replaced it: a ToyOS process cannot link an object, and +host tests, and no image ships a linker: `ld.lld` links inside ToyOS only +where `hosted_clang_hello` stages it, and the ToyOS-hosted rustc names no linker the guest has (`issues/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). *Exit*: the commit that lands M2's compile-and-run test, which links inside the guest. -**What stops M2: clang and lld running inside ToyOS.** `cargo run -- +**What stops M2: the driver's link and the install.** `cargo run -- --hosted-clang` makes a clang and an `ld.lld` for `x86_64-unknown-toyos`, static PIEs that name no library (`src/hostedclang.rs`): 147.5 MB and -85.6 MB, 121.9 MB and 70.9 MB stripped. No guest has run either. -- Run: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` lists, and - `issues/libc-has-no-pread-or-pwrite.md`. +85.6 MB, 121.9 MB and 70.9 MB stripped. Both run in a guest: +`hosted_clang_hello` (`tests/common/hostedclang.rs`, run on demand) stages them +on ROOT, compiles `hello.c` with `clang -c`, links it by running `ld.lld` on the +line the driver gives it, runs the program, and both products are the host's +bytes. +- Link through the driver: `clang hello.c` starts LLD as a child, which libc + cannot (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`). +- Install: `pkg` installs neither (**To build**, "`pkg` installs a toolchain"). +- Run past `hello.c`: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` + lists, and `issues/libc-has-no-pread-or-pwrite.md`. **What M3 adds: a rustc that carries that LLVM**, after all of M2's. - Build: `issues/rustc-llvm-cannot-build-for-a-toyos-host.md`, and diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs index baf731ccf19..ebeb96d801f 100644 --- a/tests/common/hostedclang.rs +++ b/tests/common/hostedclang.rs @@ -1,5 +1,5 @@ //! clang and LLD built for ToyOS (`src/hostedclang.rs`), run inside a guest: -//! M2a of `issues/toyos-builds-itself.md`. +//! M2 of `issues/toyos-builds-itself.md`. //! //! The binaries, clang's resource headers and the C sysroot's headers and //! `libtoyos_c.a` go on the ROOT of `tests/hostedclangcase`, as fixtures From 1c0fd478b321e66afb66c95223e74f9fa834637f Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:27:05 +0200 Subject: [PATCH 5/8] A filter only an on-demand test's name contains is refused, not run as nothing The dead-filter check matched by substring, so `-- hosted` passed it and then selected nothing: `running 0 tests`, exit 0. It asks `kept` now, the rule select uses. `-- hosted` exits 1, "No test matches filter", and with the old line restored it exits 0 on 0 tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/toyos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index ab0a9afb0f0..b64113625e8 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -6972,7 +6972,7 @@ fn main() { // A filter that takes nothing would be dropped in silence beside one that // takes something. - if let Some(dead) = filters.iter().find(|f| !declared().any(|name| name.contains(**f))) { + if let Some(dead) = filters.iter().find(|f| !declared().any(|name| kept(std::slice::from_ref(f), name))) { eprintln!("No test matches filter {dead:?}"); run.exit(1); } From 650613b0114c778ca39dc7d55800e0093f599781 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:36:25 +0200 Subject: [PATCH 6/8] a_run_selects_by_filter holds the on-demand rule An unfiltered run takes every declared test but hosted_clang_hello, a filter only its name contains takes nothing, and its whole name takes it. With ON_DEMAND emptied the check exits 101 on the substring case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/checks.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/checks.rs b/tests/checks.rs index 2cab3761f6a..6c5b140cd44 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -780,9 +780,11 @@ mod checks { .collect() }; let names = |of: &[&str]| -> BTreeSet { of.iter().map(|n| n.to_string()).collect() }; - let every: BTreeSet = declared().map(String::from).collect(); - let cases: [(&[&str], _); 6] = [ + let every: BTreeSet = declared().filter(|n| !ON_DEMAND.contains(n)).map(String::from).collect(); + let cases: [(&[&str], _); 8] = [ (&[], every), + (&["hosted_clang"], BTreeSet::new()), + (&["hosted_clang_hello"], names(&["hosted_clang_hello"])), (&["virt_el2"], names(&["virt_el2_drop"])), (&["el2_drop"], names(&["virt_el2_drop"])), (&["nested_nmi"], names(&["nested_nmi_is_loud"])), From 8ea0f825b0a95e5c626461ab5e4fdef5a7ba8504 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:52:23 +0200 Subject: [PATCH 7/8] An on-demand test is left out of an unfiltered run, and a filter matches it like any other Whole-name matching for ON_DEMAND went beyond the owner's ruling, which asks only that CI's unfiltered run not boot hosted_clang_hello. One way of matching remains: a filter that is a substring of a name runs it, so the dead-filter check is main's again and the `hosted_clang` case goes. The empty-filter case names the excluded test literally rather than reading ON_DEMAND, so emptying the list turns it red. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- tests/checks.rs | 5 ++--- tests/toyos.rs | 15 ++++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/checks.rs b/tests/checks.rs index 6c5b140cd44..d8149a545bc 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -780,10 +780,9 @@ mod checks { .collect() }; let names = |of: &[&str]| -> BTreeSet { of.iter().map(|n| n.to_string()).collect() }; - let every: BTreeSet = declared().filter(|n| !ON_DEMAND.contains(n)).map(String::from).collect(); - let cases: [(&[&str], _); 8] = [ + let every: BTreeSet = declared().filter(|n| *n != "hosted_clang_hello").map(String::from).collect(); + let cases: [(&[&str], _); 7] = [ (&[], every), - (&["hosted_clang"], BTreeSet::new()), (&["hosted_clang_hello"], names(&["hosted_clang_hello"])), (&["virt_el2"], names(&["virt_el2_drop"])), (&["el2_drop"], names(&["virt_el2_drop"])), diff --git a/tests/toyos.rs b/tests/toyos.rs index b64113625e8..d28a9df3523 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -6755,18 +6755,15 @@ fn build_tasks( machine.chain(screen).collect() } -/// Machine tests a run boots only when a filter is the whole name, so no -/// unfiltered run, CI's included, boots one: each needs a product only +/// Machine tests a run boots only when a filter names them, so no unfiltered +/// run, CI's included, boots one: each needs a product only /// `cargo run -- --hosted-clang` builds, which no CI job holds. const ON_DEMAND: &[&str] = &["hosted_clang_hello"]; -/// Whether a run takes `name`: it has no filter, or one that matches it; one -/// [`ON_DEMAND`] is taken only by its whole name. +/// Whether a run takes `name`: a filter matches it, or there is none and it is +/// not [`ON_DEMAND`]. fn kept(filters: &[&str], name: &str) -> bool { - if ON_DEMAND.contains(&name) { - return filters.contains(&name); - } - filters.is_empty() || filters.iter().any(|f| name.contains(f)) + filters.iter().any(|f| name.contains(f)) || (filters.is_empty() && !ON_DEMAND.contains(&name)) } /// The machine tests and the screen tests a run boots. @@ -6972,7 +6969,7 @@ fn main() { // A filter that takes nothing would be dropped in silence beside one that // takes something. - if let Some(dead) = filters.iter().find(|f| !declared().any(|name| kept(std::slice::from_ref(f), name))) { + if let Some(dead) = filters.iter().find(|f| !declared().any(|name| name.contains(**f))) { eprintln!("No test matches filter {dead:?}"); run.exit(1); } From 0e3de9158b26aed0e94d26ef68feb855bfb53740 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 11 Oct 2026 11:52:24 +0200 Subject: [PATCH 8/8] The CI-tier issue's exit deletes ON_DEMAND Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- issues/ci-runs-the-whole-guest-suite.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/issues/ci-runs-the-whole-guest-suite.md b/issues/ci-runs-the-whole-guest-suite.md index 3de92351150..9433460b4fa 100644 --- a/issues/ci-runs-the-whole-guest-suite.md +++ b/issues/ci-runs-the-whole-guest-suite.md @@ -17,4 +17,6 @@ dispatches the reshape. **Exit condition**: `--ci guest` runs a declared list in `tests/toyos.rs` and nothing else, the rest of the suite runs on demand, and root `CLAUDE.md`'s "The guest suite is their `guest / -suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff. +suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff. The same diff deletes +`ON_DEMAND` in `tests/toyos.rs`, the stopgap that keeps a test out of an unfiltered run until then: +once `--ci guest` runs a declared list, everything outside it is on demand.