Fix/nightly bench sysroot

.github/workflows/bench-vs-nightly.yml

@@ -48,8 +48,20 @@ jobs:
             --no-color
 
       - name: Run ethrex block benchmarks
+        id: ethrex_bench
+        # continue-on-error so the artifact upload, summary, and Slack steps below still
+        # run even when the ethrex bench fails; the "Fail if ethrex benchmark failed" step
+        # at the end of the job re-surfaces the failure so the run shows red.
         continue-on-error: true
         run: |
+          # Provision the RISC-V sysroot in a user-writable dir instead of the default
+          # /opt/lambda-vm-sysroot, which on the self-hosted bench runner is root-owned
+          # and was never fully provisioned (missing libc headers for guest C dependencies).
+          # `make` (via SYSROOT_DIR ?=) picks this up and passes it as clang's
+          # --sysroot, so the guest ELF rebuild self-provisions with no sudo, and the
+          # extracted sysroot persists in $HOME across runs on the persistent
+          # self-hosted runner (no actions/cache step is involved).
+          export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
           bash ./bench_vs/run_ethrex.sh \
             --report-dir bench_vs_artifacts \
             --rebuild-elf \
@@ -70,3 +82,9 @@ jobs:
         env:
           SLACK_WEBHOOK: ${{ github.event_name == 'workflow_dispatch' && secrets.BENCH_VS_SLACK_WEBHOOK_TEST || secrets.BENCH_VS_SLACK_WEBHOOK }}
         run: bash .github/scripts/publish_bench_vs.sh "$SLACK_WEBHOOK"
+
+      - name: Fail if ethrex benchmark failed
+        if: always() && steps.ethrex_bench.outcome == 'failure'
+        run: |
+          echo "::error::ethrex block benchmark step failed - see the 'Run ethrex block benchmarks' step logs"
+          exit 1

Makefile

@@ -48,9 +48,9 @@ BENCH_ARTIFACTS := $(addprefix $(BENCH_ARTIFACTS_DIR)/, $(addsuffix .elf, $(BENC
 # Override with: make ... SYSROOT_DIR=$HOME/.lambda-vm-sysroot
 # to install the sysroot in a user-writable location and avoid sudo.
 SYSROOT_DIR ?= /opt/lambda-vm-sysroot
-SYSROOT_TARBALL := /tmp/lambda-vm-sysroot-rv64im.tar.gz
 SYSROOT_URL := https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz
-# CFLAGS for ckzg / ethrex guest programs: overrides the hardcoded `/opt/lambda-vm-sysroot`
+SYSROOT_SHA256 := 420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7
+# CFLAGS for guest programs with C dependencies: overrides the hardcoded `/opt/lambda-vm-sysroot`
 # in their .cargo/config.toml so cargo picks up our $(SYSROOT_DIR) instead.
 # $(abspath ...) because the build rule cd's into the program dir before invoking cargo.
 SYSROOT_CFLAGS := --target=riscv64 -march=rv64im -mabi=lp64 --sysroot=$(abspath $(SYSROOT_DIR))
@@ -64,27 +64,61 @@ RV64_TARGET_SPEC=$(CURDIR)/executor/programs/riscv64im-lambda-vm-elf.json
 
 .PHONY: test prepare-sysroot
 
+# The guard checks for include/stdlib.h (not just the include/ dir) so that a PARTIAL
+# sysroot — directories present but missing the C standard library headers — is detected
+# as incomplete and re-provisioned, instead of being mistaken for a complete one. When it
+# re-provisions, it first removes any existing $(SYSROOT_DIR) and re-extracts from scratch,
+# so a partial/stale/corrupt sysroot self-heals without manual intervention on the runner.
+# A basename allowlist guards the rm -rf: SYSROOT_DIR must end in lambda-vm-sysroot or
+# .lambda-vm-sysroot, so an accidental override (e.g. SYSROOT_DIR=/opt) can't be wiped,
+# especially via the sudo fallback. This is typo/misconfig prevention, NOT a security
+# boundary — a caller that controls SYSROOT_DIR can still point it at any */lambda-vm-sysroot.
 prepare-sysroot:
-	@if [ -d "$(SYSROOT_DIR)/include" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \
+	@set -e; \
+	if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \
 		echo "Sysroot already exists at $(SYSROOT_DIR)"; \
 	else \
-		echo "Downloading lambda-vm-sysroot-rv64im.tar.gz..."; \
-		curl -L "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)"; \
+		case "$$(basename "$(SYSROOT_DIR)")" in \
+			lambda-vm-sysroot|.lambda-vm-sysroot) : ;; \
+			*) echo "prepare-sysroot: refusing to (sudo) rm -rf SYSROOT_DIR=$(SYSROOT_DIR) - expected a path ending in lambda-vm-sysroot or .lambda-vm-sysroot"; exit 1 ;; \
+		esac; \
+		tmp_dir=""; \
+		cleanup() { if [ -n "$$tmp_dir" ]; then rm -rf "$$tmp_dir"; fi; }; \
+		trap 'cleanup' EXIT; \
+		trap 'cleanup; exit 130' INT; \
+		trap 'cleanup; exit 143' TERM; \
+		tmp_dir="$$(mktemp -d /tmp/lambda-vm-sysroot.XXXXXX)"; \
+		tarball="$$tmp_dir/lambda-vm-sysroot-rv64im.tar.gz"; \
+		echo "Provisioning sysroot at $(SYSROOT_DIR) (downloading lambda-vm-sysroot-rv64im.tar.gz)..."; \
+		curl -fL --proto '=https' "$(SYSROOT_URL)" -o "$$tarball"; \
+		echo "Verifying sysroot checksum..."; \
+		checksum_ok=false; \
+		if command -v sha256sum >/dev/null 2>&1; then \
+			printf '%s  %s\n' "$(SYSROOT_SHA256)" "$$tarball" | sha256sum -c - >/dev/null && checksum_ok=true; \
+		elif command -v shasum >/dev/null 2>&1; then \
+			actual="$$(shasum -a 256 "$$tarball" | awk '{print $$1}')"; \
+			[ "$$actual" = "$(SYSROOT_SHA256)" ] && checksum_ok=true; \
+		else \
+			echo "prepare-sysroot: missing sha256sum or shasum for checksum verification" >&2; \
+			exit 1; \
+		fi; \
+		if [ "$$checksum_ok" != true ]; then \
+			echo "prepare-sysroot: checksum mismatch for $(SYSROOT_URL)" >&2; \
+			exit 1; \
+		fi; \
 		echo "Extracting sysroot to $(SYSROOT_DIR)..."; \
 		if mkdir -p "$(SYSROOT_DIR)" 2>/dev/null && [ -w "$(SYSROOT_DIR)" ]; then \
-			tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \
-				|| { rm -rf "$(SYSROOT_DIR)" "$(SYSROOT_TARBALL)"; exit 1; }; \
+			rm -rf "$(SYSROOT_DIR)" && mkdir -p "$(SYSROOT_DIR)" \
+				&& tar -xzf "$$tarball" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \
+				|| { rm -rf "$(SYSROOT_DIR)"; exit 1; }; \
 		else \
 			echo "$(SYSROOT_DIR) is not writable; using sudo."; \
 			echo "Tip: re-run with SYSROOT_DIR=\$$HOME/.lambda-vm-sysroot to avoid sudo."; \
-			sudo mkdir -p "$(SYSROOT_DIR)" \
-				&& sudo tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \
-				|| { sudo rm -rf "$(SYSROOT_DIR)"; rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \
+			sudo rm -rf "$(SYSROOT_DIR)" && sudo mkdir -p "$(SYSROOT_DIR)" \
+				&& sudo tar -xzf "$$tarball" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \
+				|| { sudo rm -rf "$(SYSROOT_DIR)"; exit 1; }; \
 		fi; \
-		rm "$(SYSROOT_TARBALL)"; \
 	fi
-# Note: the tarball rm above only runs on success — each error handler
-# cleans up the tarball itself before `exit 1`.
 
 compile-programs-asm:
 	@mkdir -p $(ASM_ARTIFACTS_DIR)
@@ -101,7 +135,12 @@ compile-programs: compile-programs-asm compile-programs-rust compile-bench
 
 
 # Compile rust (64-bit)
-$(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml
+# Order-only `| prepare-sysroot` so a direct `make .../foo.elf` provisions the sysroot
+# first (the aggregate compile-programs-rust/compile-bench targets already do, but a
+# bare pattern-rule invocation like `make -B .../ethrex.elf` would otherwise skip it
+# and fail to compile guest C dependencies). Order-only because prepare-sysroot is
+# .PHONY — a normal prereq would force a rebuild every time; its recipe is idempotent.
+$(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml | prepare-sysroot
 	@mkdir -p $(RUST_ARTIFACTS_DIR)
 	cd $(RUST_PROGRAMS_DIR)/$* && \
 		CARGO_TARGET_DIR=$(abspath $(SHARED_TARGET_DIR)) \
@@ -114,7 +153,7 @@ $(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml
 	cp $(SHARED_TARGET_DIR)/riscv64im-lambda-vm-elf/release/$* $@
 
 # Compile rust benches (64-bit)
-$(BENCH_ARTIFACTS_DIR)/%.elf: $(BENCH_PROGRAMS_DIR)/%/Cargo.toml
+$(BENCH_ARTIFACTS_DIR)/%.elf: $(BENCH_PROGRAMS_DIR)/%/Cargo.toml | prepare-sysroot
 	@mkdir -p $(BENCH_ARTIFACTS_DIR)
 	cd $(BENCH_PROGRAMS_DIR)/$* && \
 		CARGO_TARGET_DIR=$(abspath $(SHARED_TARGET_DIR)) \

README.md

@@ -50,14 +50,15 @@ Some of the tests require linking with C libraries.
 The easiest way is to let `make` do it:
 
 ```sh
+SYSROOT_DIR=$HOME/.lambda-vm-sysroot make prepare-sysroot  # recommended: user-writable, no sudo
 make prepare-sysroot                                       # installs to /opt (uses sudo)
-SYSROOT_DIR=$HOME/.lambda-vm-sysroot make prepare-sysroot  # user-writable, no sudo
 ```
 
 Or do it manually:
 
 ```sh
 wget https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz
+echo "420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7  lambda-vm-sysroot-rv64im.tar.gz" | sha256sum -c -
 sudo mkdir -p /opt && sudo tar -xzf lambda-vm-sysroot-rv64im.tar.gz -C /opt
 ```
 

bench_vs/run_ethrex.sh

@@ -10,7 +10,8 @@
 #
 # Prerequisites:
 #   - Lambda VM CLI build dependencies available
-#   - Sysroot present at /opt/lambda-vm-sysroot (run `make prepare-sysroot` first)
+#   - RISC-V sysroot: auto-provisioned by the guest ELF build (the .elf rules depend on
+#     `make prepare-sysroot`). Override the location with SYSROOT_DIR (default /opt/lambda-vm-sysroot).
 #   - Rust stable + nightly-2026-02-01 installed
 
 set -euo pipefail

infra/provision.sh

@@ -149,14 +149,30 @@ grep -qxF "$PATH_LINE" "$HOME/.bashrc" 2>/dev/null \
 APP_CLAUDE
 
 # --- 8. lambda-vm sysroot (rv64im) ------------------------------------------
+# Guard on include/stdlib.h and re-extract from scratch so a partial/interrupted extract
+# self-heals on re-run; a bare `[ ! -d ]` guard left a headerless sysroot that broke
+# guest C dependencies.
 SYSROOT_DIR=/opt/lambda-vm-sysroot
 SYSROOT_URL=https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz
-if [ ! -d "$SYSROOT_DIR" ]; then
-    log "downloading sysroot to $SYSROOT_DIR"
-    curl -L "$SYSROOT_URL" -o /tmp/sysroot.tar.gz
+SYSROOT_SHA256=420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7
+if [ -f "$SYSROOT_DIR/include/stdlib.h" ] && [ -d "$SYSROOT_DIR/lib" ]; then
+    log "sysroot already present at $SYSROOT_DIR"
+else
+    log "provisioning sysroot at $SYSROOT_DIR"
+    sysroot_tmp_dir=$(mktemp -d /tmp/lambda-vm-sysroot.XXXXXX)
+    sysroot_tarball="$sysroot_tmp_dir/lambda-vm-sysroot-rv64im.tar.gz"
+    cleanup_sysroot_tmp() { rm -rf "$sysroot_tmp_dir"; }
+    trap cleanup_sysroot_tmp EXIT
+
+    curl -fL --proto '=https' "$SYSROOT_URL" -o "$sysroot_tarball"
+    printf '%s  %s\n' "$SYSROOT_SHA256" "$sysroot_tarball" | sha256sum -c -
+
+    rm -rf "$SYSROOT_DIR"
     mkdir -p /opt
-    tar -xzf /tmp/sysroot.tar.gz -C /opt
-    rm /tmp/sysroot.tar.gz
+    tar -xzf "$sysroot_tarball" -C /opt --no-same-owner \
+        || { rm -rf "$SYSROOT_DIR"; exit 1; }
+    rm -rf "$sysroot_tmp_dir"
+    trap - EXIT
 fi
 
 # --- 9. Clone lambda_vm (as app, public repo over HTTPS) ---------------------