From 09bd611ad5509a81b8006f0c78ee348b452a8087 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Tue, 16 Jun 2026 17:43:58 -0300 Subject: [PATCH 1/7] fix ethrex bench --- .github/workflows/bench-vs-nightly.yml | 10 ++++++++++ Makefile | 23 +++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench-vs-nightly.yml b/.github/workflows/bench-vs-nightly.yml index 5b439b527..3192e5677 100644 --- a/.github/workflows/bench-vs-nightly.yml +++ b/.github/workflows/bench-vs-nightly.yml @@ -48,6 +48,10 @@ jobs: --no-color - name: Run ethrex block benchmarks + id: ethrex_bench + # continue-on-error so the artifact upload, summary, and Slack steps below still + # run even when the ethrex bench fails; the "Fail if ethrex benchmark failed" step + # at the end of the job re-surfaces the failure so the run shows red. continue-on-error: true run: | bash ./bench_vs/run_ethrex.sh \ @@ -70,3 +74,9 @@ jobs: env: SLACK_WEBHOOK: ${{ github.event_name == 'workflow_dispatch' && secrets.BENCH_VS_SLACK_WEBHOOK_TEST || secrets.BENCH_VS_SLACK_WEBHOOK }} run: bash .github/scripts/publish_bench_vs.sh "$SLACK_WEBHOOK" + + - name: Fail if ethrex benchmark failed + if: always() && steps.ethrex_bench.outcome == 'failure' + run: | + echo "::error::ethrex block benchmark step failed — see the 'Run ethrex block benchmarks' step logs" + exit 1 diff --git a/Makefile b/Makefile index fb4782497..3be73348f 100644 --- a/Makefile +++ b/Makefile @@ -73,20 +73,26 @@ prepare-test-data: echo "ethrex_hoodi.bin already exists"; \ fi +# The guard checks for include/stdlib.h (not just the include/ dir) so that a PARTIAL +# sysroot — directories present but missing the C standard library headers — is detected +# as incomplete and re-provisioned, instead of being mistaken for a complete one. When it +# re-provisions, it first removes any existing $(SYSROOT_DIR) and re-extracts from scratch, +# so a partial/stale/corrupt sysroot self-heals without manual intervention on the runner. prepare-sysroot: - @if [ -d "$(SYSROOT_DIR)/include" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ + @if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ echo "Sysroot already exists at $(SYSROOT_DIR)"; \ else \ - echo "Downloading lambda-vm-sysroot-rv64im.tar.gz..."; \ + echo "Provisioning sysroot at $(SYSROOT_DIR) (downloading lambda-vm-sysroot-rv64im.tar.gz)..."; \ curl -L "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)"; \ echo "Extracting sysroot to $(SYSROOT_DIR)..."; \ if mkdir -p "$(SYSROOT_DIR)" 2>/dev/null && [ -w "$(SYSROOT_DIR)" ]; then \ - tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \ + rm -rf "$(SYSROOT_DIR)" && mkdir -p "$(SYSROOT_DIR)" \ + && tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \ || { rm -rf "$(SYSROOT_DIR)" "$(SYSROOT_TARBALL)"; exit 1; }; \ else \ echo "$(SYSROOT_DIR) is not writable; using sudo."; \ echo "Tip: re-run with SYSROOT_DIR=\$$HOME/.lambda-vm-sysroot to avoid sudo."; \ - sudo mkdir -p "$(SYSROOT_DIR)" \ + sudo rm -rf "$(SYSROOT_DIR)" && sudo mkdir -p "$(SYSROOT_DIR)" \ && sudo tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \ || { sudo rm -rf "$(SYSROOT_DIR)"; rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \ fi; \ @@ -110,7 +116,12 @@ compile-programs: compile-programs-asm compile-programs-rust compile-bench # Compile rust (64-bit) -$(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml +# Order-only `| prepare-sysroot` so a direct `make .../foo.elf` provisions the sysroot +# first (the aggregate compile-programs-rust/compile-bench targets already do, but a +# bare pattern-rule invocation like `make -B .../ethrex.elf` would otherwise skip it +# and fail to compile C deps such as c-kzg). Order-only because prepare-sysroot is +# .PHONY — a normal prereq would force a rebuild every time; its recipe is idempotent. +$(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml | prepare-sysroot @mkdir -p $(RUST_ARTIFACTS_DIR) cd $(RUST_PROGRAMS_DIR)/$* && \ CARGO_TARGET_DIR=$(abspath $(SHARED_TARGET_DIR)) \ @@ -123,7 +134,7 @@ $(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml cp $(SHARED_TARGET_DIR)/riscv64im-lambda-vm-elf/release/$* $@ # Compile rust benches (64-bit) -$(BENCH_ARTIFACTS_DIR)/%.elf: $(BENCH_PROGRAMS_DIR)/%/Cargo.toml +$(BENCH_ARTIFACTS_DIR)/%.elf: $(BENCH_PROGRAMS_DIR)/%/Cargo.toml | prepare-sysroot @mkdir -p $(BENCH_ARTIFACTS_DIR) cd $(BENCH_PROGRAMS_DIR)/$* && \ CARGO_TARGET_DIR=$(abspath $(SHARED_TARGET_DIR)) \ From 2d113cfd065f62677353fc8eea7d4a2a8cf5510e Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Wed, 17 Jun 2026 11:49:25 -0300 Subject: [PATCH 2/7] Provision bench sysroot in a user-writable dir --- .github/workflows/bench-vs-nightly.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/bench-vs-nightly.yml b/.github/workflows/bench-vs-nightly.yml index 3192e5677..a53c63c7c 100644 --- a/.github/workflows/bench-vs-nightly.yml +++ b/.github/workflows/bench-vs-nightly.yml @@ -54,6 +54,13 @@ jobs: # at the end of the job re-surfaces the failure so the run shows red. continue-on-error: true run: | + # Provision the RISC-V sysroot in a user-writable dir instead of the default + # /opt/lambda-vm-sysroot, which on the self-hosted bench runner is root-owned + # and was never fully provisioned (missing libc headers → c-kzg failed to find + # stdlib.h). `make` (via SYSROOT_DIR ?=) picks this up and passes it as clang's + # --sysroot, so the guest ELF rebuild self-provisions with no sudo, and the + # extracted sysroot is cached in $HOME across nightly runs. + export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" bash ./bench_vs/run_ethrex.sh \ --report-dir bench_vs_artifacts \ --rebuild-elf \ From 0d6da4a6594f8b67bbf3e731f096be79047a1c12 Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Thu, 18 Jun 2026 10:55:02 -0300 Subject: [PATCH 3/7] Guard SYSROOT_DIR rm -rf and harden provisioning --- Makefile | 9 ++++++++- bench_vs/run_ethrex.sh | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3be73348f..7c0b95610 100644 --- a/Makefile +++ b/Makefile @@ -78,12 +78,19 @@ prepare-test-data: # as incomplete and re-provisioned, instead of being mistaken for a complete one. When it # re-provisions, it first removes any existing $(SYSROOT_DIR) and re-extracts from scratch, # so a partial/stale/corrupt sysroot self-heals without manual intervention on the runner. +# A basename allowlist guards the rm -rf: since SYSROOT_DIR is overrideable, it must end in +# lambda-vm-sysroot or .lambda-vm-sysroot, so an accidental override (e.g. SYSROOT_DIR=/opt) +# can never be wiped — especially via the sudo fallback path. prepare-sysroot: @if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ echo "Sysroot already exists at $(SYSROOT_DIR)"; \ else \ + case "$$(basename "$(SYSROOT_DIR)")" in \ + lambda-vm-sysroot|.lambda-vm-sysroot) : ;; \ + *) echo "prepare-sysroot: refusing to (sudo) rm -rf SYSROOT_DIR=$(SYSROOT_DIR) — expected a path ending in lambda-vm-sysroot or .lambda-vm-sysroot"; exit 1 ;; \ + esac; \ echo "Provisioning sysroot at $(SYSROOT_DIR) (downloading lambda-vm-sysroot-rv64im.tar.gz)..."; \ - curl -L "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)"; \ + curl -fL "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)"; \ echo "Extracting sysroot to $(SYSROOT_DIR)..."; \ if mkdir -p "$(SYSROOT_DIR)" 2>/dev/null && [ -w "$(SYSROOT_DIR)" ]; then \ rm -rf "$(SYSROOT_DIR)" && mkdir -p "$(SYSROOT_DIR)" \ diff --git a/bench_vs/run_ethrex.sh b/bench_vs/run_ethrex.sh index 23b99f9a1..b84900264 100755 --- a/bench_vs/run_ethrex.sh +++ b/bench_vs/run_ethrex.sh @@ -10,7 +10,8 @@ # # Prerequisites: # - Lambda VM CLI build dependencies available -# - Sysroot present at /opt/lambda-vm-sysroot (run `make prepare-sysroot` first) +# - RISC-V sysroot: auto-provisioned by the guest ELF build (the .elf rules depend on +# `make prepare-sysroot`). Override the location with SYSROOT_DIR (default /opt/lambda-vm-sysroot). # - Rust stable + nightly-2026-02-01 installed set -euo pipefail From 69ea2b8be0f6568e2472244bd075dad70308bbac Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Thu, 18 Jun 2026 12:21:25 -0300 Subject: [PATCH 4/7] Clarify sysroot guard comment scope --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7c0b95610..766d3efb9 100644 --- a/Makefile +++ b/Makefile @@ -78,9 +78,10 @@ prepare-test-data: # as incomplete and re-provisioned, instead of being mistaken for a complete one. When it # re-provisions, it first removes any existing $(SYSROOT_DIR) and re-extracts from scratch, # so a partial/stale/corrupt sysroot self-heals without manual intervention on the runner. -# A basename allowlist guards the rm -rf: since SYSROOT_DIR is overrideable, it must end in -# lambda-vm-sysroot or .lambda-vm-sysroot, so an accidental override (e.g. SYSROOT_DIR=/opt) -# can never be wiped — especially via the sudo fallback path. +# A basename allowlist guards the rm -rf: SYSROOT_DIR must end in lambda-vm-sysroot or +# .lambda-vm-sysroot, so an accidental override (e.g. SYSROOT_DIR=/opt) can't be wiped, +# especially via the sudo fallback. This is typo/misconfig prevention, NOT a security +# boundary — a caller that controls SYSROOT_DIR can still point it at any */lambda-vm-sysroot. prepare-sysroot: @if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ echo "Sysroot already exists at $(SYSROOT_DIR)"; \ From 50162a56c6115f36ec3d525c656f4eeae25eadd7 Mon Sep 17 00:00:00 2001 From: Mauro Toscano <12560266+MauroToscano@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:37:01 -0300 Subject: [PATCH 5/7] Merge pull request #677 from yetanotherco/fix/sysroot-download-robustness Robustness + review fixes for nightly bench sysroot --- .github/workflows/bench-vs-nightly.yml | 5 +++-- Makefile | 27 ++++++++++++++++---------- README.md | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bench-vs-nightly.yml b/.github/workflows/bench-vs-nightly.yml index a53c63c7c..a0f4b416b 100644 --- a/.github/workflows/bench-vs-nightly.yml +++ b/.github/workflows/bench-vs-nightly.yml @@ -59,7 +59,8 @@ jobs: # and was never fully provisioned (missing libc headers → c-kzg failed to find # stdlib.h). `make` (via SYSROOT_DIR ?=) picks this up and passes it as clang's # --sysroot, so the guest ELF rebuild self-provisions with no sudo, and the - # extracted sysroot is cached in $HOME across nightly runs. + # extracted sysroot persists in $HOME across runs on the persistent + # self-hosted runner (no actions/cache step is involved). export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" bash ./bench_vs/run_ethrex.sh \ --report-dir bench_vs_artifacts \ @@ -85,5 +86,5 @@ jobs: - name: Fail if ethrex benchmark failed if: always() && steps.ethrex_bench.outcome == 'failure' run: | - echo "::error::ethrex block benchmark step failed — see the 'Run ethrex block benchmarks' step logs" + echo "::error::ethrex block benchmark step failed - see the 'Run ethrex block benchmarks' step logs" exit 1 diff --git a/Makefile b/Makefile index 766d3efb9..6efedc417 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,10 @@ ETHREX_URL := https://lambda.alignedlayer.com/ethrex_hoodi.bin # Override with: make ... SYSROOT_DIR=$HOME/.lambda-vm-sysroot # to install the sysroot in a user-writable location and avoid sudo. SYSROOT_DIR ?= /opt/lambda-vm-sysroot +# Fixed, global path: prepare-sysroot assumes a single writer at a time. The recipe +# `rm -f`s this before downloading, so a stale tarball can't be extracted — but two +# concurrent `make prepare-sysroot` on one host would race on it. The current CI runs +# no concurrent jobs sharing a SYSROOT_DIR; revisit (e.g. mktemp/flock) if that changes. SYSROOT_TARBALL := /tmp/lambda-vm-sysroot-rv64im.tar.gz SYSROOT_URL := https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz # CFLAGS for ckzg / ethrex guest programs: overrides the hardcoded `/opt/lambda-vm-sysroot` @@ -66,9 +70,11 @@ RV64_TARGET_SPEC=$(CURDIR)/executor/programs/riscv64im-lambda-vm-elf.json .PHONY: test prepare-test-data prepare-sysroot prepare-test-data: - @if [ ! -f "$(ETHREX_FILE)" ]; then \ + @set -e; \ + if [ ! -f "$(ETHREX_FILE)" ]; then \ echo "Downloading ethrex_hoodi.bin..."; \ - curl -L "$(ETHREX_URL)" -o "$(ETHREX_FILE)"; \ + curl -fL --proto '=https' "$(ETHREX_URL)" -o "$(ETHREX_FILE)" \ + || { rm -f "$(ETHREX_FILE)"; exit 1; }; \ else \ echo "ethrex_hoodi.bin already exists"; \ fi @@ -83,31 +89,32 @@ prepare-test-data: # especially via the sudo fallback. This is typo/misconfig prevention, NOT a security # boundary — a caller that controls SYSROOT_DIR can still point it at any */lambda-vm-sysroot. prepare-sysroot: - @if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ + @set -e; \ + if [ -f "$(SYSROOT_DIR)/include/stdlib.h" ] && [ -d "$(SYSROOT_DIR)/lib" ]; then \ echo "Sysroot already exists at $(SYSROOT_DIR)"; \ else \ case "$$(basename "$(SYSROOT_DIR)")" in \ lambda-vm-sysroot|.lambda-vm-sysroot) : ;; \ - *) echo "prepare-sysroot: refusing to (sudo) rm -rf SYSROOT_DIR=$(SYSROOT_DIR) — expected a path ending in lambda-vm-sysroot or .lambda-vm-sysroot"; exit 1 ;; \ + *) echo "prepare-sysroot: refusing to (sudo) rm -rf SYSROOT_DIR=$(SYSROOT_DIR) - expected a path ending in lambda-vm-sysroot or .lambda-vm-sysroot"; exit 1 ;; \ esac; \ echo "Provisioning sysroot at $(SYSROOT_DIR) (downloading lambda-vm-sysroot-rv64im.tar.gz)..."; \ - curl -fL "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)"; \ + rm -f "$(SYSROOT_TARBALL)"; \ + curl -fL --proto '=https' "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)" \ + || { rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \ echo "Extracting sysroot to $(SYSROOT_DIR)..."; \ if mkdir -p "$(SYSROOT_DIR)" 2>/dev/null && [ -w "$(SYSROOT_DIR)" ]; then \ rm -rf "$(SYSROOT_DIR)" && mkdir -p "$(SYSROOT_DIR)" \ - && tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \ + && tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ || { rm -rf "$(SYSROOT_DIR)" "$(SYSROOT_TARBALL)"; exit 1; }; \ else \ echo "$(SYSROOT_DIR) is not writable; using sudo."; \ echo "Tip: re-run with SYSROOT_DIR=\$$HOME/.lambda-vm-sysroot to avoid sudo."; \ sudo rm -rf "$(SYSROOT_DIR)" && sudo mkdir -p "$(SYSROOT_DIR)" \ - && sudo tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 \ + && sudo tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ || { sudo rm -rf "$(SYSROOT_DIR)"; rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \ fi; \ - rm "$(SYSROOT_TARBALL)"; \ + rm -f "$(SYSROOT_TARBALL)"; \ fi -# Note: the tarball rm above only runs on success — each error handler -# cleans up the tarball itself before `exit 1`. compile-programs-asm: @mkdir -p $(ASM_ARTIFACTS_DIR) diff --git a/README.md b/README.md index 2e96d7fc0..9c34da4f1 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ Some of the tests require linking with C libraries. The easiest way is to let `make` do it: ```sh +SYSROOT_DIR=$HOME/.lambda-vm-sysroot make prepare-sysroot # recommended: user-writable, no sudo make prepare-sysroot # installs to /opt (uses sudo) -SYSROOT_DIR=$HOME/.lambda-vm-sysroot make prepare-sysroot # user-writable, no sudo ``` Or do it manually: From 10950acfe417543c4456522bbe4fa3d2949c5fcf Mon Sep 17 00:00:00 2001 From: jotabulacios Date: Thu, 18 Jun 2026 15:06:20 -0300 Subject: [PATCH 6/7] Harden sysroot provisioning in provision.sh --- infra/provision.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/infra/provision.sh b/infra/provision.sh index 158c11d35..ec6a3f6dd 100755 --- a/infra/provision.sh +++ b/infra/provision.sh @@ -149,14 +149,21 @@ grep -qxF "$PATH_LINE" "$HOME/.bashrc" 2>/dev/null \ APP_CLAUDE # --- 8. lambda-vm sysroot (rv64im) ------------------------------------------ +# Guard on include/stdlib.h and re-extract from scratch so a partial/interrupted extract +# self-heals on re-run; a bare `[ ! -d ]` guard left a headerless sysroot that broke c-kzg. SYSROOT_DIR=/opt/lambda-vm-sysroot SYSROOT_URL=https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz -if [ ! -d "$SYSROOT_DIR" ]; then - log "downloading sysroot to $SYSROOT_DIR" - curl -L "$SYSROOT_URL" -o /tmp/sysroot.tar.gz +if [ -f "$SYSROOT_DIR/include/stdlib.h" ] && [ -d "$SYSROOT_DIR/lib" ]; then + log "sysroot already present at $SYSROOT_DIR" +else + log "provisioning sysroot at $SYSROOT_DIR" + curl -fL --proto '=https' "$SYSROOT_URL" -o /tmp/sysroot.tar.gz \ + || { rm -f /tmp/sysroot.tar.gz; exit 1; } + rm -rf "$SYSROOT_DIR" mkdir -p /opt - tar -xzf /tmp/sysroot.tar.gz -C /opt - rm /tmp/sysroot.tar.gz + tar -xzf /tmp/sysroot.tar.gz -C /opt --no-same-owner \ + || { rm -rf "$SYSROOT_DIR"; rm -f /tmp/sysroot.tar.gz; exit 1; } + rm -f /tmp/sysroot.tar.gz fi # --- 9. Clone lambda_vm (as app, public repo over HTTPS) --------------------- From 384b3abbee06b5017f6f100ca6381b30d01a0318 Mon Sep 17 00:00:00 2001 From: Mauro Toscano <12560266+MauroToscano@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:57:47 -0300 Subject: [PATCH 7/7] Verify sysroot tarball before extraction (#684) --- .github/workflows/bench-vs-nightly.yml | 4 +-- Makefile | 45 +++++++++++++++++--------- README.md | 1 + infra/provision.sh | 21 ++++++++---- 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/.github/workflows/bench-vs-nightly.yml b/.github/workflows/bench-vs-nightly.yml index a0f4b416b..04d07a5e8 100644 --- a/.github/workflows/bench-vs-nightly.yml +++ b/.github/workflows/bench-vs-nightly.yml @@ -56,8 +56,8 @@ jobs: run: | # Provision the RISC-V sysroot in a user-writable dir instead of the default # /opt/lambda-vm-sysroot, which on the self-hosted bench runner is root-owned - # and was never fully provisioned (missing libc headers → c-kzg failed to find - # stdlib.h). `make` (via SYSROOT_DIR ?=) picks this up and passes it as clang's + # and was never fully provisioned (missing libc headers for guest C dependencies). + # `make` (via SYSROOT_DIR ?=) picks this up and passes it as clang's # --sysroot, so the guest ELF rebuild self-provisions with no sudo, and the # extracted sysroot persists in $HOME across runs on the persistent # self-hosted runner (no actions/cache step is involved). diff --git a/Makefile b/Makefile index c7b4639af..b5e342c54 100644 --- a/Makefile +++ b/Makefile @@ -48,13 +48,9 @@ BENCH_ARTIFACTS := $(addprefix $(BENCH_ARTIFACTS_DIR)/, $(addsuffix .elf, $(BENC # Override with: make ... SYSROOT_DIR=$HOME/.lambda-vm-sysroot # to install the sysroot in a user-writable location and avoid sudo. SYSROOT_DIR ?= /opt/lambda-vm-sysroot -# Fixed, global path: prepare-sysroot assumes a single writer at a time. The recipe -# `rm -f`s this before downloading, so a stale tarball can't be extracted — but two -# concurrent `make prepare-sysroot` on one host would race on it. The current CI runs -# no concurrent jobs sharing a SYSROOT_DIR; revisit (e.g. mktemp/flock) if that changes. -SYSROOT_TARBALL := /tmp/lambda-vm-sysroot-rv64im.tar.gz SYSROOT_URL := https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz -# CFLAGS for ckzg / ethrex guest programs: overrides the hardcoded `/opt/lambda-vm-sysroot` +SYSROOT_SHA256 := 420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7 +# CFLAGS for guest programs with C dependencies: overrides the hardcoded `/opt/lambda-vm-sysroot` # in their .cargo/config.toml so cargo picks up our $(SYSROOT_DIR) instead. # $(abspath ...) because the build rule cd's into the program dir before invoking cargo. SYSROOT_CFLAGS := --target=riscv64 -march=rv64im -mabi=lp64 --sysroot=$(abspath $(SYSROOT_DIR)) @@ -86,23 +82,42 @@ prepare-sysroot: lambda-vm-sysroot|.lambda-vm-sysroot) : ;; \ *) echo "prepare-sysroot: refusing to (sudo) rm -rf SYSROOT_DIR=$(SYSROOT_DIR) - expected a path ending in lambda-vm-sysroot or .lambda-vm-sysroot"; exit 1 ;; \ esac; \ + tmp_dir=""; \ + cleanup() { if [ -n "$$tmp_dir" ]; then rm -rf "$$tmp_dir"; fi; }; \ + trap 'cleanup' EXIT; \ + trap 'cleanup; exit 130' INT; \ + trap 'cleanup; exit 143' TERM; \ + tmp_dir="$$(mktemp -d /tmp/lambda-vm-sysroot.XXXXXX)"; \ + tarball="$$tmp_dir/lambda-vm-sysroot-rv64im.tar.gz"; \ echo "Provisioning sysroot at $(SYSROOT_DIR) (downloading lambda-vm-sysroot-rv64im.tar.gz)..."; \ - rm -f "$(SYSROOT_TARBALL)"; \ - curl -fL --proto '=https' "$(SYSROOT_URL)" -o "$(SYSROOT_TARBALL)" \ - || { rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \ + curl -fL --proto '=https' "$(SYSROOT_URL)" -o "$$tarball"; \ + echo "Verifying sysroot checksum..."; \ + checksum_ok=false; \ + if command -v sha256sum >/dev/null 2>&1; then \ + printf '%s %s\n' "$(SYSROOT_SHA256)" "$$tarball" | sha256sum -c - >/dev/null && checksum_ok=true; \ + elif command -v shasum >/dev/null 2>&1; then \ + actual="$$(shasum -a 256 "$$tarball" | awk '{print $$1}')"; \ + [ "$$actual" = "$(SYSROOT_SHA256)" ] && checksum_ok=true; \ + else \ + echo "prepare-sysroot: missing sha256sum or shasum for checksum verification" >&2; \ + exit 1; \ + fi; \ + if [ "$$checksum_ok" != true ]; then \ + echo "prepare-sysroot: checksum mismatch for $(SYSROOT_URL)" >&2; \ + exit 1; \ + fi; \ echo "Extracting sysroot to $(SYSROOT_DIR)..."; \ if mkdir -p "$(SYSROOT_DIR)" 2>/dev/null && [ -w "$(SYSROOT_DIR)" ]; then \ rm -rf "$(SYSROOT_DIR)" && mkdir -p "$(SYSROOT_DIR)" \ - && tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ - || { rm -rf "$(SYSROOT_DIR)" "$(SYSROOT_TARBALL)"; exit 1; }; \ + && tar -xzf "$$tarball" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ + || { rm -rf "$(SYSROOT_DIR)"; exit 1; }; \ else \ echo "$(SYSROOT_DIR) is not writable; using sudo."; \ echo "Tip: re-run with SYSROOT_DIR=\$$HOME/.lambda-vm-sysroot to avoid sudo."; \ sudo rm -rf "$(SYSROOT_DIR)" && sudo mkdir -p "$(SYSROOT_DIR)" \ - && sudo tar -xzf "$(SYSROOT_TARBALL)" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ - || { sudo rm -rf "$(SYSROOT_DIR)"; rm -f "$(SYSROOT_TARBALL)"; exit 1; }; \ + && sudo tar -xzf "$$tarball" -C "$(SYSROOT_DIR)" --strip-components=1 --no-same-owner \ + || { sudo rm -rf "$(SYSROOT_DIR)"; exit 1; }; \ fi; \ - rm -f "$(SYSROOT_TARBALL)"; \ fi compile-programs-asm: @@ -123,7 +138,7 @@ compile-programs: compile-programs-asm compile-programs-rust compile-bench # Order-only `| prepare-sysroot` so a direct `make .../foo.elf` provisions the sysroot # first (the aggregate compile-programs-rust/compile-bench targets already do, but a # bare pattern-rule invocation like `make -B .../ethrex.elf` would otherwise skip it -# and fail to compile C deps such as c-kzg). Order-only because prepare-sysroot is +# and fail to compile guest C dependencies). Order-only because prepare-sysroot is # .PHONY — a normal prereq would force a rebuild every time; its recipe is idempotent. $(RUST_ARTIFACTS_DIR)/%.elf: $(RUST_PROGRAMS_DIR)/%/Cargo.toml | prepare-sysroot @mkdir -p $(RUST_ARTIFACTS_DIR) diff --git a/README.md b/README.md index 9c34da4f1..151934433 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Or do it manually: ```sh wget https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz +echo "420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7 lambda-vm-sysroot-rv64im.tar.gz" | sha256sum -c - sudo mkdir -p /opt && sudo tar -xzf lambda-vm-sysroot-rv64im.tar.gz -C /opt ``` diff --git a/infra/provision.sh b/infra/provision.sh index beabc06bc..356b91420 100755 --- a/infra/provision.sh +++ b/infra/provision.sh @@ -150,20 +150,29 @@ APP_CLAUDE # --- 8. lambda-vm sysroot (rv64im) ------------------------------------------ # Guard on include/stdlib.h and re-extract from scratch so a partial/interrupted extract -# self-heals on re-run; a bare `[ ! -d ]` guard left a headerless sysroot that broke c-kzg. +# self-heals on re-run; a bare `[ ! -d ]` guard left a headerless sysroot that broke +# guest C dependencies. SYSROOT_DIR=/opt/lambda-vm-sysroot SYSROOT_URL=https://lambda.alignedlayer.com/lambda-vm-sysroot-rv64im.tar.gz +SYSROOT_SHA256=420e394a096f3859235e3a8121a8d5a10f995ac48e636e8d700f17d50803a0e7 if [ -f "$SYSROOT_DIR/include/stdlib.h" ] && [ -d "$SYSROOT_DIR/lib" ]; then log "sysroot already present at $SYSROOT_DIR" else log "provisioning sysroot at $SYSROOT_DIR" - curl -fL --proto '=https' "$SYSROOT_URL" -o /tmp/sysroot.tar.gz \ - || { rm -f /tmp/sysroot.tar.gz; exit 1; } + sysroot_tmp_dir=$(mktemp -d /tmp/lambda-vm-sysroot.XXXXXX) + sysroot_tarball="$sysroot_tmp_dir/lambda-vm-sysroot-rv64im.tar.gz" + cleanup_sysroot_tmp() { rm -rf "$sysroot_tmp_dir"; } + trap cleanup_sysroot_tmp EXIT + + curl -fL --proto '=https' "$SYSROOT_URL" -o "$sysroot_tarball" + printf '%s %s\n' "$SYSROOT_SHA256" "$sysroot_tarball" | sha256sum -c - + rm -rf "$SYSROOT_DIR" mkdir -p /opt - tar -xzf /tmp/sysroot.tar.gz -C /opt --no-same-owner \ - || { rm -rf "$SYSROOT_DIR"; rm -f /tmp/sysroot.tar.gz; exit 1; } - rm -f /tmp/sysroot.tar.gz + tar -xzf "$sysroot_tarball" -C /opt --no-same-owner \ + || { rm -rf "$SYSROOT_DIR"; exit 1; } + rm -rf "$sysroot_tmp_dir" + trap - EXIT fi # --- 9. Clone lambda_vm (as app, public repo over HTTPS) ---------------------