diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 286e4b0..13c7d8a 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -10,7 +10,7 @@ on: description: Instance family to use for the benchmark runner type: string required: false - default: g6e.2xlarge + default: g7e.2xlarge mode: description: Host harness mode type: choice @@ -25,8 +25,9 @@ on: description: Cargo profile for the host harness type: choice required: false - default: release + default: maxperf options: + - maxperf - dev - release iterations: @@ -49,18 +50,18 @@ jobs: benchmark: name: Run Host Benchmark env: - BENCH_HOST_PROFILE: ${{ inputs.host_profile || 'release' }} + BENCH_HOST_PROFILE: ${{ inputs.host_profile || 'maxperf' }} BENCH_ITERATIONS: ${{ inputs.iterations || '' }} BENCH_MODE: ${{ inputs.mode || 'prove-stark' }} runs-on: - runs-on - run-id=${{ github.run_id }} - - family=${{ inputs.instance_family || 'g6e.2xlarge' }} + - family=${{ inputs.instance_family || 'g7e.2xlarge' }} - disk=large - tag=bench-coremark-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} - extras=s3-cache - spot=false - - image=${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') && 'ubuntu24-gpu-x64' || contains(inputs.instance_family || 'g6e.2xlarge', 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }} + - image=${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') && 'ubuntu24-gpu-x64' || contains(inputs.instance_family || 'g7e.2xlarge', 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }} timeout-minutes: 180 steps: @@ -89,8 +90,13 @@ jobs: sudo apt-get update sudo apt-get install -y gnuplot + # TODO: remove once https://github.com/runs-on/runner-images-for-aws/issues/55 is fixed + - name: Setup GPU driver + if: ${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') }} + run: bash ./host/scripts/gpu_driver_setup.sh + - name: Verify GPU setup - if: ${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') }} + if: ${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') }} run: | nvidia-smi nvidia-smi -L diff --git a/README.md b/README.md index 8ffb7a0..10973b5 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ The wrapper script builds the host binary from `host/`, runs it against the guest ELF staged at `host/elf/openvm-coremark`, and enables some host-specific features automatically based on the machine it is running on. -By default, it runs in `prove-stark` mode with the `release` Cargo profile. +By default, it runs in `prove-stark` mode with the `maxperf` Cargo profile. On `x86_64`, it also enables the host `aot` feature. If `nvidia-smi` is available, the script automatically enables CUDA and records GPU memory usage to `gpu_memory_usage.csv`. If no NVIDIA tooling is available, the host harness @@ -196,7 +196,7 @@ still runs without those profiling features. ### `run_coremark.sh` options - `--mode `: choose one of `execute`, `execute-metered`, `prove-app`, or `prove-stark` -- `--profile `: build the host binary with `dev`, `release`, or a custom Cargo profile such as `profiling` +- `--profile `: build the host binary with `dev`, `release`, `maxperf`, or `profiling` - `--cuda`: force CUDA acceleration instead of relying on auto-detection via `nvidia-smi` - `--nsys`: run under NVIDIA Nsight Systems profiling; this implies CUDA and uses `sudo nsys profile` - `--memcheck`: run under `compute-sanitizer --tool memcheck` diff --git a/host/Cargo.toml b/host/Cargo.toml index 4c698e4..0386536 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -31,3 +31,16 @@ jemalloc = ["openvm-sdk/jemalloc"] cuda = ["dep:openvm-cuda-backend", "openvm-sdk/cuda"] unprotected = ["openvm-sdk/unprotected"] nvtx = ["openvm-stark-sdk/nvtx"] + +[profile.release] +lto = "thin" + +[profile.profiling] +inherits = "maxperf" +debug = "full" +strip = "none" + +[profile.maxperf] +inherits = "release" +lto = "fat" +codegen-units = 1 diff --git a/host/scripts/gpu_driver_setup.sh b/host/scripts/gpu_driver_setup.sh new file mode 100755 index 0000000..dcb9b4f --- /dev/null +++ b/host/scripts/gpu_driver_setup.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# Ensures a working NVIDIA driver, switching to the open kernel module flavor +# if needed. Blackwell GPUs require open kernel modules. +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then + nvidia-smi + exit 0 +fi + +KERNEL_RELEASE="$(uname -r)" + +installed_nvidia_packages() { + dpkg-query -W -f='${binary:Package}\n' 2>/dev/null || true +} + +detect_driver_series() { + local package series + + if [[ -n "${NVIDIA_DRIVER_SERIES:-}" ]]; then + echo "$NVIDIA_DRIVER_SERIES" + return + fi + + while IFS= read -r package; do + if [[ "$package" == linux-modules-nvidia-*-open-"$KERNEL_RELEASE" || + "$package" == linux-modules-nvidia-*-"$KERNEL_RELEASE" ]]; then + series="${package#linux-modules-nvidia-}" + series="${series%-"$KERNEL_RELEASE"}" + series="${series%-open}" + if [[ "$series" =~ ^[0-9]+$ ]]; then + echo "$series" + return + fi + fi + done < <(installed_nvidia_packages) + + installed_nvidia_packages | + sed -nE 's/^(nvidia-utils|nvidia-driver|nvidia-dkms|linux-modules-nvidia)-([0-9]+)(-.+)?$/\2/p' | + sort -V | + tail -n1 +} + +DRIVER_SERIES="$(detect_driver_series)" +if ! [[ "$DRIVER_SERIES" =~ ^[0-9]+$ ]]; then + echo "Unable to detect installed NVIDIA driver series; set NVIDIA_DRIVER_SERIES to a numeric series." >&2 + exit 1 +fi + +OPEN_MODULE_PKG="linux-modules-nvidia-${DRIVER_SERIES}-open-${KERNEL_RELEASE}" + +remove_closed_kernel_modules() { + mapfile -t closed_pkgs < <( + installed_nvidia_packages | + awk -v series="$DRIVER_SERIES" ' + $0 ~ "^(nvidia-driver|nvidia-dkms|nvidia-kernel-source)-" series "($|-)" && $0 !~ "-open($|-)" { + print + next + } + $0 ~ "^linux-modules-nvidia-" series "($|-)" && $0 !~ "-open($|-)" { + print + } + ' + ) + + if ((${#closed_pkgs[@]})); then + sudo apt-get remove -y "${closed_pkgs[@]}" + fi +} + +has_apt_candidate() { + local package="$1" + local candidate + candidate="$(apt-cache policy "$package" | awk '/Candidate:/ { print $2; exit }')" + [[ -n "$candidate" && "$candidate" != "(none)" ]] +} + +echo "nvidia-smi not functional; switching to open kernel module driver flavor..." +set -ex +sudo apt-get update -qq +remove_closed_kernel_modules + +# Prefer the prebuilt open modules for the running kernel. Installing the +# nvidia-driver-*-open meta package here pulls DKMS, which conflicts with the +# same .ko files shipped by the prebuilt linux-modules package. +if has_apt_candidate "$OPEN_MODULE_PKG"; then + sudo apt-get install -y --no-install-recommends "$OPEN_MODULE_PKG" +else + sudo apt-get install -y --no-install-recommends \ + "nvidia-driver-${DRIVER_SERIES}-open" \ + "nvidia-dkms-${DRIVER_SERIES}-open" \ + "linux-headers-${KERNEL_RELEASE}" +fi + +if ! command -v nvidia-smi >/dev/null 2>&1; then + sudo apt-get install -y --no-install-recommends "nvidia-utils-${DRIVER_SERIES}" +fi + +sudo rmmod nvidia_uvm nvidia_drm nvidia_modeset nvidia 2>/dev/null || true +sudo modprobe nvidia +sudo modprobe nvidia_uvm +nvidia-smi diff --git a/host/scripts/run_coremark.sh b/host/scripts/run_coremark.sh index 5dfb4b3..4bce2b6 100755 --- a/host/scripts/run_coremark.sh +++ b/host/scripts/run_coremark.sh @@ -5,8 +5,8 @@ # Options: # --mode Set the mode (default: prove-stark) # Valid modes: execute, execute-metered, prove-app, prove-stark -# --profile Set the Cargo build profile (default: release) -# Valid profiles: dev, release, profiling +# --profile Set the Cargo build profile (default: maxperf) +# Valid profiles: dev, release, maxperf, profiling # --cuda Force CUDA acceleration (auto-detected if nvidia-smi available) # --nsys Run with nsys profiling and output summary stats # -- Run with compute-sanitizer --tool (memcheck, synccheck, racecheck) @@ -109,7 +109,7 @@ fi MODE="${MODE_OVERRIDE:-prove-stark}" # Map profile aliases and set target directory -case "${PROFILE_OVERRIDE:-release}" in +case "${PROFILE_OVERRIDE:-maxperf}" in dev|debug) PROFILE="dev" TARGET_DIR="debug" @@ -140,12 +140,15 @@ if [ "$USE_NSYS" = "true" ]; then fi arch=$(uname -m) +TARGET_CPU=$(rustc --print target-cpus | sed -n 's/^ *native.*(currently \([^)]*\)).*/\1/p') +TARGET_CPU="${TARGET_CPU:-native}" +echo "Resolved target CPU: $TARGET_CPU" +RUSTFLAGS="-Ctarget-cpu=$TARGET_CPU" + case $arch in arm64|aarch64) - RUSTFLAGS="-Ctarget-cpu=native" ;; x86_64|amd64) - RUSTFLAGS="-Ctarget-cpu=native" FEATURES="$FEATURES,aot" ;; *)