Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions .github/workflows/run-benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
description: Instance family to use for the benchmark runner
type: string
required: false
default: g6e.2xlarge
default: g7e.2xlarge
mode:
description: Host harness mode
type: choice
Expand All @@ -25,8 +25,9 @@ on:
description: Cargo profile for the host harness
type: choice
required: false
default: release
default: maxperf
options:
- maxperf
- dev
- release
iterations:
Expand All @@ -49,18 +50,18 @@ jobs:
benchmark:
name: Run Host Benchmark
env:
BENCH_HOST_PROFILE: ${{ inputs.host_profile || 'release' }}
BENCH_HOST_PROFILE: ${{ inputs.host_profile || 'maxperf' }}
BENCH_ITERATIONS: ${{ inputs.iterations || '' }}
BENCH_MODE: ${{ inputs.mode || 'prove-stark' }}
runs-on:
- runs-on
- run-id=${{ github.run_id }}
- family=${{ inputs.instance_family || 'g6e.2xlarge' }}
- family=${{ inputs.instance_family || 'g7e.2xlarge' }}
- disk=large
- tag=bench-coremark-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
- extras=s3-cache
- spot=false
- image=${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') && 'ubuntu24-gpu-x64' || contains(inputs.instance_family || 'g6e.2xlarge', 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }}
- image=${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') && 'ubuntu24-gpu-x64' || contains(inputs.instance_family || 'g7e.2xlarge', 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }}
timeout-minutes: 180

steps:
Expand Down Expand Up @@ -89,8 +90,13 @@ jobs:
sudo apt-get update
sudo apt-get install -y gnuplot

# TODO: remove once https://github.com/runs-on/runner-images-for-aws/issues/55 is fixed
- name: Setup GPU driver
if: ${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') }}
run: bash ./host/scripts/gpu_driver_setup.sh

- name: Verify GPU setup
if: ${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') }}
if: ${{ startsWith(inputs.instance_family || 'g7e.2xlarge', 'g') }}
run: |
nvidia-smi
nvidia-smi -L
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ The wrapper script builds the host binary from `host/`, runs it against the
guest ELF staged at `host/elf/openvm-coremark`, and enables some host-specific
features automatically based on the machine it is running on.

By default, it runs in `prove-stark` mode with the `release` Cargo profile.
By default, it runs in `prove-stark` mode with the `maxperf` Cargo profile.
On `x86_64`, it also enables the host `aot` feature. If `nvidia-smi` is
available, the script automatically enables CUDA and records GPU memory usage to
`gpu_memory_usage.csv`. If no NVIDIA tooling is available, the host harness
Expand All @@ -196,7 +196,7 @@ still runs without those profiling features.
### `run_coremark.sh` options

- `--mode <MODE>`: choose one of `execute`, `execute-metered`, `prove-app`, or `prove-stark`
- `--profile <PROFILE>`: build the host binary with `dev`, `release`, or a custom Cargo profile such as `profiling`
- `--profile <PROFILE>`: build the host binary with `dev`, `release`, `maxperf`, or `profiling`
- `--cuda`: force CUDA acceleration instead of relying on auto-detection via `nvidia-smi`
- `--nsys`: run under NVIDIA Nsight Systems profiling; this implies CUDA and uses `sudo nsys profile`
- `--memcheck`: run under `compute-sanitizer --tool memcheck`
Expand Down
13 changes: 13 additions & 0 deletions host/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,16 @@ jemalloc = ["openvm-sdk/jemalloc"]
cuda = ["dep:openvm-cuda-backend", "openvm-sdk/cuda"]
unprotected = ["openvm-sdk/unprotected"]
nvtx = ["openvm-stark-sdk/nvtx"]

[profile.release]
lto = "thin"

[profile.profiling]
inherits = "maxperf"
debug = "full"
strip = "none"

[profile.maxperf]
inherits = "release"
lto = "fat"
codegen-units = 1
104 changes: 104 additions & 0 deletions host/scripts/gpu_driver_setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#!/usr/bin/env bash
#
# Ensures a working NVIDIA driver, switching to the open kernel module flavor
# if needed. Blackwell GPUs require open kernel modules.
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive

if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
nvidia-smi
exit 0
fi

KERNEL_RELEASE="$(uname -r)"

installed_nvidia_packages() {
dpkg-query -W -f='${binary:Package}\n' 2>/dev/null || true
}

detect_driver_series() {
local package series

if [[ -n "${NVIDIA_DRIVER_SERIES:-}" ]]; then
echo "$NVIDIA_DRIVER_SERIES"
return
fi

while IFS= read -r package; do
if [[ "$package" == linux-modules-nvidia-*-open-"$KERNEL_RELEASE" ||
"$package" == linux-modules-nvidia-*-"$KERNEL_RELEASE" ]]; then
series="${package#linux-modules-nvidia-}"
series="${series%-"$KERNEL_RELEASE"}"
series="${series%-open}"
if [[ "$series" =~ ^[0-9]+$ ]]; then
echo "$series"
return
fi
fi
done < <(installed_nvidia_packages)

installed_nvidia_packages |
sed -nE 's/^(nvidia-utils|nvidia-driver|nvidia-dkms|linux-modules-nvidia)-([0-9]+)(-.+)?$/\2/p' |
sort -V |
tail -n1
}

DRIVER_SERIES="$(detect_driver_series)"
if ! [[ "$DRIVER_SERIES" =~ ^[0-9]+$ ]]; then
echo "Unable to detect installed NVIDIA driver series; set NVIDIA_DRIVER_SERIES to a numeric series." >&2
exit 1
fi

OPEN_MODULE_PKG="linux-modules-nvidia-${DRIVER_SERIES}-open-${KERNEL_RELEASE}"

remove_closed_kernel_modules() {
mapfile -t closed_pkgs < <(
installed_nvidia_packages |
awk -v series="$DRIVER_SERIES" '
$0 ~ "^(nvidia-driver|nvidia-dkms|nvidia-kernel-source)-" series "($|-)" && $0 !~ "-open($|-)" {
print
next
}
$0 ~ "^linux-modules-nvidia-" series "($|-)" && $0 !~ "-open($|-)" {
print
}
'
)

if ((${#closed_pkgs[@]})); then
sudo apt-get remove -y "${closed_pkgs[@]}"
fi
}

has_apt_candidate() {
local package="$1"
local candidate
candidate="$(apt-cache policy "$package" | awk '/Candidate:/ { print $2; exit }')"
[[ -n "$candidate" && "$candidate" != "(none)" ]]
}

echo "nvidia-smi not functional; switching to open kernel module driver flavor..."
set -ex
sudo apt-get update -qq
remove_closed_kernel_modules

# Prefer the prebuilt open modules for the running kernel. Installing the
# nvidia-driver-*-open meta package here pulls DKMS, which conflicts with the
# same .ko files shipped by the prebuilt linux-modules package.
if has_apt_candidate "$OPEN_MODULE_PKG"; then
sudo apt-get install -y --no-install-recommends "$OPEN_MODULE_PKG"
else
sudo apt-get install -y --no-install-recommends \
"nvidia-driver-${DRIVER_SERIES}-open" \
"nvidia-dkms-${DRIVER_SERIES}-open" \
"linux-headers-${KERNEL_RELEASE}"
fi

if ! command -v nvidia-smi >/dev/null 2>&1; then
sudo apt-get install -y --no-install-recommends "nvidia-utils-${DRIVER_SERIES}"
fi

sudo rmmod nvidia_uvm nvidia_drm nvidia_modeset nvidia 2>/dev/null || true
sudo modprobe nvidia
sudo modprobe nvidia_uvm
nvidia-smi
13 changes: 8 additions & 5 deletions host/scripts/run_coremark.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
# Options:
# --mode <MODE> Set the mode (default: prove-stark)
# Valid modes: execute, execute-metered, prove-app, prove-stark
# --profile <PROFILE> Set the Cargo build profile (default: release)
# Valid profiles: dev, release, profiling
# --profile <PROFILE> Set the Cargo build profile (default: maxperf)
# Valid profiles: dev, release, maxperf, profiling
# --cuda Force CUDA acceleration (auto-detected if nvidia-smi available)
# --nsys Run with nsys profiling and output summary stats
# --<tool> Run with compute-sanitizer --tool <tool> (memcheck, synccheck, racecheck)
Expand Down Expand Up @@ -109,7 +109,7 @@ fi
MODE="${MODE_OVERRIDE:-prove-stark}"

# Map profile aliases and set target directory
case "${PROFILE_OVERRIDE:-release}" in
case "${PROFILE_OVERRIDE:-maxperf}" in
dev|debug)
PROFILE="dev"
TARGET_DIR="debug"
Expand Down Expand Up @@ -140,12 +140,15 @@ if [ "$USE_NSYS" = "true" ]; then
fi

arch=$(uname -m)
TARGET_CPU=$(rustc --print target-cpus | sed -n 's/^ *native.*(currently \([^)]*\)).*/\1/p')
TARGET_CPU="${TARGET_CPU:-native}"
echo "Resolved target CPU: $TARGET_CPU"
RUSTFLAGS="-Ctarget-cpu=$TARGET_CPU"

case $arch in
arm64|aarch64)
RUSTFLAGS="-Ctarget-cpu=native"
;;
x86_64|amd64)
RUSTFLAGS="-Ctarget-cpu=native"
FEATURES="$FEATURES,aot"
;;
*)
Expand Down
Loading