From 01f818b982e62cb99456f8c88307432c7f5918bf Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 1 Oct 2026 09:52:13 -0700 Subject: [PATCH 1/4] ci: add runner benchmark workflow Compares ubuntu-latest with Blacksmith 4 and 8 vCPU runners on identical work: a cold go build, go vet, go test -race and fixed-iteration benchmarks, five attempts each on the same commit with a fresh build cache. The report job publishes medians, speedups and runner queue times in the job summary, plus a benchstat comparison. Runs on workflow_dispatch or a push to ci/runner-benchmark, so it stays out of regular CI. Claude-Session: https://claude.ai/code/session_01QKDYQr53zNKkR7nif2CAAq --- .github/actionlint.yaml | 1 + .github/workflows/runner-benchmark.yml | 202 +++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 .github/workflows/runner-benchmark.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 25298b7bb..8cb2d49f5 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,7 @@ self-hosted-runner: # Blacksmith runners (https://blacksmith.sh) labels: + - blacksmith-4vcpu-ubuntu-2404 - blacksmith-8vcpu-ubuntu-2404 - blacksmith-12vcpu-macos-latest - blacksmith-8vcpu-windows-2025 diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml new file mode 100644 index 000000000..135036434 --- /dev/null +++ b/.github/workflows/runner-benchmark.yml @@ -0,0 +1,202 @@ +name: Runner Benchmark + +# Compares CI runner performance on identical work. Not part of regular CI: +# run it manually, or push to the ci/runner-benchmark branch. +on: + workflow_dispatch: + push: + branches: + - ci/runner-benchmark + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + GO_VERSION: "1.27" + + # https://github.com/actions/checkout/commit/df4cb1c069e1874edd31b4311f1884172cec0e10 + CHECKOUT_ACTION: &checkout_action actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + # https://github.com/actions/setup-go/commit/924ae3a1cded613372ab5595356fb5720e22ba16 + SETUP_GO_ACTION: &setup_go_action actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + + # https://github.com/actions/upload-artifact/commit/043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + UPLOAD_ARTIFACT_ACTION: &upload_artifact_action actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + + # https://github.com/actions/download-artifact/commit/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + DOWNLOAD_ARTIFACT_ACTION: &download_artifact_action actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + +jobs: + measure: + strategy: + fail-fast: false + matrix: + # ubuntu-latest and blacksmith-4vcpu both have 4 vCPUs (like for like); + # blacksmith-8vcpu is the tier Echo's regular CI uses. + runner: [ubuntu-latest, blacksmith-4vcpu-ubuntu-2404, blacksmith-8vcpu-ubuntu-2404] + attempt: [1, 2, 3, 4, 5] + name: ${{ matrix.runner }} (${{ matrix.attempt }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + steps: + - name: Checkout Code + uses: *checkout_action + + - name: Set up Go + uses: *setup_go_action + with: + go-version: ${{ env.GO_VERSION }} + cache: false # every runner starts cold + + - name: Measure + env: + RUNNER_LABEL: ${{ matrix.runner }} + ATTEMPT: ${{ matrix.attempt }} + run: | + set -euo pipefail + out="$RUNNER_TEMP/result" + mkdir -p "$out" + + # A fresh build cache on every runner, so cached builds don't skew the timings. + export GOCACHE="$RUNNER_TEMP/gocache" + go mod download + + ms() { date +%s%3N; } + timings="$out/timings.txt" + measure() { + local name=$1; shift + local start; start=$(ms) + "$@" + echo "${name}_ms=$(( $(ms) - start ))" >> "$timings" + } + + measure build go build ./... + measure vet go vet ./... + measure test_race go test -race -count=1 ./... > /dev/null + # Fixed iteration counts, so faster CPUs finish sooner instead of running more iterations. + measure bench go test -run='^$' -bench=. -benchtime=2000x -count=1 ./... > "$out/bench.txt" + + { + echo "runner=$RUNNER_LABEL" + echo "attempt=$ATTEMPT" + echo "nproc=$(nproc)" + echo "cpu=$(lscpu | sed -n 's/^Model name:[[:space:]]*//p' | head -1)" + echo "mem_gb=$(free -g | awk '/^Mem:/ {print $2}')" + } >> "$timings" + cat "$timings" + + - name: Upload Result + uses: *upload_artifact_action + with: + name: result-${{ matrix.runner }}-${{ matrix.attempt }} + path: ${{ runner.temp }}/result + + report: + needs: measure + if: always() + name: Report + runs-on: ubuntu-latest + permissions: + actions: read # to read job queue times + steps: + - name: Download Results + uses: *download_artifact_action + with: + pattern: result-* + path: results + + - name: Set up Go + uses: *setup_go_action + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Fetch Queue Times + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api --paginate "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \ + --jq '.jobs[] | select(.name != "Report") | [.name, .created_at, .started_at] | @tsv' > queue.tsv + + - name: Summarize + run: | + python3 - <<'EOF' >> "$GITHUB_STEP_SUMMARY" + import glob, os, statistics + from datetime import datetime + + def parse(path): + d = {} + for line in open(path): + if "=" in line: + k, v = line.rstrip("\n").split("=", 1) + d[k] = v + return d + + runs = [parse(p) for p in glob.glob("results/*/timings.txt")] + order = ["ubuntu-latest", "blacksmith-4vcpu-ubuntu-2404", "blacksmith-8vcpu-ubuntu-2404"] + by_runner = {r: [x for x in runs if x.get("runner") == r] for r in order} + + queue = {r: [] for r in order} + for line in open("queue.tsv"): + name, created, started = line.rstrip("\n").split("\t") + runner = name.split(" (")[0] + if runner in queue and started: + t = lambda s: datetime.fromisoformat(s.replace("Z", "+00:00")) + queue[runner].append((t(started) - t(created)).total_seconds()) + + def med(values): + return statistics.median(values) if values else None + + metrics = [("build_ms", "Cold `go build ./...`"), ("vet_ms", "`go vet ./...`"), + ("test_race_ms", "`go test -race ./...`"), ("bench_ms", "Benchmarks (2000 iterations each)")] + + print("## Runner benchmark\n") + print("Medians of %s runs per runner, same commit, cold build cache.\n" % max(len(v) for v in by_runner.values())) + print("| | " + " | ".join(order) + " |") + print("|---|" + "---|" * len(order)) + for key, label in metrics: + cells = [] + base = med([int(x[key]) for x in by_runner[order[0]] if key in x]) + for r in order: + m = med([int(x[key]) for x in by_runner[r] if key in x]) + if m is None: + cells.append("n/a") + elif base and r != order[0]: + cells.append("%.1f s (%.2fx)" % (m / 1000, base / m)) + else: + cells.append("%.1f s" % (m / 1000)) + print("| %s | %s |" % (label, " | ".join(cells))) + print("| Queue time (wait for runner) | %s |" % " | ".join( + "%.0f s" % med(queue[r]) if queue[r] else "n/a" for r in order)) + print("| vCPUs / RAM | %s |" % " | ".join( + ("%s / %s GB" % (by_runner[r][0].get("nproc"), by_runner[r][0].get("mem_gb"))) if by_runner[r] else "n/a" + for r in order)) + print("| CPU | %s |" % " | ".join(by_runner[r][0].get("cpu", "?") if by_runner[r] else "n/a" for r in order)) + print("\nSpeedup is relative to ubuntu-latest (higher is faster).\n") + + os.makedirs("merged", exist_ok=True) + for r in order: + with open("merged/%s.txt" % r, "w") as f: + for p in sorted(glob.glob("results/result-%s-*/bench.txt" % r)): + f.write(open(p).read()) + EOF + + "$(go env GOPATH)/bin/benchstat" \ + ubuntu-latest=merged/ubuntu-latest.txt \ + blacksmith-4vcpu=merged/blacksmith-4vcpu-ubuntu-2404.txt \ + blacksmith-8vcpu=merged/blacksmith-8vcpu-ubuntu-2404.txt > merged/benchstat.txt || true + { + echo "### benchstat geomean (sec/op across runners; full output in the report artifact)" + echo '```' + grep -E 'sec/op|geomean' merged/benchstat.txt || echo "benchstat produced no comparison" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Report + uses: *upload_artifact_action + with: + name: runner-benchmark-report + path: merged From 8a735736d6ca69853dc867c6f8a5439ecb714f86 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 1 Oct 2026 09:57:03 -0700 Subject: [PATCH 2/4] ci: measure cold and default-setup runner timings separately Blacksmith enables a transparent Go build cache through GOCACHEPROG, which bypasses GOCACHE, so the first run's "cold" builds were warm on Blacksmith. Cold timings now unset GOCACHEPROG and use an empty GOCACHE; separate "default setup" timings keep each runner as it comes. The report also normalizes benchmark names and CPU lines so benchstat compares the three runners in one table, and runs the benchmarks twice per attempt (10 samples per runner) so benchstat can report confidence intervals. Claude-Session: https://claude.ai/code/session_01QKDYQr53zNKkR7nif2CAAq --- .github/workflows/runner-benchmark.yml | 40 ++++++++++++++++++-------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index 135036434..2e4dd63bd 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -57,8 +57,6 @@ jobs: out="$RUNNER_TEMP/result" mkdir -p "$out" - # A fresh build cache on every runner, so cached builds don't skew the timings. - export GOCACHE="$RUNNER_TEMP/gocache" go mod download ms() { date +%s%3N; } @@ -70,15 +68,23 @@ jobs: echo "${name}_ms=$(( $(ms) - start ))" >> "$timings" } - measure build go build ./... - measure vet go vet ./... - measure test_race go test -race -count=1 ./... > /dev/null + # Cold: no build cache at all, so the timings compare CPUs. Blacksmith sets + # GOCACHEPROG for its transparent Go build cache, which would bypass GOCACHE. + cold() { env -u GOCACHEPROG GOCACHE="$RUNNER_TEMP/gocache-cold" "$@"; } + measure build_cold cold go build ./... + measure vet_cold cold go vet ./... + measure test_race_cold cold go test -race -count=1 ./... > /dev/null # Fixed iteration counts, so faster CPUs finish sooner instead of running more iterations. - measure bench go test -run='^$' -bench=. -benchtime=2000x -count=1 ./... > "$out/bench.txt" + measure bench cold go test -run='^$' -bench=. -benchtime=2000x -count=2 ./... > "$out/bench.txt" + + # Default: each runner's out-of-the-box setup (GitHub: empty cache; Blacksmith: transparent cache). + measure build_default go build ./... + measure test_race_default go test -race -count=1 ./... > /dev/null { echo "runner=$RUNNER_LABEL" echo "attempt=$ATTEMPT" + echo "gocacheprog=${GOCACHEPROG:+yes}" echo "nproc=$(nproc)" echo "cpu=$(lscpu | sed -n 's/^Model name:[[:space:]]*//p' | head -1)" echo "mem_gb=$(free -g | awk '/^Mem:/ {print $2}')" @@ -150,11 +156,17 @@ jobs: def med(values): return statistics.median(values) if values else None - metrics = [("build_ms", "Cold `go build ./...`"), ("vet_ms", "`go vet ./...`"), - ("test_race_ms", "`go test -race ./...`"), ("bench_ms", "Benchmarks (2000 iterations each)")] + metrics = [("build_cold_ms", "Cold `go build ./...`"), ("vet_cold_ms", "Cold `go vet ./...`"), + ("test_race_cold_ms", "Cold `go test -race ./...`"), + ("bench_ms", "Benchmarks, cold (2000 iterations, 2 runs)"), + ("build_default_ms", "Default setup: `go build ./...`"), + ("test_race_default_ms", "Default setup: `go test -race ./...`")] print("## Runner benchmark\n") - print("Medians of %s runs per runner, same commit, cold build cache.\n" % max(len(v) for v in by_runner.values())) + print("Medians of %s runs per runner on the same commit. **Cold** rows run with no build cache " + "(compares CPUs). **Default setup** rows use each runner as it comes " + "(GitHub: empty cache; Blacksmith: its transparent Go build cache).\n" + % max(len(v) for v in by_runner.values())) print("| | " + " | ".join(order) + " |") print("|---|" + "---|" * len(order)) for key, label in metrics: @@ -184,10 +196,14 @@ jobs: f.write(open(p).read()) EOF + # Drop the GOMAXPROCS suffix and CPU line so benchstat compares the runners in one table. + for f in merged/*.txt; do + sed -E -e 's/^(Benchmark[^[:space:]]*)-[0-9]+([[:space:]])/\1\2/' -e 's/^cpu:.*/cpu: (normalized)/' "$f" > "${f%.txt}.norm" + done "$(go env GOPATH)/bin/benchstat" \ - ubuntu-latest=merged/ubuntu-latest.txt \ - blacksmith-4vcpu=merged/blacksmith-4vcpu-ubuntu-2404.txt \ - blacksmith-8vcpu=merged/blacksmith-8vcpu-ubuntu-2404.txt > merged/benchstat.txt || true + ubuntu-latest=merged/ubuntu-latest.norm \ + blacksmith-4vcpu=merged/blacksmith-4vcpu-ubuntu-2404.norm \ + blacksmith-8vcpu=merged/blacksmith-8vcpu-ubuntu-2404.norm > merged/benchstat.txt || true { echo "### benchstat geomean (sec/op across runners; full output in the report artifact)" echo '```' From 5145d9d7cb3f3b5d7745408c54cdf0a512965af8 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 1 Oct 2026 10:03:20 -0700 Subject: [PATCH 3/4] ci: compare runners with each provider's recommended Go caching The "default setup" timings ran GitHub-hosted runners with an empty build cache, while GitHub's recommended setup restores the Actions cache through actions/setup-go. That overstated Blacksmith's advantage. A warm-up job now fills the Actions cache first, ubuntu-latest jobs restore it with setup-go (cache: true), and Blacksmith jobs keep setup-go caching off, as Blacksmith's docs recommend, since its Go build cache replaces it. The report adds the Set up Go step duration, which includes the cache restore on GitHub. Cold timings are unchanged. Claude-Session: https://claude.ai/code/session_01QKDYQr53zNKkR7nif2CAAq --- .github/workflows/runner-benchmark.yml | 53 ++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index 2e4dd63bd..42b5f4732 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -27,7 +27,28 @@ env: DOWNLOAD_ARTIFACT_ACTION: &download_artifact_action actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 jobs: + # Fills the Actions cache that actions/setup-go restores on GitHub-hosted runners, + # so the "default setup" timings reflect GitHub's recommended (cached) setup. + warm-github-cache: + name: Warm GitHub cache + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: *checkout_action + + - name: Set up Go + uses: *setup_go_action + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Build and Test + run: | + go build ./... + go test -race -count=1 ./... > /dev/null + measure: + needs: warm-github-cache strategy: fail-fast: false matrix: @@ -35,6 +56,15 @@ jobs: # blacksmith-8vcpu is the tier Echo's regular CI uses. runner: [ubuntu-latest, blacksmith-4vcpu-ubuntu-2404, blacksmith-8vcpu-ubuntu-2404] attempt: [1, 2, 3, 4, 5] + # Each runner's recommended setup: GitHub restores the Actions cache through setup-go; + # Blacksmith's Go build cache replaces it (docs.blacksmith.sh/blacksmith-caching/go-build-caching). + include: + - runner: ubuntu-latest + setup_go_cache: true + - runner: blacksmith-4vcpu-ubuntu-2404 + setup_go_cache: false + - runner: blacksmith-8vcpu-ubuntu-2404 + setup_go_cache: false name: ${{ matrix.runner }} (${{ matrix.attempt }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 @@ -46,7 +76,7 @@ jobs: uses: *setup_go_action with: go-version: ${{ env.GO_VERSION }} - cache: false # every runner starts cold + cache: ${{ matrix.setup_go_cache }} - name: Measure env: @@ -77,7 +107,8 @@ jobs: # Fixed iteration counts, so faster CPUs finish sooner instead of running more iterations. measure bench cold go test -run='^$' -bench=. -benchtime=2000x -count=2 ./... > "$out/bench.txt" - # Default: each runner's out-of-the-box setup (GitHub: empty cache; Blacksmith: transparent cache). + # Default: each runner's recommended setup (GitHub: Actions cache restored by setup-go; + # Blacksmith: its transparent Go build cache). measure build_default go build ./... measure test_race_default go test -race -count=1 ./... > /dev/null @@ -125,7 +156,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: | gh api --paginate "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \ - --jq '.jobs[] | select(.name != "Report") | [.name, .created_at, .started_at] | @tsv' > queue.tsv + --jq '.jobs[] | select(.name != "Report") | [.name, .created_at, .started_at, + ([.steps[]? | select(.name == "Set up Go")][0] // {} | .started_at // ""), + ([.steps[]? | select(.name == "Set up Go")][0] // {} | .completed_at // "")] | @tsv' > queue.tsv - name: Summarize run: | @@ -145,13 +178,16 @@ jobs: order = ["ubuntu-latest", "blacksmith-4vcpu-ubuntu-2404", "blacksmith-8vcpu-ubuntu-2404"] by_runner = {r: [x for x in runs if x.get("runner") == r] for r in order} + t = lambda s: datetime.fromisoformat(s.replace("Z", "+00:00")) queue = {r: [] for r in order} + setup_go = {r: [] for r in order} for line in open("queue.tsv"): - name, created, started = line.rstrip("\n").split("\t") + name, created, started, setup_start, setup_end = line.rstrip("\n").split("\t") runner = name.split(" (")[0] if runner in queue and started: - t = lambda s: datetime.fromisoformat(s.replace("Z", "+00:00")) queue[runner].append((t(started) - t(created)).total_seconds()) + if runner in setup_go and setup_start and setup_end: + setup_go[runner].append((t(setup_end) - t(setup_start)).total_seconds()) def med(values): return statistics.median(values) if values else None @@ -164,8 +200,9 @@ jobs: print("## Runner benchmark\n") print("Medians of %s runs per runner on the same commit. **Cold** rows run with no build cache " - "(compares CPUs). **Default setup** rows use each runner as it comes " - "(GitHub: empty cache; Blacksmith: its transparent Go build cache).\n" + "(compares CPUs). **Default setup** rows use each runner's recommended setup " + "(GitHub: Actions cache restored by `setup-go`, timed in the Set up Go row; " + "Blacksmith: its transparent Go build cache).\n" % max(len(v) for v in by_runner.values())) print("| | " + " | ".join(order) + " |") print("|---|" + "---|" * len(order)) @@ -181,6 +218,8 @@ jobs: else: cells.append("%.1f s" % (m / 1000)) print("| %s | %s |" % (label, " | ".join(cells))) + print("| Set up Go step (GitHub: includes cache restore) | %s |" % " | ".join( + "%.0f s" % med(setup_go[r]) if setup_go[r] else "n/a" for r in order)) print("| Queue time (wait for runner) | %s |" % " | ".join( "%.0f s" % med(queue[r]) if queue[r] else "n/a" for r in order)) print("| vCPUs / RAM | %s |" % " | ".join( From 32b4f2226a038a6147ad4afa7ce3b8df737ada99 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 1 Oct 2026 10:09:51 -0700 Subject: [PATCH 4/4] ci: give GitHub runners a warm cache for this commit in runner benchmark setup-go keys its cache on go.sum only. The warm-up job restored a stale master entry from September, hit the primary key and skipped saving, so the GitHub "default setup" timings ran effectively uncached. The warm-up job now saves the Go build and module caches with actions/cache under a key unique to the run, and ubuntu-latest jobs restore that exact entry (failing on a miss). The restore is timed with the Set up Go row. The CPU row now lists every CPU model seen, since both providers mix hardware across jobs. Claude-Session: https://claude.ai/code/session_01QKDYQr53zNKkR7nif2CAAq --- .github/workflows/runner-benchmark.yml | 59 +++++++++++++++++++------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/.github/workflows/runner-benchmark.yml b/.github/workflows/runner-benchmark.yml index 42b5f4732..d0a3912b2 100644 --- a/.github/workflows/runner-benchmark.yml +++ b/.github/workflows/runner-benchmark.yml @@ -20,6 +20,16 @@ env: # https://github.com/actions/setup-go/commit/924ae3a1cded613372ab5595356fb5720e22ba16 SETUP_GO_ACTION: &setup_go_action actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + # https://github.com/actions/cache/commit/55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + CACHE_SAVE_ACTION: &cache_save_action actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + CACHE_RESTORE_ACTION: &cache_restore_action actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + + # Paths and key of the Go cache the warm-up job saves for GitHub-hosted runners. + GO_CACHE_PATHS: | + ~/.cache/go-build + ~/go/pkg/mod + GO_CACHE_KEY: runner-benchmark-go-${{ github.run_id }}-${{ github.run_attempt }} + # https://github.com/actions/upload-artifact/commit/043fb46d1a93c77aae656e7c1c64a875d1fc6a0a UPLOAD_ARTIFACT_ACTION: &upload_artifact_action actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -27,8 +37,10 @@ env: DOWNLOAD_ARTIFACT_ACTION: &download_artifact_action actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 jobs: - # Fills the Actions cache that actions/setup-go restores on GitHub-hosted runners, - # so the "default setup" timings reflect GitHub's recommended (cached) setup. + # Saves a Go build and module cache for this commit, so the "default setup" timings give + # GitHub-hosted runners a fully warm Actions cache. setup-go's own cache is keyed on go.sum + # only: it restores whatever entry already exists for that key (often stale) and then skips + # saving, so it can't be relied on to hold this commit's build. warm-github-cache: name: Warm GitHub cache runs-on: ubuntu-latest @@ -40,13 +52,19 @@ jobs: uses: *setup_go_action with: go-version: ${{ env.GO_VERSION }} - cache: true + cache: false - name: Build and Test run: | go build ./... go test -race -count=1 ./... > /dev/null + - name: Save Go Cache + uses: *cache_save_action + with: + path: ${{ env.GO_CACHE_PATHS }} + key: ${{ env.GO_CACHE_KEY }} + measure: needs: warm-github-cache strategy: @@ -56,15 +74,11 @@ jobs: # blacksmith-8vcpu is the tier Echo's regular CI uses. runner: [ubuntu-latest, blacksmith-4vcpu-ubuntu-2404, blacksmith-8vcpu-ubuntu-2404] attempt: [1, 2, 3, 4, 5] - # Each runner's recommended setup: GitHub restores the Actions cache through setup-go; - # Blacksmith's Go build cache replaces it (docs.blacksmith.sh/blacksmith-caching/go-build-caching). + # Each runner's recommended setup: GitHub restores the Actions cache; Blacksmith's + # Go build cache replaces it (docs.blacksmith.sh/blacksmith-caching/go-build-caching). include: - runner: ubuntu-latest - setup_go_cache: true - - runner: blacksmith-4vcpu-ubuntu-2404 - setup_go_cache: false - - runner: blacksmith-8vcpu-ubuntu-2404 - setup_go_cache: false + github_cache: true name: ${{ matrix.runner }} (${{ matrix.attempt }}) runs-on: ${{ matrix.runner }} timeout-minutes: 30 @@ -76,7 +90,15 @@ jobs: uses: *setup_go_action with: go-version: ${{ env.GO_VERSION }} - cache: ${{ matrix.setup_go_cache }} + cache: false + + - name: Restore Go Cache + if: matrix.github_cache + uses: *cache_restore_action + with: + path: ${{ env.GO_CACHE_PATHS }} + key: ${{ env.GO_CACHE_KEY }} + fail-on-cache-miss: true - name: Measure env: @@ -107,7 +129,7 @@ jobs: # Fixed iteration counts, so faster CPUs finish sooner instead of running more iterations. measure bench cold go test -run='^$' -bench=. -benchtime=2000x -count=2 ./... > "$out/bench.txt" - # Default: each runner's recommended setup (GitHub: Actions cache restored by setup-go; + # Default: each runner's recommended setup (GitHub: the Actions cache restored above; # Blacksmith: its transparent Go build cache). measure build_default go build ./... measure test_race_default go test -race -count=1 ./... > /dev/null @@ -158,7 +180,7 @@ jobs: gh api --paginate "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \ --jq '.jobs[] | select(.name != "Report") | [.name, .created_at, .started_at, ([.steps[]? | select(.name == "Set up Go")][0] // {} | .started_at // ""), - ([.steps[]? | select(.name == "Set up Go")][0] // {} | .completed_at // "")] | @tsv' > queue.tsv + ([.steps[]? | select(.name == "Set up Go" or .name == "Restore Go Cache") | .completed_at // ""] | max // "")] | @tsv' > queue.tsv - name: Summarize run: | @@ -201,7 +223,7 @@ jobs: print("## Runner benchmark\n") print("Medians of %s runs per runner on the same commit. **Cold** rows run with no build cache " "(compares CPUs). **Default setup** rows use each runner's recommended setup " - "(GitHub: Actions cache restored by `setup-go`, timed in the Set up Go row; " + "(GitHub: a warm Actions cache for this commit, restore timed in the Set up Go row; " "Blacksmith: its transparent Go build cache).\n" % max(len(v) for v in by_runner.values())) print("| | " + " | ".join(order) + " |") @@ -218,14 +240,19 @@ jobs: else: cells.append("%.1f s" % (m / 1000)) print("| %s | %s |" % (label, " | ".join(cells))) - print("| Set up Go step (GitHub: includes cache restore) | %s |" % " | ".join( + print("| Set up Go (GitHub: includes Actions cache restore) | %s |" % " | ".join( "%.0f s" % med(setup_go[r]) if setup_go[r] else "n/a" for r in order)) print("| Queue time (wait for runner) | %s |" % " | ".join( "%.0f s" % med(queue[r]) if queue[r] else "n/a" for r in order)) print("| vCPUs / RAM | %s |" % " | ".join( ("%s / %s GB" % (by_runner[r][0].get("nproc"), by_runner[r][0].get("mem_gb"))) if by_runner[r] else "n/a" for r in order)) - print("| CPU | %s |" % " | ".join(by_runner[r][0].get("cpu", "?") if by_runner[r] else "n/a" for r in order)) + def cpus(r): + counts = {} + for x in by_runner[r]: + counts[x.get("cpu", "?")] = counts.get(x.get("cpu", "?"), 0) + 1 + return ", ".join("%s (%d)" % (c, n) for c, n in sorted(counts.items(), key=lambda kv: -kv[1])) or "n/a" + print("| CPU (runs) | %s |" % " | ".join(cpus(r) for r in order)) print("\nSpeedup is relative to ubuntu-latest (higher is faster).\n") os.makedirs("merged", exist_ok=True)