From 2214ba0326205c2ab0919f0286f7478e8e17dfca Mon Sep 17 00:00:00 2001 From: coderloganli Date: Mon, 17 Aug 2026 15:50:56 -0400 Subject: [PATCH] Publish the latency the system was measured at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README and architecture.md claimed sub-2s and carried no number. The figures existed — a live run over the evaluation set on 2026-08-15 — and had never left the JSON. 854 ms system response, 1553 ms perceived, at p50; 976 and 1677 at p95. Corpus WER 4.4%. Both latency figures are reported together wherever either appears (ADR-0010), and every figure names the run it came from. That run's limits are stated beside the numbers rather than left for a reader to discover: one epoch, 15 clips where the set now holds 16, and no record of whether the build was a release one. Re-running it at three epochs is a ticket of its own, for after the work still to come. check-published-figures.sh, in the CI docs job, holds the documents to the run: a figure has to be the figure the newest run measured, under the label it measured it under, and the document has to cite that run. The next run turns CI red until the three documents are rewritten, which is what makes the re-run ticket land rather than linger. BatchReport gains `build`, so the release-build rule product.md states is checkable after the fact instead of taken on trust. Reports written before it still deserialise. Android moves to what is still to be built: product.md listed it as the client and architecture.md called it the client in production, and no Android source has ever been in this repository. The v1/v2 labels go with it — this project has no version scheme. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HNYsFLhu6UE4Q4GY7nG2ng --- .claude/new-task.md | 44 ++++++- .github/workflows/ci.yml | 5 + README.md | 26 +++- crates/harness/OPTIMISATION-LOG.md | 46 +++++++ crates/harness/src/render.rs | 1 + crates/harness/src/report.rs | 15 +++ crates/harness/src/runner.rs | 4 +- docs/architecture.md | 32 ++++- docs/product.md | 15 ++- scripts/check-published-figures.sh | 204 +++++++++++++++++++++++++++++ 10 files changed, 373 insertions(+), 19 deletions(-) create mode 100755 scripts/check-published-figures.sh diff --git a/.claude/new-task.md b/.claude/new-task.md index b54509e..ad6d41f 100644 --- a/.claude/new-task.md +++ b/.claude/new-task.md @@ -53,5 +53,45 @@ scripts/dev.sh cargo run --release -p harness --features live -- \ run evals/set.jsonl --live # the running service ``` -`--live` needs the stack up and `SONARI_BASE_URL` set. Timings mean nothing from -a debug build. +`--live` needs the stack up, and four variables. In full: + +```bash +set -a; . ./.env; set +a +docker compose up -d # sonari, postgres, livekit + +# Compose network names, not localhost: the dev container is on that network. +export SONARI_BASE_URL=http://sonari:8080 +export SONARI_LIVEKIT_URL=ws://livekit:7880 +export SONARI_CHARACTER_ID=$(curl -s localhost:8080/api/personas | jq -r '.data[0].id') + +# Every marker the live solver reports comes from GET +# /api/admin/call-logs/{id}/timeline, which requires an admin token. Nothing +# issues one — POST /api/session issues `user` — so the run mints it, signed +# with JWT_SECRET (default `dev-secret`, crates/app/src/config.rs). `sub` must be +# the literal "access": validate_access_token rejects every other value +# (crates/auth/adapters/jwt.rs). +export SONARI_ADMIN_TOKEN=$(python - <<'TOKEN' +import base64, hashlib, hmac, json, time +def b64(raw): return base64.urlsafe_b64encode(raw).rstrip(b"=") +now = int(time.time()) +head = b64(json.dumps({"alg": "HS256", "typ": "JWT"}).encode()) +body = b64(json.dumps({"sub": "access", "user_id": 1, "role": "admin", + "perms": [], "iat": now, "exp": now + 3600}).encode()) +sig = b64(hmac.new(b"dev-secret", head + b"." + body, hashlib.sha256).digest()) +print((head + b"." + body + b"." + sig).decode()) +TOKEN +) + +scripts/dev.sh cargo run --release -p harness --features live -- \ + run evals/set.jsonl --live --epochs 3 --out evals/runs-live +``` + +The token recipe is read off `crates/auth/adapters/jwt.rs` and +`crates/app/src/config.rs` and has not been exercised against a running stack, so +if the admin surface answers 401, start with the claims: `sub` and `role` are both +checked, and the secret has to be the one the service booted with. + +`--out` matters: the default is `evals/runs`, where the component-level runs go. +Timings mean nothing from a debug build — the run records which build it was, and +`scripts/check-published-figures.sh` will not let a figure into a document that +is not in the newest run under `evals/runs-live/`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85284d6..db41c25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,3 +55,8 @@ jobs: - uses: actions/checkout@v4 - name: ADR index is consistent run: ./scripts/check-adr-index.sh + # No figure enters a document before it has been measured (product.md + # §Evidence). The run file the documents quote is in the repository, so + # this is checkable without credentials or a stack. + - name: Published figures are measured ones + run: ./scripts/check-published-figures.sh diff --git a/README.md b/README.md index fa3d4b7..7ffb03a 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,28 @@ figure that has been measured. ## Measuring it -The eval harness runs a recording through the whole pipeline without LiveKit, a +Where it stands, over the evaluation set against the running service — a probe +joining the LiveKit room as the caller: + +| | p50 | p95 | +|---|---|---| +| **System response** — you stop talking → the first audio frame leaves | 854 ms | 976 ms | +| **Perceived latency** — your last voiced frame → the first audio frame leaves | 1553 ms | 1677 ms | + +Both figures are always reported together, because the difference between them +is a decision rather than a cost: 700 ms of silence has to pass before a turn is +called finished, and the caller sits through it. + +Recognition over the same run: corpus WER 4.4%, per-clip p50 0% and p90 18%. At +16 clips that is a regression tripwire and not an instrument — the interval is +roughly ±5-10 points absolute. + +Read the limits with the numbers: one epoch, 15 clips, and the run predates the +field that records whether the build was a release one. They come from +`evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`, and +`crates/harness/OPTIMISATION-LOG.md` states what they can and cannot claim. + +The harness also runs a recording through the whole pipeline without LiveKit, a browser or a client, and prints what each stage cost: ```bash @@ -86,7 +107,8 @@ SONARI_MODELS_DIR=./models cargo run --release -p harness -- recording.wav ``` Latency figures come from release builds only — a debug build inflated one stage -by half again, which is enough to point optimisation at the wrong place. +by half again, which is enough to point optimisation at the wrong place. Each run +records which it was. ## Development diff --git a/crates/harness/OPTIMISATION-LOG.md b/crates/harness/OPTIMISATION-LOG.md index 7fca46a..5dcfdac 100644 --- a/crates/harness/OPTIMISATION-LOG.md +++ b/crates/harness/OPTIMISATION-LOG.md @@ -29,6 +29,52 @@ produced them still applies. --- +## The first figures published — 854 ms answered, 1553 ms waited + +The first run of the whole evaluation set against the running service, over +LiveKit, with a probe joining the room as the caller. These are the figures +`README.md` and `docs/architecture.md` now carry; before this entry neither +document held a number. + +Run: `evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`. + +| | p50 | p95 | +|---|---|---| +| **System response** — `speech_end` → first audio frame | **854 ms** | **976 ms** | +| **Perceived latency** — `speech_last_voiced` → first audio frame | **1553 ms** | **1677 ms** | + +Both are under the two-second target, and the gap between them is the +endpointing hangover: 700 ms of silence has to pass before a turn is called +finished, and the caller waits through all of it. That is the largest single +cost in what a caller experiences, and it is a policy value, not a slow +component. + +Recognition quality over the same run: corpus WER 4.4%, p50 0%, p90 18%. At this +set size the confidence interval is roughly ±5-10 points absolute — a regression +tripwire and a category-failure detector, not an instrument for ranking systems a +point apart. + +**What these figures cannot claim.** Read them with all of this: + +- **One epoch.** Each clip was run once, so p95 is an interpolation near the + worst sample rather than a tail. The set is 16 clips; percentiles over 48 + samples would mean considerably more. +- **15 clips, not 16.** `idle-force-agent` was added to the set after this run. +- **The build is not recorded.** This run predates the `build` field, so the file + cannot say it came from a release build, and every figure in this repository is + supposed to be a release figure. The command used carried `--release`, but the + file is the evidence and the file is silent. +- **14 of 15 samples succeeded.** `edge-8khz-stereo` failed, and by design: the + clip is 8 kHz stereo and the pipeline carries 16 kHz mono, so it was rejected + before it reached the service. The percentiles are over the 14. Two clips — + `edge-silence` and `edge-cough` — opened no turn, which is the outcome they + test for, and there were no false triggers. + +These stand until the set is re-run at three epochs from a build that says so, +which is scheduled after the features still to be built land. + +--- + ## First measured turn on hosted inference — 1325 ms The whole path, one recording through the harness: diff --git a/crates/harness/src/render.rs b/crates/harness/src/render.rs index d2aa488..c47eb01 100644 --- a/crates/harness/src/render.rs +++ b/crates/harness/src/render.rs @@ -309,6 +309,7 @@ mod tests { BatchReport { run_at: "2026-08-13T00:00:00Z".to_owned(), solver: "live".to_owned(), + build: "release".to_owned(), epochs: 1, note: "note".to_owned(), config: Default::default(), diff --git a/crates/harness/src/report.rs b/crates/harness/src/report.rs index 0524d6f..97724c0 100644 --- a/crates/harness/src/report.rs +++ b/crates/harness/src/report.rs @@ -16,6 +16,14 @@ use crate::{markers::Markers, score::wer::WordErrors}; /// Stated in every report. Fifteen recordings cannot resolve a difference of a /// point or two, and a number without that caveat invites conclusions it cannot /// support. +/// Which build produced a report. A debug build inflated one stage by half +/// again, so a figure taken from one says nothing about the system. +pub const BUILD_PROFILE: &str = if cfg!(debug_assertions) { + "debug" +} else { + "release" +}; + pub const PRECISION_NOTE: &str = "At this set size the confidence interval on WER is roughly ±5-10 points absolute. This is a regression tripwire and a category-failure detector, not an instrument for ranking systems a point apart."; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -102,6 +110,13 @@ pub struct BatchReport { /// service and does. A reader who does not know which is which can draw a /// conclusion the run cannot support. pub solver: String, + /// `release` or `debug`. A debug build inflated one stage by half again, so + /// every figure in this repository is a release figure — and a report that + /// does not say which it came from cannot be checked against that rule + /// later. Runs that predate this field have no `build` at all, which is the + /// honest answer for them, and they still deserialise. + #[serde(default)] + pub build: String, pub epochs: usize, pub note: String, pub config: ConfigSnapshot, diff --git a/crates/harness/src/runner.rs b/crates/harness/src/runner.rs index 52d66e9..2eda618 100644 --- a/crates/harness/src/runner.rs +++ b/crates/harness/src/runner.rs @@ -14,7 +14,8 @@ use crate::{ manifest::Sample, markers::Markers, report::{ - BatchReport, ConfigSnapshot, PRECISION_NOTE, SampleReport, SampleStatus, Spread, Summary, + BUILD_PROFILE, BatchReport, ConfigSnapshot, PRECISION_NOTE, SampleReport, SampleStatus, + Spread, Summary, }, score::wer::WordErrors, solver::{Outcome, Solver}, @@ -71,6 +72,7 @@ pub async fn run_batch( Ok(BatchReport { run_at: chrono::Utc::now().to_rfc3339(), solver: config.solver.to_owned(), + build: BUILD_PROFILE.to_owned(), epochs: config.epochs, note: PRECISION_NOTE.to_owned(), config: config.snapshot.clone(), diff --git a/docs/architecture.md b/docs/architecture.md index 2359402..949e829 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,12 +51,13 @@ begins and when it ends. **One binary.** The control plane and the media plane are one process (ADR-0002); the split is logical. -**The client in the diagram is Android in production, and a browser page in -development.** The binary serves a single-page test client at `GET /dev`, with -its HTML and a vendored copy of the LiveKit browser SDK compiled in -(ADR-0018, ADR-0019). It is same-origin with the API, so there is no CORS layer -anywhere. It walks the ordinary contract and gets no privilege: session, list -personas, start call, join the room, end call. +**The client in the diagram is the browser page, and today it is the only one.** +Android is the product surface, and nothing in this repository builds it yet. +The binary serves a single-page test client at `GET /dev`, with its HTML and a +vendored copy of the LiveKit browser SDK compiled in (ADR-0018, ADR-0019). It is +same-origin with the API, so there is no CORS layer anywhere. It walks the +ordinary contract and gets no privilege: session, list personas, start call, join +the room, end call. --- @@ -166,7 +167,24 @@ rather than timestamps to subtract. Two figures are always reported together: - **Perceived latency** — `speech_last_voiced` → `audio_first_frame` No latency figure enters any document until it has been measured, and -measurements come from release builds. +measurements come from release builds. Every run records which build it came +from, and `scripts/check-published-figures.sh` refuses a figure in a document +that is not a figure in the newest run. + +Where the system stands, from +`evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`: + +| | p50 | p95 | +|---|---|---| +| System response | 854 ms | 976 ms | +| Perceived latency | 1553 ms | 1677 ms | + +The 700 ms between them is the endpointing hangover (ADR-0016), not a slow +component. Recognition over the same run: corpus WER 4.4%, per-clip p50 0% and +p90 18% — a tripwire at this set size rather than a measurement of quality. + +The run was one epoch over 15 clips and predates the build field; +`crates/harness/OPTIMISATION-LOG.md` carries the conditions and the limits. --- diff --git a/docs/product.md b/docs/product.md index 4486427..1c6f7cf 100644 --- a/docs/product.md +++ b/docs/product.md @@ -17,7 +17,7 @@ exists to make that testable. ## 2. Scope -**v1** — one voice conversation, measured. +One voice conversation, measured. | | | |---|---| @@ -26,12 +26,13 @@ exists to make that testable. | Interruption | Speaking over the agent stops it | | Personas | Operator-authored: a character and the scene they are in | | Identity | A `uid` the caller enters or is assigned | -| Client | Android — enter a `uid`, choose a character and scene, talk | -| Trying it by hand | A browser test client at `/dev`, served by the binary itself (ADR-0018) | +| Client | The browser test client at `/dev`, served by the binary itself (ADR-0018) — enter a `uid`, choose a character and scene, talk | | Evaluation | An automated harness and a headless caller, both runnable in CI | -**v2** — long-term memory, and work on how human the agent sounds. v1 records -the `uid` on every session so memory has history to work with when it arrives. +**Still to build**: the Android client — it is the product surface, and until it +exists the `/dev` page is how a person reaches a call — long-term memory, and +work on how human the agent sounds. Every session records its `uid`, so memory +has history to work with when it arrives. **Not built**: SDK surface, billing, admin console, multi-tenancy, consumer login, tool calling, self-hosted inference. @@ -96,6 +97,6 @@ say so. |---|---| | Accounts and login | A companion does not need to know who you are, only which conversation is yours | | Self-hosted models | The engineering interest is the pipeline, not operating GPUs (ADR-0014) | -| Tool calling | v1 is conversation. Tools add a second round trip inside a turn, which a phone call feels | +| Tool calling | This is conversation. Tools add a second round trip inside a turn, which a phone call feels | | Multi-tenancy | One deployment, one operator, personas in a file | -| A web *product* client | Android is the product surface. A browser page is shipped at `/dev` as a test tool and is named one (ADR-0018) — it exists because a person needs to hear the call, which no automated test can judge | +| A web *product* client | Android is the product surface. The browser page shipped at `/dev` is a test tool and is named one (ADR-0018) — it exists because a person needs to hear the call, which no automated test can judge, and it is not on its way to becoming the product | diff --git a/scripts/check-published-figures.sh b/scripts/check-published-figures.sh new file mode 100755 index 0000000..a694ae0 --- /dev/null +++ b/scripts/check-published-figures.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Enforces product.md's evidence rule: no figure enters a document before it has +# been measured. Every latency and WER figure in the published documents must be +# the figure the newest live evaluation run measured, under the label that run +# measured it under — a figure in the right document under the wrong name is +# still a wrong figure. +# +# What it reads: +# evals/runs-live/*.json the newest one, and only that one +# README.md, docs/architecture.md every figure in them +# crates/harness/OPTIMISATION-LOG.md its newest entry only — the entries +# below it are history, and some describe +# a system that no longer exists +set -euo pipefail + +cd "$(dirname "$0")/.." +fail=0 + +err() { + echo "published figures: $*" >&2 + fail=1 +} + +# --- The run ----------------------------------------------------------------- + +run=$(find evals/runs-live -maxdepth 1 -name '*.json' | sort | tail -1) +if [ -z "$run" ]; then + err "nothing has been measured: no run file under evals/runs-live/" + exit 1 +fi + +# The newest log entry is everything from the first "## " heading after the +# conditions table down to the one below it. +entry_file=$(mktemp) +trap 'rm -f "$entry_file"' EXIT +awk ' + /^## / && seen { exit } + /^## / && !/^## Conditions$/ && !/^## Optimisation Log$/ { seen = 1 } + seen { print } +' crates/harness/OPTIMISATION-LOG.md > "$entry_file" + +documents=(README.md docs/architecture.md "$entry_file") +name_of() { + case "$1" in + "$entry_file") echo "OPTIMISATION-LOG.md (newest entry)" ;; + *) echo "$1" ;; + esac +} + +solver=$(jq -r '.solver // "missing"' "$run") +build=$(jq -r '.build // "absent"' "$run") +[ "$solver" = "live" ] || err "$run: solver is \"$solver\", not \"live\" — component runs do not measure what a caller waits" + +# `build` is written by every run since the field was added. An older file +# predates it and cannot say; a file that does say must say release, because a +# debug build inflates a stage by half again. +if [ "$build" != "absent" ] && [ "$build" != "release" ]; then + err "$run: build is \"$build\" — figures come from release builds (product.md §Evidence)" +fi + +ok=$(jq -r '.summary.samples_ok // 0' "$run") +[ "$ok" -ge 14 ] || err "$run: samples_ok is $ok, fewer than the 14 of the 2026-08-15 run — the percentiles cover a hole" + +# Provenance. A document quoting figures names the run they came from, and it has +# to be the run this script checks against — otherwise a newer run lands, the +# published figures go stale, and they all still pass because they were once true. +for doc in "${documents[@]}"; do + cited=$(grep -oE 'evals/runs-live/[^ )`]+\.json' "$doc" | sort -u || true) + count=$(printf '%s' "$cited" | grep -c . || true) + if [ "$count" -eq 0 ]; then + err "$(name_of "$doc"): quotes no run file; a figure without its provenance cannot be checked" + elif [ "$count" -gt 1 ]; then + err "$(name_of "$doc"): quotes more than one run file, so which one its figures came from is a guess" + elif [ "$cited" != "$run" ]; then + err "$(name_of "$doc"): quotes $cited, but the newest run is $run — the figures are stale" + fi +done + +# --- What the run measured --------------------------------------------------- +# +# Milliseconds are published rounded to the integer, WER as a percentage to one +# decimal. + +ms() { jq -r "$1 | round" "$run"; } +pct() { jq -r "$1 | . * 1000 | round | . / 10" "$run" | sed 's/\.0$//'; } + +system_p50=$(ms '.summary.system_response.p50') +system_p95=$(ms '.summary.system_response.p95') +perceived_p50=$(ms '.summary.perceived_latency.p50') +perceived_p95=$(ms '.summary.perceived_latency.p95') +summary_ms=$(printf '%s\n' "$system_p50" "$system_p95" "$perceived_p50" "$perceived_p95" | sort -u) +# The endpointing values the run was taken under. A document explaining why the +# two latency figures differ has to be able to name the hangover, and only a line +# talking about endpointing may quote one. +config_ms=$(jq -r '.config | (.silence_flush_ms, .min_utterance_ms, .min_speech_confirm_ms)' "$run" | sort -u) +wer_all=$( + printf '%s\n' "$(pct '.summary.corpus_wer')" "$(pct '.summary.wer_p50')" "$(pct '.summary.wer_p90')" | + sort -u +) + +member() { echo "$2" | grep -qx "$1"; } + +# --- Every figure, under the label it was measured under --------------------- + +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + line_number=0 + while IFS= read -r line; do + line_number=$((line_number + 1)) + + found_ms=$(echo "$line" | grep -oE '[0-9]+ ?ms' | grep -oE '[0-9]+' | tr '\n' ' ' | sed 's/ $//' || true) + if [ -n "$found_ms" ]; then + case "$line" in + *"system response"* | *"System response"*) expected="$system_p50 $system_p95" ;; + *"perceived latency"* | *"Perceived latency"*) expected="$perceived_p50 $perceived_p95" ;; + *) expected="" ;; + esac + + if [ -n "$expected" ]; then + # A row naming one of the two figures carries that figure's p50 and p95, + # in that order. Quoting the other figure's numbers under this name, or + # swapping the pair, is what this catches. + if [ "$found_ms" != "$expected" ] && + [ "$found_ms" != "${expected%% *}" ] && + [ "$found_ms" != "${expected##* }" ]; then + err "$name:$line_number: $found_ms is not what the run measured under that name (${expected})" + fi + else + for value in $found_ms; do + case "$line" in + *hangover* | *endpoint* | *silence*) + member "$value" "$config_ms" || + err "$name:$line_number: $value ms is not an endpointing value in $run" + ;; + *) + member "$value" "$summary_ms" || + err "$name:$line_number: $value ms is not a figure in $run" + ;; + esac + done + fi + fi + + found_pct=$(echo "$line" | grep -oE '[0-9]+(\.[0-9])? ?%' | grep -oE '[0-9]+(\.[0-9])?' || true) + for value in $found_pct; do + member "$value" "$wer_all" || + err "$name:$line_number: $value% is not an error rate in $run" + done + done < "$doc" +done + +# --- How the figures are presented ------------------------------------------- + +# ADR-0010: the two latency figures are always reported together. +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + system=$(grep -ci 'system response' "$doc" || true) + perceived=$(grep -ci 'perceived latency' "$doc" || true) + if [ "$system" -gt 0 ] && [ "$perceived" -eq 0 ]; then + err "$name: system response is reported without perceived latency (ADR-0010)" + fi + if [ "$perceived" -gt 0 ] && [ "$system" -eq 0 ]; then + err "$name: perceived latency is reported without system response (ADR-0010)" + fi +done + +# A WER figure without the caveat the run carries is a figure read as an +# instrument, which at this set size it is not. +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + if grep -q 'WER' "$doc" && ! grep -qiE 'tripwire|points absolute|confidence interval' "$doc"; then + err "$name: a WER figure is published without the caveat about the size of the set" + fi +done + +# The conditions are part of the figure: how many epochs the percentiles cover, +# and — while a run predating the `build` field is still the newest — that the +# build was not recorded. +epochs=$(jq -r '.epochs // 0' "$run") +grep -qi 'epoch' "$entry_file" || + err "OPTIMISATION-LOG.md (newest entry): does not say the figures come from $epochs epoch(s); the sample count is part of the figure" +if [ "$build" = "absent" ]; then + grep -qiE 'build [a-z ]{0,12}not recorded|does not record the build' "$entry_file" || + err "OPTIMISATION-LOG.md (newest entry): $run predates the build field, and the entry does not say so" +fi + +# --- What the documents must not say ----------------------------------------- + +# Requirements state the target; the measurement lives in the log. +if grep -Eq '[0-9]+ ?ms' docs/product.md; then + err "docs/product.md: a measurement belongs in the optimisation log, not in the requirements" +fi + +# Android is the product surface and is not built. No document may present it as +# something a caller can use. docs/adr/ is not read: accepted records are history +# and are never edited. +if grep -qEi '^\| Client \|.*Android|client .*is Android|Android in production' docs/product.md docs/architecture.md; then + err "Android is presented as an existing client; it is still to be built" +fi + +if [ "$fail" -eq 0 ]; then + echo "published figures: consistent with $run" +fi +exit "$fail"