diff --git a/.claude/new-task.md b/.claude/new-task.md index b54509e..ad6d41f 100644 --- a/.claude/new-task.md +++ b/.claude/new-task.md @@ -53,5 +53,45 @@ scripts/dev.sh cargo run --release -p harness --features live -- \ run evals/set.jsonl --live # the running service ``` -`--live` needs the stack up and `SONARI_BASE_URL` set. Timings mean nothing from -a debug build. +`--live` needs the stack up, and four variables. In full: + +```bash +set -a; . ./.env; set +a +docker compose up -d # sonari, postgres, livekit + +# Compose network names, not localhost: the dev container is on that network. +export SONARI_BASE_URL=http://sonari:8080 +export SONARI_LIVEKIT_URL=ws://livekit:7880 +export SONARI_CHARACTER_ID=$(curl -s localhost:8080/api/personas | jq -r '.data[0].id') + +# Every marker the live solver reports comes from GET +# /api/admin/call-logs/{id}/timeline, which requires an admin token. Nothing +# issues one — POST /api/session issues `user` — so the run mints it, signed +# with JWT_SECRET (default `dev-secret`, crates/app/src/config.rs). `sub` must be +# the literal "access": validate_access_token rejects every other value +# (crates/auth/adapters/jwt.rs). +export SONARI_ADMIN_TOKEN=$(python - <<'TOKEN' +import base64, hashlib, hmac, json, time +def b64(raw): return base64.urlsafe_b64encode(raw).rstrip(b"=") +now = int(time.time()) +head = b64(json.dumps({"alg": "HS256", "typ": "JWT"}).encode()) +body = b64(json.dumps({"sub": "access", "user_id": 1, "role": "admin", + "perms": [], "iat": now, "exp": now + 3600}).encode()) +sig = b64(hmac.new(b"dev-secret", head + b"." + body, hashlib.sha256).digest()) +print((head + b"." + body + b"." + sig).decode()) +TOKEN +) + +scripts/dev.sh cargo run --release -p harness --features live -- \ + run evals/set.jsonl --live --epochs 3 --out evals/runs-live +``` + +The token recipe is read off `crates/auth/adapters/jwt.rs` and +`crates/app/src/config.rs` and has not been exercised against a running stack, so +if the admin surface answers 401, start with the claims: `sub` and `role` are both +checked, and the secret has to be the one the service booted with. + +`--out` matters: the default is `evals/runs`, where the component-level runs go. +Timings mean nothing from a debug build — the run records which build it was, and +`scripts/check-published-figures.sh` will not let a figure into a document that +is not in the newest run under `evals/runs-live/`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85284d6..db41c25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,3 +55,8 @@ jobs: - uses: actions/checkout@v4 - name: ADR index is consistent run: ./scripts/check-adr-index.sh + # No figure enters a document before it has been measured (product.md + # §Evidence). The run file the documents quote is in the repository, so + # this is checkable without credentials or a stack. + - name: Published figures are measured ones + run: ./scripts/check-published-figures.sh diff --git a/README.md b/README.md index fa3d4b7..7ffb03a 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,28 @@ figure that has been measured. ## Measuring it -The eval harness runs a recording through the whole pipeline without LiveKit, a +Where it stands, over the evaluation set against the running service — a probe +joining the LiveKit room as the caller: + +| | p50 | p95 | +|---|---|---| +| **System response** — you stop talking → the first audio frame leaves | 854 ms | 976 ms | +| **Perceived latency** — your last voiced frame → the first audio frame leaves | 1553 ms | 1677 ms | + +Both figures are always reported together, because the difference between them +is a decision rather than a cost: 700 ms of silence has to pass before a turn is +called finished, and the caller sits through it. + +Recognition over the same run: corpus WER 4.4%, per-clip p50 0% and p90 18%. At +16 clips that is a regression tripwire and not an instrument — the interval is +roughly ±5-10 points absolute. + +Read the limits with the numbers: one epoch, 15 clips, and the run predates the +field that records whether the build was a release one. They come from +`evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`, and +`crates/harness/OPTIMISATION-LOG.md` states what they can and cannot claim. + +The harness also runs a recording through the whole pipeline without LiveKit, a browser or a client, and prints what each stage cost: ```bash @@ -86,7 +107,8 @@ SONARI_MODELS_DIR=./models cargo run --release -p harness -- recording.wav ``` Latency figures come from release builds only — a debug build inflated one stage -by half again, which is enough to point optimisation at the wrong place. +by half again, which is enough to point optimisation at the wrong place. Each run +records which it was. ## Development diff --git a/crates/harness/OPTIMISATION-LOG.md b/crates/harness/OPTIMISATION-LOG.md index 7fca46a..5dcfdac 100644 --- a/crates/harness/OPTIMISATION-LOG.md +++ b/crates/harness/OPTIMISATION-LOG.md @@ -29,6 +29,52 @@ produced them still applies. --- +## The first figures published — 854 ms answered, 1553 ms waited + +The first run of the whole evaluation set against the running service, over +LiveKit, with a probe joining the room as the caller. These are the figures +`README.md` and `docs/architecture.md` now carry; before this entry neither +document held a number. + +Run: `evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`. + +| | p50 | p95 | +|---|---|---| +| **System response** — `speech_end` → first audio frame | **854 ms** | **976 ms** | +| **Perceived latency** — `speech_last_voiced` → first audio frame | **1553 ms** | **1677 ms** | + +Both are under the two-second target, and the gap between them is the +endpointing hangover: 700 ms of silence has to pass before a turn is called +finished, and the caller waits through all of it. That is the largest single +cost in what a caller experiences, and it is a policy value, not a slow +component. + +Recognition quality over the same run: corpus WER 4.4%, p50 0%, p90 18%. At this +set size the confidence interval is roughly ±5-10 points absolute — a regression +tripwire and a category-failure detector, not an instrument for ranking systems a +point apart. + +**What these figures cannot claim.** Read them with all of this: + +- **One epoch.** Each clip was run once, so p95 is an interpolation near the + worst sample rather than a tail. The set is 16 clips; percentiles over 48 + samples would mean considerably more. +- **15 clips, not 16.** `idle-force-agent` was added to the set after this run. +- **The build is not recorded.** This run predates the `build` field, so the file + cannot say it came from a release build, and every figure in this repository is + supposed to be a release figure. The command used carried `--release`, but the + file is the evidence and the file is silent. +- **14 of 15 samples succeeded.** `edge-8khz-stereo` failed, and by design: the + clip is 8 kHz stereo and the pipeline carries 16 kHz mono, so it was rejected + before it reached the service. The percentiles are over the 14. Two clips — + `edge-silence` and `edge-cough` — opened no turn, which is the outcome they + test for, and there were no false triggers. + +These stand until the set is re-run at three epochs from a build that says so, +which is scheduled after the features still to be built land. + +--- + ## First measured turn on hosted inference — 1325 ms The whole path, one recording through the harness: diff --git a/crates/harness/src/render.rs b/crates/harness/src/render.rs index d2aa488..c47eb01 100644 --- a/crates/harness/src/render.rs +++ b/crates/harness/src/render.rs @@ -309,6 +309,7 @@ mod tests { BatchReport { run_at: "2026-08-13T00:00:00Z".to_owned(), solver: "live".to_owned(), + build: "release".to_owned(), epochs: 1, note: "note".to_owned(), config: Default::default(), diff --git a/crates/harness/src/report.rs b/crates/harness/src/report.rs index 0524d6f..97724c0 100644 --- a/crates/harness/src/report.rs +++ b/crates/harness/src/report.rs @@ -16,6 +16,14 @@ use crate::{markers::Markers, score::wer::WordErrors}; /// Stated in every report. Fifteen recordings cannot resolve a difference of a /// point or two, and a number without that caveat invites conclusions it cannot /// support. +/// Which build produced a report. A debug build inflated one stage by half +/// again, so a figure taken from one says nothing about the system. +pub const BUILD_PROFILE: &str = if cfg!(debug_assertions) { + "debug" +} else { + "release" +}; + pub const PRECISION_NOTE: &str = "At this set size the confidence interval on WER is roughly ±5-10 points absolute. This is a regression tripwire and a category-failure detector, not an instrument for ranking systems a point apart."; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -102,6 +110,13 @@ pub struct BatchReport { /// service and does. A reader who does not know which is which can draw a /// conclusion the run cannot support. pub solver: String, + /// `release` or `debug`. A debug build inflated one stage by half again, so + /// every figure in this repository is a release figure — and a report that + /// does not say which it came from cannot be checked against that rule + /// later. Runs that predate this field have no `build` at all, which is the + /// honest answer for them, and they still deserialise. + #[serde(default)] + pub build: String, pub epochs: usize, pub note: String, pub config: ConfigSnapshot, diff --git a/crates/harness/src/runner.rs b/crates/harness/src/runner.rs index 52d66e9..2eda618 100644 --- a/crates/harness/src/runner.rs +++ b/crates/harness/src/runner.rs @@ -14,7 +14,8 @@ use crate::{ manifest::Sample, markers::Markers, report::{ - BatchReport, ConfigSnapshot, PRECISION_NOTE, SampleReport, SampleStatus, Spread, Summary, + BUILD_PROFILE, BatchReport, ConfigSnapshot, PRECISION_NOTE, SampleReport, SampleStatus, + Spread, Summary, }, score::wer::WordErrors, solver::{Outcome, Solver}, @@ -71,6 +72,7 @@ pub async fn run_batch( Ok(BatchReport { run_at: chrono::Utc::now().to_rfc3339(), solver: config.solver.to_owned(), + build: BUILD_PROFILE.to_owned(), epochs: config.epochs, note: PRECISION_NOTE.to_owned(), config: config.snapshot.clone(), diff --git a/docs/architecture.md b/docs/architecture.md index 2359402..949e829 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,12 +51,13 @@ begins and when it ends. **One binary.** The control plane and the media plane are one process (ADR-0002); the split is logical. -**The client in the diagram is Android in production, and a browser page in -development.** The binary serves a single-page test client at `GET /dev`, with -its HTML and a vendored copy of the LiveKit browser SDK compiled in -(ADR-0018, ADR-0019). It is same-origin with the API, so there is no CORS layer -anywhere. It walks the ordinary contract and gets no privilege: session, list -personas, start call, join the room, end call. +**The client in the diagram is the browser page, and today it is the only one.** +Android is the product surface, and nothing in this repository builds it yet. +The binary serves a single-page test client at `GET /dev`, with its HTML and a +vendored copy of the LiveKit browser SDK compiled in (ADR-0018, ADR-0019). It is +same-origin with the API, so there is no CORS layer anywhere. It walks the +ordinary contract and gets no privilege: session, list personas, start call, join +the room, end call. --- @@ -166,7 +167,24 @@ rather than timestamps to subtract. Two figures are always reported together: - **Perceived latency** — `speech_last_voiced` → `audio_first_frame` No latency figure enters any document until it has been measured, and -measurements come from release builds. +measurements come from release builds. Every run records which build it came +from, and `scripts/check-published-figures.sh` refuses a figure in a document +that is not a figure in the newest run. + +Where the system stands, from +`evals/runs-live/2026-08-15T19-28-00.118098504+00-00.json`: + +| | p50 | p95 | +|---|---|---| +| System response | 854 ms | 976 ms | +| Perceived latency | 1553 ms | 1677 ms | + +The 700 ms between them is the endpointing hangover (ADR-0016), not a slow +component. Recognition over the same run: corpus WER 4.4%, per-clip p50 0% and +p90 18% — a tripwire at this set size rather than a measurement of quality. + +The run was one epoch over 15 clips and predates the build field; +`crates/harness/OPTIMISATION-LOG.md` carries the conditions and the limits. --- diff --git a/docs/product.md b/docs/product.md index 4486427..1c6f7cf 100644 --- a/docs/product.md +++ b/docs/product.md @@ -17,7 +17,7 @@ exists to make that testable. ## 2. Scope -**v1** — one voice conversation, measured. +One voice conversation, measured. | | | |---|---| @@ -26,12 +26,13 @@ exists to make that testable. | Interruption | Speaking over the agent stops it | | Personas | Operator-authored: a character and the scene they are in | | Identity | A `uid` the caller enters or is assigned | -| Client | Android — enter a `uid`, choose a character and scene, talk | -| Trying it by hand | A browser test client at `/dev`, served by the binary itself (ADR-0018) | +| Client | The browser test client at `/dev`, served by the binary itself (ADR-0018) — enter a `uid`, choose a character and scene, talk | | Evaluation | An automated harness and a headless caller, both runnable in CI | -**v2** — long-term memory, and work on how human the agent sounds. v1 records -the `uid` on every session so memory has history to work with when it arrives. +**Still to build**: the Android client — it is the product surface, and until it +exists the `/dev` page is how a person reaches a call — long-term memory, and +work on how human the agent sounds. Every session records its `uid`, so memory +has history to work with when it arrives. **Not built**: SDK surface, billing, admin console, multi-tenancy, consumer login, tool calling, self-hosted inference. @@ -96,6 +97,6 @@ say so. |---|---| | Accounts and login | A companion does not need to know who you are, only which conversation is yours | | Self-hosted models | The engineering interest is the pipeline, not operating GPUs (ADR-0014) | -| Tool calling | v1 is conversation. Tools add a second round trip inside a turn, which a phone call feels | +| Tool calling | This is conversation. Tools add a second round trip inside a turn, which a phone call feels | | Multi-tenancy | One deployment, one operator, personas in a file | -| A web *product* client | Android is the product surface. A browser page is shipped at `/dev` as a test tool and is named one (ADR-0018) — it exists because a person needs to hear the call, which no automated test can judge | +| A web *product* client | Android is the product surface. The browser page shipped at `/dev` is a test tool and is named one (ADR-0018) — it exists because a person needs to hear the call, which no automated test can judge, and it is not on its way to becoming the product | diff --git a/scripts/check-published-figures.sh b/scripts/check-published-figures.sh new file mode 100755 index 0000000..a694ae0 --- /dev/null +++ b/scripts/check-published-figures.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Enforces product.md's evidence rule: no figure enters a document before it has +# been measured. Every latency and WER figure in the published documents must be +# the figure the newest live evaluation run measured, under the label that run +# measured it under — a figure in the right document under the wrong name is +# still a wrong figure. +# +# What it reads: +# evals/runs-live/*.json the newest one, and only that one +# README.md, docs/architecture.md every figure in them +# crates/harness/OPTIMISATION-LOG.md its newest entry only — the entries +# below it are history, and some describe +# a system that no longer exists +set -euo pipefail + +cd "$(dirname "$0")/.." +fail=0 + +err() { + echo "published figures: $*" >&2 + fail=1 +} + +# --- The run ----------------------------------------------------------------- + +run=$(find evals/runs-live -maxdepth 1 -name '*.json' | sort | tail -1) +if [ -z "$run" ]; then + err "nothing has been measured: no run file under evals/runs-live/" + exit 1 +fi + +# The newest log entry is everything from the first "## " heading after the +# conditions table down to the one below it. +entry_file=$(mktemp) +trap 'rm -f "$entry_file"' EXIT +awk ' + /^## / && seen { exit } + /^## / && !/^## Conditions$/ && !/^## Optimisation Log$/ { seen = 1 } + seen { print } +' crates/harness/OPTIMISATION-LOG.md > "$entry_file" + +documents=(README.md docs/architecture.md "$entry_file") +name_of() { + case "$1" in + "$entry_file") echo "OPTIMISATION-LOG.md (newest entry)" ;; + *) echo "$1" ;; + esac +} + +solver=$(jq -r '.solver // "missing"' "$run") +build=$(jq -r '.build // "absent"' "$run") +[ "$solver" = "live" ] || err "$run: solver is \"$solver\", not \"live\" — component runs do not measure what a caller waits" + +# `build` is written by every run since the field was added. An older file +# predates it and cannot say; a file that does say must say release, because a +# debug build inflates a stage by half again. +if [ "$build" != "absent" ] && [ "$build" != "release" ]; then + err "$run: build is \"$build\" — figures come from release builds (product.md §Evidence)" +fi + +ok=$(jq -r '.summary.samples_ok // 0' "$run") +[ "$ok" -ge 14 ] || err "$run: samples_ok is $ok, fewer than the 14 of the 2026-08-15 run — the percentiles cover a hole" + +# Provenance. A document quoting figures names the run they came from, and it has +# to be the run this script checks against — otherwise a newer run lands, the +# published figures go stale, and they all still pass because they were once true. +for doc in "${documents[@]}"; do + cited=$(grep -oE 'evals/runs-live/[^ )`]+\.json' "$doc" | sort -u || true) + count=$(printf '%s' "$cited" | grep -c . || true) + if [ "$count" -eq 0 ]; then + err "$(name_of "$doc"): quotes no run file; a figure without its provenance cannot be checked" + elif [ "$count" -gt 1 ]; then + err "$(name_of "$doc"): quotes more than one run file, so which one its figures came from is a guess" + elif [ "$cited" != "$run" ]; then + err "$(name_of "$doc"): quotes $cited, but the newest run is $run — the figures are stale" + fi +done + +# --- What the run measured --------------------------------------------------- +# +# Milliseconds are published rounded to the integer, WER as a percentage to one +# decimal. + +ms() { jq -r "$1 | round" "$run"; } +pct() { jq -r "$1 | . * 1000 | round | . / 10" "$run" | sed 's/\.0$//'; } + +system_p50=$(ms '.summary.system_response.p50') +system_p95=$(ms '.summary.system_response.p95') +perceived_p50=$(ms '.summary.perceived_latency.p50') +perceived_p95=$(ms '.summary.perceived_latency.p95') +summary_ms=$(printf '%s\n' "$system_p50" "$system_p95" "$perceived_p50" "$perceived_p95" | sort -u) +# The endpointing values the run was taken under. A document explaining why the +# two latency figures differ has to be able to name the hangover, and only a line +# talking about endpointing may quote one. +config_ms=$(jq -r '.config | (.silence_flush_ms, .min_utterance_ms, .min_speech_confirm_ms)' "$run" | sort -u) +wer_all=$( + printf '%s\n' "$(pct '.summary.corpus_wer')" "$(pct '.summary.wer_p50')" "$(pct '.summary.wer_p90')" | + sort -u +) + +member() { echo "$2" | grep -qx "$1"; } + +# --- Every figure, under the label it was measured under --------------------- + +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + line_number=0 + while IFS= read -r line; do + line_number=$((line_number + 1)) + + found_ms=$(echo "$line" | grep -oE '[0-9]+ ?ms' | grep -oE '[0-9]+' | tr '\n' ' ' | sed 's/ $//' || true) + if [ -n "$found_ms" ]; then + case "$line" in + *"system response"* | *"System response"*) expected="$system_p50 $system_p95" ;; + *"perceived latency"* | *"Perceived latency"*) expected="$perceived_p50 $perceived_p95" ;; + *) expected="" ;; + esac + + if [ -n "$expected" ]; then + # A row naming one of the two figures carries that figure's p50 and p95, + # in that order. Quoting the other figure's numbers under this name, or + # swapping the pair, is what this catches. + if [ "$found_ms" != "$expected" ] && + [ "$found_ms" != "${expected%% *}" ] && + [ "$found_ms" != "${expected##* }" ]; then + err "$name:$line_number: $found_ms is not what the run measured under that name (${expected})" + fi + else + for value in $found_ms; do + case "$line" in + *hangover* | *endpoint* | *silence*) + member "$value" "$config_ms" || + err "$name:$line_number: $value ms is not an endpointing value in $run" + ;; + *) + member "$value" "$summary_ms" || + err "$name:$line_number: $value ms is not a figure in $run" + ;; + esac + done + fi + fi + + found_pct=$(echo "$line" | grep -oE '[0-9]+(\.[0-9])? ?%' | grep -oE '[0-9]+(\.[0-9])?' || true) + for value in $found_pct; do + member "$value" "$wer_all" || + err "$name:$line_number: $value% is not an error rate in $run" + done + done < "$doc" +done + +# --- How the figures are presented ------------------------------------------- + +# ADR-0010: the two latency figures are always reported together. +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + system=$(grep -ci 'system response' "$doc" || true) + perceived=$(grep -ci 'perceived latency' "$doc" || true) + if [ "$system" -gt 0 ] && [ "$perceived" -eq 0 ]; then + err "$name: system response is reported without perceived latency (ADR-0010)" + fi + if [ "$perceived" -gt 0 ] && [ "$system" -eq 0 ]; then + err "$name: perceived latency is reported without system response (ADR-0010)" + fi +done + +# A WER figure without the caveat the run carries is a figure read as an +# instrument, which at this set size it is not. +for doc in "${documents[@]}"; do + name=$(name_of "$doc") + if grep -q 'WER' "$doc" && ! grep -qiE 'tripwire|points absolute|confidence interval' "$doc"; then + err "$name: a WER figure is published without the caveat about the size of the set" + fi +done + +# The conditions are part of the figure: how many epochs the percentiles cover, +# and — while a run predating the `build` field is still the newest — that the +# build was not recorded. +epochs=$(jq -r '.epochs // 0' "$run") +grep -qi 'epoch' "$entry_file" || + err "OPTIMISATION-LOG.md (newest entry): does not say the figures come from $epochs epoch(s); the sample count is part of the figure" +if [ "$build" = "absent" ]; then + grep -qiE 'build [a-z ]{0,12}not recorded|does not record the build' "$entry_file" || + err "OPTIMISATION-LOG.md (newest entry): $run predates the build field, and the entry does not say so" +fi + +# --- What the documents must not say ----------------------------------------- + +# Requirements state the target; the measurement lives in the log. +if grep -Eq '[0-9]+ ?ms' docs/product.md; then + err "docs/product.md: a measurement belongs in the optimisation log, not in the requirements" +fi + +# Android is the product surface and is not built. No document may present it as +# something a caller can use. docs/adr/ is not read: accepted records are history +# and are never edited. +if grep -qEi '^\| Client \|.*Android|client .*is Android|Android in production' docs/product.md docs/architecture.md; then + err "Android is presented as an existing client; it is still to be built" +fi + +if [ "$fail" -eq 0 ]; then + echo "published figures: consistent with $run" +fi +exit "$fail"