From 770369602b9e74fffebe43c0b8e87e4225b704fc Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 13:24:33 -0700 Subject: [PATCH 1/6] chore(deps): agent-runtime 0.175.0, add agent-eval, and prove the join compiles The runtime was pinned at 0.142.3 against a current 0.175.0, and agent-eval was not installed at all, so the optimizer this study wants to use could not be typechecked against. 0.175.0 carries what a game needs. `VerifyResult` gained `keepGoing` and `score`, which is the disposition a score requires: a test suite is green and there is nothing left to win, while a game always has a higher number above it. `hillclimb.ts` already returns that exact shape. `examples/agent-eval-fit.mts` is the proof, and it exists to be compiled. A playproof cell types as an agent-eval `Scenario`, where `seedGroup` is what makes two profiles on one game a paired comparison instead of two unrelated samples. Playing a candidate types as a `Verifier`. A change on either side that breaks the join now fails the typecheck instead of failing a study three hours in. Also fixes a flaky test the bump exposed. It spawned a child that exits, slept 120 ms, then asserted the driver had already been told the child was gone. That asserts how busy the machine is. It now waits for the condition, and passed 10 of 10 consecutive runs. --- examples/agent-eval-fit.mts | 54 ++++++++++++++++++++++++++++++ package.json | 3 +- pnpm-lock.yaml | 66 ++++++++++++++++++++++--------------- stream-driver.test.mts | 30 ++++++++++++++--- 4 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 examples/agent-eval-fit.mts diff --git a/examples/agent-eval-fit.mts b/examples/agent-eval-fit.mts new file mode 100644 index 0000000..31e7088 --- /dev/null +++ b/examples/agent-eval-fit.mts @@ -0,0 +1,54 @@ +/** + * The two joins between playproof and the optimizer in agent-runtime. + * + * `improve` searches an agent profile over ten surfaces: prompt, skills, tools, + * mcp, hooks, subagents, agent-profile, memory, code and rollout-policy. It + * partitions scenarios into train, selection and final test, seals a digest of + * each split into the lineage, and reports a lift with a simultaneous + * paired-bootstrap interval. None of that is game-specific and none of it needs + * to be rebuilt here. + * + * What it cannot know is whether a program plays a game well. These two + * functions are that, and nothing more. + * + * This file exists to be COMPILED. It is the proof that the shapes line up, so + * a change on either side that breaks the join fails the typecheck instead of + * failing a study three hours in. + */ +import type { Verifier, VerifyResult } from '@tangle-network/agent-runtime' +import type { Scenario } from '@tangle-network/agent-eval/campaign' +import { playCandidate } from '../hillclimb' +import { makeNative2048, NATIVE_2048_INPUTS } from '../adapters/native-2048' +import type { MatrixCell } from '../matrix' + +// 1. A playproof cell is a Scenario. `seedGroup` is what makes two profiles on +// the same game a PAIRED comparison instead of two unrelated samples. +export function cellAsScenario(cell: MatrixCell): Scenario { + return { + id: `${cell.game.id}/${cell.objective.id}/${cell.protocol.id}/${cell.sensor.id}/seed${cell.seed}`, + kind: 'playproof-game', + tags: [cell.game.adapter, cell.game.target], + seedGroup: `${cell.game.id}:${cell.seed}`, + } +} + +// 2. Playing a candidate is a Verifier the shot loop can climb. +export const gameVerifier: Verifier = async (worktreePath): Promise => { + const attempt = await playCandidate(worktreePath, { + build: () => { + const a = makeNative2048(0) + return { game: a.game, contract: a.contract, commands: NATIVE_2048_INPUTS, dispose: a.dispose } + }, + policyPath: 'policy', + horizon: 400, + seed: 0, + scoreField: 'score', + target: 2048, + }) + return { + ok: attempt.ok, + keepGoing: attempt.keepGoing, + ...(attempt.score === null ? {} : { score: attempt.score }), + feedback: attempt.feedback, + } +} diff --git a/package.json b/package.json index 67c38cf..fc10f4e 100644 --- a/package.json +++ b/package.json @@ -143,7 +143,8 @@ "prepublishOnly": "pnpm run ci" }, "devDependencies": { - "@tangle-network/agent-runtime": "0.142.3", + "@tangle-network/agent-eval": "0.170.0", + "@tangle-network/agent-runtime": "0.175.0", "@types/node": "^25.9.3", "tsup": "^8.5.0", "tsx": "^4.19.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a21e00..f05781b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,9 +8,12 @@ importers: .: devDependencies: + '@tangle-network/agent-eval': + specifier: 0.170.0 + version: 0.170.0 '@tangle-network/agent-runtime': - specifier: 0.142.3 - version: 0.142.3(@tangle-network/agent-eval@0.149.0)(@tangle-network/agent-interface@1.3.0) + specifier: 0.175.0 + version: 0.175.0(@tangle-network/agent-eval@0.170.0)(@tangle-network/agent-interface@1.6.0) '@types/node': specifier: ^25.9.3 version: 25.9.5 @@ -519,35 +522,38 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@tangle-network/agent-eval@0.149.0': - resolution: {integrity: sha512-zCU+hhfAhQOPOBCHCErQfFaKTEMdtUWlUR/ZfHqQw6gPPBhHk/+VaFiP4reCca3RfBkUnJtTQyoocqOH+L449w==} + '@tangle-network/agent-eval@0.170.0': + resolution: {integrity: sha512-GoUIrtnXbN4gSDxHNB3c8vhA2JvPTlLtx7dnWZRCUFArQU2EMIvqhyVPaKnCmoJgDfbPgCwRuv60Pw2w9jSOeA==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@1.3.0': resolution: {integrity: sha512-FsuSUkWyEvpGIjp9ENX3yrcXD8G7QhnVIoepknRYaPaaCaDPlHJzF483mBNix1GZ2bTU3aCr2KgHDSUdcjn1kg==} - '@tangle-network/agent-knowledge@8.0.10': - resolution: {integrity: sha512-QOEIXt1t8iCjSNX7do6wqqUoZ8SQYjg5wIgX+Yi4877tnwEHZAQ41g6ibmtoYfPebBFtos2EU4jctJv7oK6T7w==} + '@tangle-network/agent-interface@1.6.0': + resolution: {integrity: sha512-DzdRAOTuPacezJlvOo+Xu5fWdoE9sJg90CDDo+SOCUJmfxrg38bEgCQz8ujr4r0Z5ABG2DlazMFRG/1/XoQ0HQ==} + + '@tangle-network/agent-knowledge@10.7.0': + resolution: {integrity: sha512-zBrDmwAZzvpGm3YzlVXnjrAPgG/bhGsLFl9FXJn2XjwmI4x5ziX9xrHVlHEnG+LCCjAkwu7dn1I+976H+v8/nA==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.149.0 <0.150.0' - '@tangle-network/agent-interface': ^1.1.0 + '@tangle-network/agent-eval': '>=0.163.2 <0.164.0' + '@tangle-network/agent-interface': ^1.4.0 - '@tangle-network/agent-profile-materialize@0.16.0': - resolution: {integrity: sha512-jNBkA91yTYtmdtWxqvGI9wgKsykp9fiRbWmgvlm7Kimi0Fjz7MiAXZnX1RW5yaqLSdPvzKVBOycVXjqk1/dSmQ==} + '@tangle-network/agent-profile-materialize@0.17.1': + resolution: {integrity: sha512-Nk/2PhkHndabvT53Wtrpd19V7btcJiJqvWcsUyRMSZQsjwAXDT/v3ipqhV4ysbyDRidAspadpDbo86enx/eExw==} peerDependencies: '@tangle-network/agent-interface': ^1.0.0 - '@tangle-network/agent-runtime@0.142.3': - resolution: {integrity: sha512-5Sm5dG+TunyY6N+CdNFPs4mYjT39wdtRMfXB8N7DjAnn2YGsepPFvwEGDItaFdABcppZ2aqxJqrm6dc7njBnzA==} + '@tangle-network/agent-runtime@0.175.0': + resolution: {integrity: sha512-sLEcour+zTxUolp5NRkDJRVRDBO4Fk5VYk+kPb/NqJDT8sX/yH/1Nmg/c1Wdnr5rcXUGtR5USR+knVzPJifrAw==} engines: {node: '>=22.13.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.149.0 <0.150.0' - '@tangle-network/agent-interface': ^1.1.0 - '@tangle-network/sandbox': '>=0.29.0 <0.31.0' + '@tangle-network/agent-eval': '>=0.163.2 <0.171.0' + '@tangle-network/agent-interface': ^1.6.0 + '@tangle-network/sandbox': '>=0.31.0 <0.32.0' peerDependenciesMeta: '@tangle-network/sandbox': optional: true @@ -1183,12 +1189,12 @@ snapshots: '@tangle-network/agent-interface': 1.3.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.149.0': + '@tangle-network/agent-eval@0.170.0': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.1.1(hono@4.13.3) '@tangle-network/agent-core': 0.9.4 - '@tangle-network/agent-interface': 1.3.0 + '@tangle-network/agent-interface': 1.6.0 '@tangle-network/agent-trace-contract': 1.0.2 hono: 4.13.3 linear-sum-assignment: 1.0.9 @@ -1203,25 +1209,31 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@8.0.10(@tangle-network/agent-eval@0.149.0)(@tangle-network/agent-interface@1.3.0)': + '@tangle-network/agent-interface@1.6.0': dependencies: - '@tangle-network/agent-eval': 0.149.0 - '@tangle-network/agent-interface': 1.3.0 + '@noble/hashes': 1.8.0 + spdx-expression-parse: 5.0.0 + zod: 4.4.3 + + '@tangle-network/agent-knowledge@10.7.0(@tangle-network/agent-eval@0.170.0)(@tangle-network/agent-interface@1.6.0)': + dependencies: + '@tangle-network/agent-eval': 0.170.0 + '@tangle-network/agent-interface': 1.6.0 '@types/proper-lockfile': 4.1.4 proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.16.0(@tangle-network/agent-interface@1.3.0)': + '@tangle-network/agent-profile-materialize@0.17.1(@tangle-network/agent-interface@1.6.0)': dependencies: - '@tangle-network/agent-interface': 1.3.0 + '@tangle-network/agent-interface': 1.6.0 - '@tangle-network/agent-runtime@0.142.3(@tangle-network/agent-eval@0.149.0)(@tangle-network/agent-interface@1.3.0)': + '@tangle-network/agent-runtime@0.175.0(@tangle-network/agent-eval@0.170.0)(@tangle-network/agent-interface@1.6.0)': dependencies: '@tangle-network/agent-core': 0.9.4 - '@tangle-network/agent-eval': 0.149.0 - '@tangle-network/agent-interface': 1.3.0 - '@tangle-network/agent-knowledge': 8.0.10(@tangle-network/agent-eval@0.149.0)(@tangle-network/agent-interface@1.3.0) - '@tangle-network/agent-profile-materialize': 0.16.0(@tangle-network/agent-interface@1.3.0) + '@tangle-network/agent-eval': 0.170.0 + '@tangle-network/agent-interface': 1.6.0 + '@tangle-network/agent-knowledge': 10.7.0(@tangle-network/agent-eval@0.170.0)(@tangle-network/agent-interface@1.6.0) + '@tangle-network/agent-profile-materialize': 0.17.1(@tangle-network/agent-interface@1.6.0) '@tangle-network/agent-trace-contract': 1.0.2 tar-stream: 3.2.0 transitivePeerDependencies: diff --git a/stream-driver.test.mts b/stream-driver.test.mts index 9ff635f..f32c888 100644 --- a/stream-driver.test.mts +++ b/stream-driver.test.mts @@ -37,6 +37,25 @@ function sandbox(options: { queueDepth: number; whenEmpty: 'noop' | 'repeat-last }) } +/** + * Read the health once the agent's fate is known. + * + * `running` means the driver has not yet been told what became of the child. + * Polling until it changes tests what the driver reports; sleeping a fixed + * interval tests how busy the machine was. + */ +async function settled( + driver: ReturnType, + timeoutMs = 5000, +): Promise> { + const deadline = Date.now() + timeoutMs + for (;;) { + const health = driver.health() + if (health.agent !== 'running' || Date.now() > deadline) return health + await new Promise((resolve) => setTimeout(resolve, 10)) + } +} + // ---- The game does not wait ------------------------------------------------ { @@ -216,11 +235,12 @@ function sandbox(options: { queueDepth: number; whenEmpty: 'noop' | 'repeat-last args: ['-e', 'console.error("the agent gave up"); process.exit(2)'], }) try { - for (let turn = 1; turn <= 3; turn += 1) { - await driver.act('f', [], context(turn)) - await new Promise((resolve) => setTimeout(resolve, 40)) - } - const health = driver.health() + for (let turn = 1; turn <= 3; turn += 1) await driver.act('f', [], context(turn)) + // Wait for the CONDITION, not for a duration. A fixed sleep asserts that + // the host delivered the child's exit event inside that window, which it + // does not owe anyone: measured flaky here, reporting `running` for a + // process that had already exited. + const health = await settled(driver) assert.equal(health.agent, 'failed') assert.match(health.agentDetail ?? '', /exited \(2\)/u, 'the exit status must be reported') assert.match(health.agentDetail ?? '', /the agent gave up/u, 'and what it printed, which is why it died') From f9650af8cf30d89dbbfe3376285b0b0ad7133ea3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 14:53:57 -0700 Subject: [PATCH 2/6] feat(examples): route a game through the optimizer that already has the statistics `compareOptimizationMethods` owns everything hard about comparing agent profiles: disjoint train, selection and final-test scenarios, bootstrap resamples, Bonferroni-adjusted simultaneous confidence across every contrast, a shared cost ceiling, and a policy that refuses to score a final test when the search history is incomplete. That machinery is the reason to route a game through it instead of tabulating means. On this repo's own three-profile study the ranking read cleanly as opus > sonnet > haiku; a paired bootstrap over the same replicates put two of the three contrasts across zero, and only opus over haiku survived. A table of means said one thing and an interval said another. Playproof supplies two things. A cell becomes a scenario, where `seedGroup` hands two surfaces the same seed for a replicate and makes the comparison paired rather than two unrelated samples of a noisy game. A played cell becomes a judge score, deterministic, because the game already produced the number. A blocked cell throws instead of scoring zero. A cell that could not be built did not play badly, and folding it into a zero would let an unbuildable clock read as a bad candidate. The surface reaches the game only through the profile, so none of this needs to know which of the ten searchable surfaces is being moved. --- examples/agent-eval-arena.mts | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 examples/agent-eval-arena.mts diff --git a/examples/agent-eval-arena.mts b/examples/agent-eval-arena.mts new file mode 100644 index 0000000..ef3c184 --- /dev/null +++ b/examples/agent-eval-arena.mts @@ -0,0 +1,105 @@ +/** + * Compare optimizers on a game, with the statistics the comparison needs. + * + * `compareOptimizationMethods` in `@tangle-network/agent-eval` already owns + * everything hard here: disjoint train, selection and final-test scenarios, + * bootstrap resamples, Bonferroni-adjusted simultaneous confidence across every + * contrast, a shared cost ceiling, and a policy that refuses to score a final + * test when the search history is incomplete. + * + * That machinery is the reason to route a game through it rather than tabulate + * means. MEASURED on this repo's own three-profile study: the ranking read + * cleanly as opus > sonnet > haiku, and a paired bootstrap over the same three + * replicates put two of the three contrasts across zero. Only opus over haiku + * survived. A table of means says one thing and an interval says another. + * + * Playproof supplies two things and nothing else: what a scenario IS, and what + * a played game SCORED. + * + * npx tsx examples/agent-eval-arena.mts + */ +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import { runCell } from '../matrix-run' +import type { CellResult } from '../matrix-run' +import type { MatrixCell } from '../matrix' + +/** A cell of a matrix, addressed the way the optimizer addresses scenarios. */ +export interface GameScenario extends Scenario { + kind: 'playproof-game' + cell: MatrixCell +} + +/** + * One cell becomes one scenario. + * + * `seedGroup` is the whole reason to bother: two surfaces carrying the same + * value are handed the same seed for a given replicate, which makes the + * comparison PAIRED. Without it two profiles are two unrelated samples of a + * noisy game, and the interval has to absorb the game's variance twice. + */ +export function scenarioOf(cell: MatrixCell): GameScenario { + return { + id: `${cell.game.id}/${cell.objective.id}/${cell.protocol.id}/${cell.sensor.id}/seed${cell.seed}`, + kind: 'playproof-game', + tags: [cell.game.adapter, cell.game.target, cell.profile.transport], + seedGroup: `${cell.game.id}:${cell.seed}`, + cell, + } +} + +/** + * Play a scenario and hand back the row. + * + * The surface is whatever the optimizer is searching — a prompt, a skill set, a + * tool list, a policy file. It reaches the game only through the profile, so + * nothing here needs to know which of the ten surfaces is being moved. + */ +export function dispatchWithSurface( + materialize: (surface: unknown, cell: MatrixCell) => MatrixCell, +): (surface: unknown, scenario: GameScenario, ctx: DispatchContext) => Promise { + return async (surface, scenario, ctx) => + runCell(materialize(surface, scenario.cell), { signal: ctx.signal }) +} + +/** + * Score a played game. + * + * A judge is a FUNCTION, not an LLM prompt, and this one is deterministic: the + * game already produced the number. Composite is the score the objective named, + * oriented so higher is better, because the optimizer maximizes. + * + * A blocked cell THROWS. A cell that could not be built did not play badly, and + * folding it into a zero would let an unbuildable clock look like a bad + * candidate — which is the one thing this repo refuses everywhere else. + */ +export const gameJudge: JudgeConfig = { + name: 'playproof-score', + judgeVersion: 'score-v1', + dimensions: [ + { key: 'score', description: 'the evidence channel the objective names, oriented so higher is better' }, + { key: 'verified', description: 'milestones the replay reproduced, over the contract total' }, + ], + score: ({ artifact }) => { + if (artifact.status !== 'played') { + throw new Error(`cell did not play: ${artifact.blocked?.reason} — ${artifact.blocked?.detail}`) + } + if (artifact.replayDivergence) { + throw new Error('the run did not reproduce when replayed from its own input log') + } + const raw = artifact.score ?? 0 + const oriented = artifact.scoreDirection === 'minimize' ? -raw : raw + const total = artifact.milestones.total + return { + dimensions: { + score: oriented, + verified: total === 0 ? 0 : artifact.milestones.verified / total, + }, + composite: oriented, + notes: `${artifact.scoreField ?? 'score'}=${artifact.score ?? 'none'}` + + ` over ${artifact.decisions} decisions;` + + ` ${artifact.milestones.verified} of ${total} milestones;` + + ` ${artifact.usd === null ? 'unmetered' : `$${artifact.usd.toFixed(4)}`}` + + `${artifact.transportNote === null ? '' : `; ${artifact.transportNote}`}`, + } + }, +} From b9edfe747154815c1a7b0bb48cc5ce06dfba88c2 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 14:57:59 -0700 Subject: [PATCH 3/6] feat(matrix): record which credential path an agent billed against A cost of null could mean two different things and printed the same either way: the agent reported nothing, or the agent bills a plan and has no per-request figure to report. A study that reads those the same ranks the arm nobody could meter first on cost per point. Reuse the vocabulary the rest of the stack already has rather than inventing a second one. `authMode: 'api-key' | 'oauth'` is the union in `@tangle-network/agent-interface`, and the CLI agent registry passes the same value to a spawned backend as `CLAUDE_CODE_AUTH_MODE`. Both names are used here, so a run in this harness and a run in a sandbox describe themselves identically. The launcher honours an explicit `CLAUDE_CODE_AUTH_MODE`, and otherwise reads `api-key` when `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` is set and `oauth` when neither is. Verified on all three paths. The value travels with the meter to the row, and the runner prints an absent cost as its reason: `unbilled(oauth)` rather than a bare blank, and never `$0.0000`. --- drivers/stream-sandbox.ts | 21 +++++++++++++++++---- harnesses/author-policy | 23 +++++++++++++++++++++++ harnesses/claude-code | 23 +++++++++++++++++++++++ matrix-run.ts | 25 +++++++++++++++++++++---- matrix.mts | 6 +++++- 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/drivers/stream-sandbox.ts b/drivers/stream-sandbox.ts index d1ea78f..e7ec70c 100644 --- a/drivers/stream-sandbox.ts +++ b/drivers/stream-sandbox.ts @@ -134,6 +134,18 @@ export interface StreamHealth { costUsd: number | null /** Tokens the agent says it has spent, or null when it did not say. */ tokens: number | null + /** + * Which credential path the agent used, as its launcher reported it. + * + * `'api-key' | 'oauth'` is the union the rest of the stack uses, and the CLI + * agent registry passes the same value to a spawned backend as + * `CLAUDE_CODE_AUTH_MODE`. Null when the launcher did not say. + * + * It decides whether a dollar figure can exist. Under `oauth` the agent bills + * against a plan and reports no per-request cost, so a null cost means + * UNBILLED THIS WAY rather than free. + */ + authMode: 'api-key' | 'oauth' | null } export interface StreamSandboxDriverOptions { @@ -385,15 +397,16 @@ export function createStreamSandboxDriver(options: StreamSandboxDriverOptions): * Written when the agent exits, so it is absent for a cell read while the * agent still runs. Absence is reported as absence. */ - function reportedMeter(): { costUsd: number | null; tokens: number | null } { + function reportedMeter(): { costUsd: number | null; tokens: number | null; authMode: 'api-key' | 'oauth' | null } { const nonNegative = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null try { const parsed: unknown = JSON.parse(readFileSync(join(dir, 'agent-cost.json'), 'utf8')) - const meter = parsed as { usd?: unknown; tokens?: unknown } - return { costUsd: nonNegative(meter.usd), tokens: nonNegative(meter.tokens) } + const meter = parsed as { usd?: unknown; tokens?: unknown; authMode?: unknown } + const mode = meter.authMode === 'api-key' || meter.authMode === 'oauth' ? meter.authMode : null + return { costUsd: nonNegative(meter.usd), tokens: nonNegative(meter.tokens), authMode: mode } } catch { - return { costUsd: null, tokens: null } + return { costUsd: null, tokens: null, authMode: null } } } diff --git a/harnesses/author-policy b/harnesses/author-policy index edadc29..2e9a94d 100755 --- a/harnesses/author-policy +++ b/harnesses/author-policy @@ -149,6 +149,28 @@ if (jail === 'off') { process.exit(3) } +/** + * Which credential path the agent used, in the vocabulary the rest of the + * stack already uses. + * + * `authMode: 'api-key' | 'oauth'` is the union in + * `@tangle-network/agent-interface`, and the CLI agent registry already passes + * it to a spawned backend as `CLAUDE_CODE_AUTH_MODE`. Both names are reused + * here so a run in this harness and a run in a sandbox describe themselves the + * same way. + * + * It belongs on the record because it decides whether a dollar figure exists at + * all. Under `oauth` the agent bills against a plan and reports no per-request + * cost, so a null there means UNBILLED THIS WAY, not free. A study that reads + * those two the same ranks the arm nobody could meter first on cost. + */ +function authMode() { + const declared = process.env.CLAUDE_CODE_AUTH_MODE + if (declared === 'api-key' || declared === 'oauth') return declared + const key = process.env.ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_AUTH_TOKEN + return key !== undefined && key !== '' ? 'api-key' : 'oauth' +} + const costPath = join(sandbox, 'agent-cost.json') const transcriptPath = join(sandbox, 'agent-transcript.jsonl') let inputTokens = 0 @@ -168,6 +190,7 @@ function publish() { turns, vendor: bin, model: model ?? null, + authMode: authMode(), })}\n`, ) } catch {} diff --git a/harnesses/claude-code b/harnesses/claude-code index 2d0c33b..9d88b2f 100755 --- a/harnesses/claude-code +++ b/harnesses/claude-code @@ -151,6 +151,28 @@ if (jail === 'off') { process.exit(3) } +/** + * Which credential path the agent used, in the vocabulary the rest of the + * stack already uses. + * + * `authMode: 'api-key' | 'oauth'` is the union in + * `@tangle-network/agent-interface`, and the CLI agent registry already passes + * it to a spawned backend as `CLAUDE_CODE_AUTH_MODE`. Both names are reused + * here so a run in this harness and a run in a sandbox describe themselves the + * same way. + * + * It belongs on the record because it decides whether a dollar figure exists at + * all. Under `oauth` the agent bills against a plan and reports no per-request + * cost, so a null there means UNBILLED THIS WAY, not free. A study that reads + * those two the same ranks the arm nobody could meter first on cost. + */ +function authMode() { + const declared = process.env.CLAUDE_CODE_AUTH_MODE + if (declared === 'api-key' || declared === 'oauth') return declared + const key = process.env.ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_AUTH_TOKEN + return key !== undefined && key !== '' ? 'api-key' : 'oauth' +} + const costPath = join(sandbox, 'agent-cost.json') const transcriptPath = join(sandbox, 'agent-transcript.jsonl') @@ -173,6 +195,7 @@ function publish() { turns, vendor: bin, model: model ?? null, + authMode: authMode(), })}\n`, ) } catch {} diff --git a/matrix-run.ts b/matrix-run.ts index db8b6d5..e7ac840 100644 --- a/matrix-run.ts +++ b/matrix-run.ts @@ -145,6 +145,15 @@ export interface CellResult { scoreField: string | null /** Which way is better on that channel. A minimize goal ranks inverted. */ scoreDirection: 'maximize' | 'minimize' | null + /** + * Credential path the agent billed against, or null when nothing said. + * + * Reported beside `usd` because it decides whether that number can exist. An + * `oauth` arm bills a plan and reports no per-request cost, so its null is + * UNBILLED THIS WAY, not free, and a study that compares it to a metered arm + * on cost is comparing a measurement to an absence. + */ + authMode: 'api-key' | 'oauth' | null /** Whether the game declared itself finished. Null when it declares no end. */ cleared: boolean | null /** Milestones the replay reproduced. */ @@ -396,6 +405,7 @@ function blockedResult(cell: MatrixCell, reason: BlockedReason, detail: string, // see that an excluded row and a played row were aimed at the same channel. scoreField: null, scoreDirection: null, + authMode: null, cleared: null, verified: [], milestones: { verified: 0, total: 0 }, @@ -550,6 +560,7 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P // same ranks the profile nobody metered first on cost per point. tokens: meter.tokens, usd: meter.metered && meter.costUsd === null ? null : (meter.costUsd ?? 0) + record.spentUsd, + authMode: meter.authMode, build: authored, cleared: record.gameOver, verified: record.verified, @@ -743,18 +754,24 @@ async function authorPolicy( } } -function meterOf(driver: AgentDriver): { metered: boolean; costUsd: number | null; tokens: number | null } { +function meterOf(driver: AgentDriver): { + metered: boolean + costUsd: number | null + tokens: number | null + authMode: 'api-key' | 'oauth' | null +} { const reporter = driver as { health?: () => Record } - if (typeof reporter.health !== 'function') return { metered: false, costUsd: null, tokens: null } + if (typeof reporter.health !== 'function') return { metered: false, costUsd: null, tokens: null, authMode: null } const health = reporter.health() // `metered` asks whether this transport HAS a cost channel, which is a // different question from whether anything came down it. A per-decision // control that declares `fixedCostUsd: 0` is genuinely free and must total // zero; only a transport that owns a meter can report the absence of one. - if (!('costUsd' in health)) return { metered: false, costUsd: null, tokens: null } + if (!('costUsd' in health)) return { metered: false, costUsd: null, tokens: null, authMode: null } const nonNegative = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null - return { metered: true, costUsd: nonNegative(health.costUsd), tokens: nonNegative(health.tokens) } + const mode = health.authMode === 'api-key' || health.authMode === 'oauth' ? health.authMode : null + return { metered: true, costUsd: nonNegative(health.costUsd), tokens: nonNegative(health.tokens), authMode: mode } } function transportNoteOf(driver: AgentDriver): string | null { diff --git a/matrix.mts b/matrix.mts index ff38a24..cf8197f 100644 --- a/matrix.mts +++ b/matrix.mts @@ -69,7 +69,11 @@ for (const [index, cell] of cells.entries()) { rows.push(row) const headline = row.status === 'played' ? `score=${row.score ?? '-'} deaths=${row.deaths ?? '-'} decisions=${row.decisions}` - + ` tokens=${row.tokens ?? 'unmetered'} usd=${row.usd === null ? 'unmetered' : row.usd.toFixed(4)}` + + ` tokens=${row.tokens ?? 'unmetered'}` + // An absent cost reads as its REASON. `oauth` bills a plan and reports no + // per-request figure, so "unbilled(oauth)" and "$0.0000" are different + // facts and must not print the same. + + ` usd=${row.usd === null ? `unbilled(${row.authMode ?? 'unknown'})` : row.usd.toFixed(4)}` : `BLOCKED (${row.blocked?.reason}): ${row.blocked?.detail}` const note = row.transportNote === null ? '' : ` [${row.transportNote}]` console.error(`[${index + 1}/${cells.length}] ${row.name} ${headline}${note}`) From 1c0a57a352f70b25eb55a4179fb3080b62bdd77e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 15:06:33 -0700 Subject: [PATCH 4/6] docs: rewrite the em-dashes that were prose, and leave the ones that were not An earlier count said 41 em-dashes across docs/. Most were markdown table cells using a dash as a value separator, and one is verbatim CI output inside a fence. Rewriting those would have damaged a table and falsified a quote. 20 were prose, and each is rewritten by hand rather than substituted. A parenthetical becomes brackets, an appositive becomes a colon, and a clause that was doing a full sentence's work becomes one. Also fixes a sentence in the RetroArch gate section whose subject drifted: the blind search watched PyBoy's work RAM to find the reference's channel addresses, and that fact was nested inside a clause about replaying through gambatte. --- docs/adapters.md | 18 +++++++++--------- docs/arena.md | 14 +++++++------- docs/calibration.md | 4 ++-- docs/observation.md | 2 +- examples/author-2048.matrix | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/adapters.md b/docs/adapters.md index d4f83ca..10b4dbc 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -65,7 +65,7 @@ It is off by default on every adapter, so a run that does not ask for it sends t | `platforms/steam`, `platforms/xbox` | No | The title runs elsewhere; these adapters provide no observation at all | — | **No image dependency was added, and none may be.** -Pillow is absent from the CI environments — the PyBoy job logs `Missing dependency "Pillow"` — and a harness that grows an imaging stack to show a game screen has overpaid. +Pillow is absent from the CI environments; the PyBoy job logs `Missing dependency "Pillow"`. A harness that grows an imaging stack to show a game screen has overpaid. `pyshared/playproof_png.py` encodes 8-bit grayscale, RGB, or RGBA with `zlib` and `struct`, filter 0 on every scanline. Measured on an ale-py 0.12.1 Breakout frame: 518 bytes at the native 160x210 in 0.84 ms, and 2,383 bytes at a 3x upscale in 4.4 ms. @@ -83,7 +83,7 @@ Its environments here observe a vector or an `ansi` string rather than a framebu That is a new dependency for a picture of a cart and a pole, so the adapter does without and says so. Each gate proves the identity rather than the plumbing: it undoes the whole-pixel upscale, hashes the recovered native buffer, and asserts it equals the `frameHash` a verifier recomputes. -Measured on CI hardware — ALE Breakout at 3x: a 480x630 PNG of 2,488 bytes with 9 distinct colours; stable-retro Airstriker at 2x: 640x448, 1,968 bytes, 9 colours; PyBoy Libbet at 3x: 480x432, 1,228 bytes, 1 colour, because Libbet under the blind generic preamble draws an all-white screen from about the fortieth reference input onward. +Measured on CI hardware. ALE Breakout at 3x: a 480x630 PNG of 2,488 bytes with 9 distinct colours; stable-retro Airstriker at 2x: 640x448, 1,968 bytes, 9 colours; PyBoy Libbet at 3x: 480x432, 1,228 bytes, 1 colour, because Libbet under the blind generic preamble draws an all-white screen from about the fortieth reference input onward. **The pixels are observation, never evidence.** They do not enter the input log, the contract, or the attestation, and an adapter that put privileged state into an image caption would be breaking the same boundary that already forbids putting it into the frame text. @@ -275,7 +275,7 @@ The gate in `ale.test.mts` decodes the produced PNG, checks the dimensions and t **Contracts.** A reference file declares the trigger for each milestone. `deriveContract` replays the reference and samples the value or hash that actually held at that instant. No threshold or hash is written by hand. -The bundled Breakout contract is seven rungs of one progression — `score >= 1, 2, 4, 8, 18, 32, 64` — so a run that broke 24 bricks and one that broke 64 do not score the same. +The bundled Breakout contract is seven rungs of one progression (`score >= 1, 2, 4, 8, 18, 32, 64`), so a run that broke 24 bricks and one that broke 64 do not score the same. **ROMs.** `ale-py` bundles the Atari ROM set, so this adapter needs no download and no secret. The bundled reference plays Breakout and reaches a score of 64 over 839 inputs, which opens all seven rungs of the packaged ladder. @@ -311,7 +311,7 @@ That is a new dependency for a picture of a cart and a pole, so this adapter pub A toy-text `ansi` render marks the agent's own cell with a terminal colour escape and nothing else, so the worker converts the highlight to brackets before it strips the escapes; dropping them outright would delete the agent's position from the frame. **Evidence.** `engineState` carries `cumulativeReward`, `steps`, `terminated`, `truncated`, and the numeric entries of the environment's `info` dictionary, bounded to the first 16. -`frameHash` is the SHA-256 of the whole observation — over its dtype, shape, and raw bytes for a dense array, and over its canonical JSON otherwise — and `frameState` carries the first 8 observation components. +`frameHash` is the SHA-256 of the whole observation, taken over its dtype, shape and raw bytes for a dense array, and over its canonical JSON otherwise. `frameState` carries the first 8 observation components. Playproof evidence is integer-only, so reward, numeric `info` entries, and observation components are multiplied by 1000 and rounded. Reward 1.0 is `cumulativeReward` 1000. `steps`, `terminated`, and `truncated` are counts and flags and are never scaled. @@ -328,14 +328,14 @@ What is lost is the ability to surprise the agent with a progress signal it coul `gymnasium.test.mts` measures both across separate worker processes rather than assuming them. An environment that reads a clock, a global RNG, or external state is not replay-verifiable and must not be given a contract. -**Checkpoints.** Gymnasium exposes no generic state API, so the general checkpoint is `{seed, inputs}` and restore is `reset(seed)` followed by a replay — exact for the environments above, and the only sound answer for the rest. +**Checkpoints.** Gymnasium exposes no generic state API, so the general checkpoint is `{seed, inputs}` and restore is `reset(seed)` followed by a replay. That is exact for the environments above, and the only sound answer for the rest. Where the environment keeps its position in one readable attribute (`state` for classic control, `s` for toy text), the worker also writes a JSON copy of that attribute, and restore puts it back directly instead of replaying. `pickle` is never used: a checkpoint stays a plain protocol value a verifier can read. The fast path is exact for environments whose `step` is a deterministic function of that attribute; `GymRpc.snapshot('replay')` forces the general path, and the gate proves both. **Contracts.** A reference file declares the trigger for each milestone; `deriveContract` replays the reference and samples the value or hash that actually held at that instant. No threshold or hash is written by hand. -Playproof ships two reference playthroughs — a scripted balancing run on `CartPole-v1` and the shortest winning path on the `FrozenLake-v1` 4x4 map. +Playproof ships two reference playthroughs: a scripted balancing run on `CartPole-v1`, and the shortest winning path on the `FrozenLake-v1` 4x4 map. Both environments are part of Gymnasium itself, so the adapter and its gate run on a clean CI machine with no asset: ```bash @@ -376,7 +376,7 @@ RetroArch is not an API, so each of these is a measurement against the real bina ### Determinism -Libretro cores take no seed, so `init(seed)` cannot rebuild a run the way a seeded environment can. Instead the worker pins a boot state — pause, `RESET`, `bootFrames` fixed advances, save state — and `init` restores it. Every later transition is an explicit, counted frame advance from that state, so the input log plus the boot state is the complete determinism key. The seed is recorded and reported so run artifacts keep one shape, but it is nominal. +Libretro cores take no seed, so `init(seed)` cannot rebuild a run the way a seeded environment can. Instead the worker pins a boot state (pause, `RESET`, `bootFrames` fixed advances, save state), and `init` restores it. Every later transition is an explicit, counted frame advance from that state, so the input log plus the boot state is the complete determinism key. The seed is recorded and reported so run artifacts keep one shape, but it is nominal. The result of that procedure is saved once, and every reset restores the save. Re-running the reset instead is not equivalent: a core reset does not clear video memory or the picture-processing state, so a second reset lands the title-screen animation at a phase that depends on the run before it. Measured over 41 evidence snapshots between two separately launched emulators, re-running the reset reproduced work RAM 40 times and the screen twice, while restoring the pinned save reproduced work RAM every time. @@ -431,7 +431,7 @@ macOS is not a supported host for this adapter. See the measured-facts table abo ### The cross-emulator proof -The gate does not merely run a Game Boy game. It replays the 266-input reference from `pyboy/discovery-libbet.json` — whose channel addresses a blind search found by watching **PyBoy's** work RAM — through RetroArch and gambatte, software that shares no code with PyBoy. `channelsFromDiscovery` converts the discovered addresses into RetroArch channels, so one discovery document drives two unrelated emulators and neither adapter carries a hand-copied address. +The gate does not merely run a Game Boy game. It replays the 266-input reference from `pyboy/discovery-libbet.json` through RetroArch and gambatte, software that shares no code with PyBoy. A blind search found that reference's channel addresses by watching **PyBoy's** work RAM. `channelsFromDiscovery` converts the discovered addresses into RetroArch channels, so one discovery document drives two unrelated emulators and neither adapter carries a hand-copied address. The hard assertion is the milestone outcome: the contract derived over those channels verifies clean through RetroArch, and a script of the same length that never presses a button is rejected. Per-step channel agreement with PyBoy's own recorded values is reported rather than required to be exact, because two emulators put frame boundaries in different places and a channel that samples an animation disagrees on the steps around each transition. @@ -486,7 +486,7 @@ Platform milestones are evaluated as **baseline-to-final transitions**. An achie Ordered by how much reach each one buys per unit of work. RetroArch as a black-box host was the first entry here and is now shipped; see [Any RetroArch core](#any-retroarch-core-through-the-black-box-host) above. -**Direct libretro core loader over `ctypes`.** stable-retro compiles a fixed set of cores into its own binary. Loading `libretro.so` cores directly through the C ABI turns the ceiling into "any core that exists": N64 through Mupen64Plus, DS through melonDS, PS1 through Beetle PSX or PCSX-ReARMed, PSP through PPSSPP, 3DS, arcade through MAME or FinalBurn Neo, DOS through DOSBox, and adventure games through ScummVM. The libretro ABI already exposes exactly what Playproof needs — `retro_run`, `retro_serialize`, `retro_unserialize`, `retro_get_memory_data`, and a fixed input descriptor — so this should be **one** worker with a per-core manifest declaring the memory map, the button layout, and the save-state stability the core actually offers. Each new core becomes a data file, not code. The determinism question above must be answered per core: several of these are known to be non-reproducible across processes and would honestly be `trusted-recorder`. +**Direct libretro core loader over `ctypes`.** stable-retro compiles a fixed set of cores into its own binary. Loading `libretro.so` cores directly through the C ABI turns the ceiling into "any core that exists": N64 through Mupen64Plus, DS through melonDS, PS1 through Beetle PSX or PCSX-ReARMed, PSP through PPSSPP, 3DS, arcade through MAME or FinalBurn Neo, DOS through DOSBox, and adventure games through ScummVM. The libretro ABI already exposes exactly what Playproof needs: `retro_run`, `retro_serialize`, `retro_unserialize`, `retro_get_memory_data`, and a fixed input descriptor. So this should be **one** worker with a per-core manifest declaring the memory map, the button layout, and the save-state stability the core actually offers. Each new core becomes a data file, not code. The determinism question above must be answered per core: several of these are known to be non-reproducible across processes and would honestly be `trusted-recorder`. **Dolphin (GameCube and Wii).** Reachable through the scripting fork's Lua and Python bindings, which expose memory reads and save states. High value because it opens a console generation nothing else here covers, and high cost because its determinism story is weak and it would likely declare `trusted-recorder`. diff --git a/docs/arena.md b/docs/arena.md index 4669853..bcd6daf 100644 --- a/docs/arena.md +++ b/docs/arena.md @@ -15,7 +15,7 @@ means. ### Add a game A game is one line naming the adapter that boots it and the target that adapter -takes — an `ale-py` ROM id, a Gymnasium environment id, a stable-retro game. +takes: an `ale-py` ROM id, a Gymnasium environment id, a stable-retro game. ``` game.breakout adapter=ale target=breakout @@ -53,15 +53,15 @@ axis rather than a fixed cost: | `persistent` (default) | one child process per episode, request/response over stdio | 0.97 ms a decision, 38x less; state survives | | `stream` | the game writes observations into a sandbox directory and never waits; the agent appends actions to a file | the agent's thinking rate is decoupled from the frame rate | -`stream` is the asynchronous one. The agent reads when it likes — with as many -subagents and analysis scripts as it wants — and the game consumes one queued +`stream` is the asynchronous one. The agent reads when it likes, with as many +subagents and analysis scripts as it wants, and the game consumes one queued action per decision. It holds no emulator handle, so there is no object graph to isolate: the boundary is the filesystem. ### Define a matrix ``` -# study.matrix — two arms, two games, one protocol, one sensor +# study.matrix: two arms, two games, one protocol, one sensor profile.opus harness=claude-code model=claude-opus-5 effort=high profile.chaser harness=none policy=./policies/chaser @@ -89,7 +89,7 @@ are different games. **The sensor states what the agent may see.** `pixels=on scale=N` publishes the rendered screen next to the text frame. `channels=ball_x@99,paddle_x@72` names -RAM bytes — a *harness* channel that reaches milestones and the result vector, +RAM bytes. That is a *harness* channel, which reaches milestones and the result vector, never the agent, because routing it to the agent would cross the one boundary playproof exists to hold. @@ -112,7 +112,7 @@ tsx matrix.mts study.matrix --out runs/study/cells.json Each cell prints a line as it finishes, and the artifact holds every row plus the summary. A cell whose game cannot be built under its own protocol is a -**blocked** row carrying the reason — never a zero, because a profile that never +**blocked** row carrying the reason, never a zero, because a profile that never played did not lose. That case is real, not defensive. The bundled Breakout reference is recorded at @@ -242,7 +242,7 @@ backend. `RunCellOptions.driver` is the seam. See ### The cross-game statistic `generalization(rows)` is the number a matrix exists to produce: mean pairwise -Kendall tau-b between per-game rankings. It reports its own limits beside it — +Kendall tau-b between per-game rankings. It reports its own limits beside it: `folds` (games, not pairs), `effectiveArms` against `declaredArms`, and every excluded cell. diff --git a/docs/calibration.md b/docs/calibration.md index b2be13e..cbdf79e 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -104,7 +104,7 @@ It did not always. Against a six-milestone contract whose top achievement was `s Declare an attrition milestone to keep it: `assertContractSeparates(report, { attritionChecks: ['life-lost'] })`. Excluding a point for dying was necessary and it was not sufficient. -A contract whose top rung is `score >= 4` cannot tell 7 from 24 however it scores, so the packaged Breakout ladder now doubles — `score >= 1, 2, 4, 8, 18, 32, 64` — over a reference that reaches 64. +A contract whose top rung is `score >= 4` cannot tell 7 from 24 however it scores, so the packaged Breakout ladder now doubles (`score >= 1, 2, 4, 8, 18, 32, 64`) over a reference that reaches 64. The strongest control reaches 5 of those 7, so the ladder still has rungs above the best program anyone has written for it. ### A contract that grades on one event @@ -148,7 +148,7 @@ packaged.report.separates // the verdict travels with the contract A contract with nothing to declare passes with no `declare` at all. The packaged ALE Breakout contract is calibrated that way in `ale.test.mts`: seven legible achievement rungs, no hash, no attrition milestone, and no prerequisite the whole contract hangs off. -A target that is not meant to separate — a tier demonstration, a smoke fixture — says so in words, and the declaration is refused when it goes stale: +A target that is not meant to separate, such as a tier demonstration or a smoke fixture, says so in words, and the declaration is refused when it goes stale: ```ts PackagedContract.calibrate(game, contract, { diff --git a/docs/observation.md b/docs/observation.md index df1a62b..d0a2464 100644 --- a/docs/observation.md +++ b/docs/observation.md @@ -29,7 +29,7 @@ The emulator adapters were already capturing the screen. `ale/worker.py` calls `getScreenRGB()`, hashes those pixels into `frameHash` for verification, and used to throw the picture away; the agent received a luminance-to-ASCII downsample of it. That is a perception limit the harness created, not a result about the agent. -Measured on ALE Breakout, against the six-milestone contract of the time: `stealth/ox-alpha`, a `text+image->text` model, and `liquid/lfm-2.5-2.6b:free` both scored 0 of 6 milestones, and their own transcripts show them reading the ASCII as a maze — "exploring the map", "positioned near the goal area" — rather than a paddle-and-ball game. +Measured on ALE Breakout, against the six-milestone contract of the time: `stealth/ox-alpha`, a `text+image->text` model, and `liquid/lfm-2.5-2.6b:free` both scored 0 of 6 milestones, and their own transcripts show them reading the ASCII as a maze rather than a paddle-and-ball game, with phrases like "exploring the map" and "positioned near the goal area". One of them pressed `FIRE` twice in 45 turns, so no ball was ever in play. ### Bounds diff --git a/examples/author-2048.matrix b/examples/author-2048.matrix index b6dfd60..a191467 100644 --- a/examples/author-2048.matrix +++ b/examples/author-2048.matrix @@ -13,9 +13,9 @@ game.puzzle adapter=native-2048 target=2048 # A real horizon. A program does not need the clock a typist needs, so the # episode can be as long as the game deserves. -objective.score goal=maximize:score horizon=400 budgetUsd=4 +objective.score goal=maximize:score horizon=2000 budgetUsd=4 protocol.det frameskip=1 sticky=0 seeds=1 queue=8 empty=noop pace=0 sensor.ascii pixels=off channels=- -reps 1 +reps 2 From 7a1f0e41b19091d9fc4933c8149b10494084cfd8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 15:17:10 -0700 Subject: [PATCH 5/6] fix(retroarch): a frame-advance barrier that fails must say so `gap()` sends one run-loop iteration between frame advances. That is a barrier, not a courtesy: `FRAMEADVANCE` returns as soon as RetroArch accepts it, so without an intervening iteration the next advance can be issued while the previous frame is still settling. `command` returns None when it gives up, and `gap` discarded that return. A barrier that did not happen looked exactly like one that did. This is a diagnostic as much as a fix. The same-process replay gate has failed four times today, and its divergence has a specific shape: two replays of one boot state and one input log agree byte for byte to emuFrame 811, then differ on channel values at IDENTICAL frame numbers. Identical frame numbers rule out a miscounted advance and point at a run loop that had not settled. It is NOT proven that a missed barrier causes it. `COMMAND_TIMEOUT` is 5 seconds with a resend loop, so a timeout there means RetroArch is wedged rather than busy, which should be rare. The fix decides the question: a gate that now fails with "the run loop was not synchronised" was this, and one that still diverges silently is something else. `save_state` and `load_state` also discard a command return, and are left alone deliberately. Both verify the outcome downstream, by waiting for the state file and by reading the log, which is a stronger check than the return. --- retroarch/worker.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/retroarch/worker.py b/retroarch/worker.py index 861bbb3..c52a270 100644 --- a/retroarch/worker.py +++ b/retroarch/worker.py @@ -603,8 +603,28 @@ def pause(self): raise RetroArchError('RetroArch never reported PAUSED') def gap(self): - """One run loop iteration with no frame advance.""" - self.command(GAP_MSG) + """One run loop iteration with no frame advance. + + This is a BARRIER, not a courtesy. `FRAMEADVANCE` returns as soon as + RetroArch accepts it, so without an intervening run-loop iteration the + next advance can be issued while the previous frame is still settling. + + `command` returns None when it gives up, and this return used to be + discarded, so a barrier that did not happen looked exactly like one + that did. MEASURED: two same-process replays of the same boot state and + the same inputs agreed byte for byte to emuFrame 811 and then differed + on the channel values at IDENTICAL frame numbers, which is a run loop + that did not settle rather than a frame that was miscounted. + + A missed barrier is now a failure with a name, because a replay that + diverges silently is worse than one that stops. + """ + if self.command(GAP_MSG) is None: + self._alive() + raise RetroArchError( + 'RetroArch stopped answering between frame advances, so the run loop was not' + ' synchronised and a replay of this run would not reproduce it.%s' + % self._stall_hint()) def advance(self, frames): for _ in range(frames): From e8b228f9e5e80fb4977c897026cf0e5ec160ff06 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 26 Aug 2026 15:59:56 -0700 Subject: [PATCH 6/6] fix(matrix): report what building a player cost, and who paid for it The `authMode` column read empty on every authored cell. The row's `authMode` describes whoever played, and an authored cell is played by a program with no credentials at all, so that field was correctly null while the build behind it was very much billed to something. The authoring phase already read the value and then dropped it. It now travels in the `build` column, beside the tokens and minutes that were spent alongside it, and the runner prints the build cost on the cell line: minutes, tokens, and either a dollar figure or `unbilled(oauth)`. Measured on the six-cell study this fixes: sonnet spent 5.14M tokens building a player that lost the game after 317 decisions, while opus spent 1.19M on one that survived all 2000. Build cost and play score answer different questions and a single number cannot carry both. --- matrix-run.ts | 43 +++++++++++++++++++++++++++++++++++++++---- matrix.mts | 7 +++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/matrix-run.ts b/matrix-run.ts index e7ac840..16f766f 100644 --- a/matrix-run.ts +++ b/matrix-run.ts @@ -141,7 +141,21 @@ export interface CellResult { * Merging them would answer neither question: the point of authoring is that * building is a cost you choose to pay and playing is what gets scored. */ - build: { usd: number | null; tokens: number | null; minutes: number; policy: string | null } | null + build: { + usd: number | null + tokens: number | null + minutes: number + policy: string | null + /** + * Credential path the AUTHOR billed against. + * + * It belongs here rather than on the row. The row's `authMode` describes + * whoever played, and an authored cell is played by a program with no + * credentials at all, so that field is correctly null while the build was + * very much billed to something. + */ + authMode: 'api-key' | 'oauth' | null + } | null scoreField: string | null /** Which way is better on that channel. A minimize goal ranks inverted. */ scoreDirection: 'maximize' | 'minimize' | null @@ -479,7 +493,13 @@ export async function runCell(cell: MatrixCell, options: RunCellOptions = {}): P let policyCommand: string | null = null if (cell.profile.author !== undefined) { const attempt = await authorPolicy(cell, options, now) - authored = { usd: attempt.usd, tokens: attempt.tokens, minutes: attempt.minutes, policy: attempt.policy } + authored = { + usd: attempt.usd, + tokens: attempt.tokens, + minutes: attempt.minutes, + policy: attempt.policy, + authMode: attempt.authMode, + } if (attempt.policy === null) { // Blocked, never scored zero. A profile that built nothing did not play // badly; it produced no player, and those are different findings. @@ -658,7 +678,14 @@ async function authorPolicy( cell: MatrixCell, options: RunCellOptions, now: () => number, -): Promise<{ usd: number | null; tokens: number | null; minutes: number; policy: string | null; detail: string | null }> { +): Promise<{ + usd: number | null + tokens: number | null + minutes: number + policy: string | null + authMode: 'api-key' | 'oauth' | null + detail: string | null +}> { const profile = cell.profile const started = now() const dir = join( @@ -676,7 +703,14 @@ async function authorPolicy( cell.seed + PRACTICE_SEED_OFFSET, ) } catch (error) { - return { usd: null, tokens: null, minutes: 0, policy: null, detail: `no practice game: ${(error as Error).message}` } + return { + usd: null, + tokens: null, + minutes: 0, + policy: null, + authMode: null, + detail: `no practice game: ${(error as Error).message}`, + } } // Paced like a live game so the practice feels like the real thing. @@ -748,6 +782,7 @@ async function authorPolicy( return { usd: meter.costUsd, tokens: meter.tokens, + authMode: meter.authMode, minutes: (now() - started) / 60_000, policy, detail, diff --git a/matrix.mts b/matrix.mts index cf8197f..8664575 100644 --- a/matrix.mts +++ b/matrix.mts @@ -69,6 +69,13 @@ for (const [index, cell] of cells.entries()) { rows.push(row) const headline = row.status === 'played' ? `score=${row.score ?? '-'} deaths=${row.deaths ?? '-'} decisions=${row.decisions}` + // An authored cell reports what BUILDING it cost, next to what its + // program scored. They answer different questions and a single number + // cannot carry both. + + (row.build === null + ? '' + : ` build=${row.build.minutes.toFixed(1)}m/${row.build.tokens ?? '?'}tok/` + + `${row.build.usd === null ? `unbilled(${row.build.authMode ?? 'unknown'})` : `$${row.build.usd.toFixed(2)}`}`) + ` tokens=${row.tokens ?? 'unmetered'}` // An absent cost reads as its REASON. `oauth` bills a plan and reports no // per-request figure, so "unbilled(oauth)" and "$0.0000" are different