feat(supervisor-run): read Runtime's settle record and add a report command

CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.175.0] — 2026-09-06
+
+### Added
+
+- `agent-eval supervisor-run report <runDir> [--format headline|markdown|json]` prints one run directory's report.
+  It exits 1 when the directory cannot be read and 2 on a usage error.
+- `OutcomeMetrics.supStatusSource`, `supReason`, and `failure` name the record a status came from, Runtime's no-winner reason, and the recorded error.
+  `readTerminalRecord` is the one derivation the analyzer and the rollout minter share.
+
+### Changed
+
+- The supervisor-run reader takes a run's status from Runtime's own settle record.
+  `result.json` is the `SupervisedResult` that `supervise()` returned, and its `kind` is the status (`runtime-result`).
+  The control-plane-era `state.json` `status` and `result.json` `sup_status` remain readable as named legacy sources (`legacy-state`, `legacy-result`).
+- A Runtime `failure.json` without a `result.json` reports status `failed` with the recorded error (`runtime-failure`) instead of an unavailable status.
+  A `failure.json` beside a settled result is reported with `earlierAttempt: true`; the settled result stays the status.
+- A directory holding `observer.jsonl` or `failure.json` is a Runtime run directory before its spawn journal exists.
+  `isRuntimeSupervisorRunDir`, `analyzeSupervisorRun`, and `findSupervisorRunDirs` recognize it, and every journal metric stays a named absence.
+- The Runtime reader no longer copies the result status onto its begin-stamp state document; `SupervisorRunSources.state` for a Runtime run carries `id` and `startedAt` only.
+  It no longer reads `trajectory.json`, which Runtime never wrote.
+
+---
+
 ## [0.174.0] — 2026-09-05
 
 ### Changed

README.md

@@ -163,7 +163,7 @@ and [DSPy](./docs/campaign-proposers.md#use-official-dspy-optimizers).
 | `@tangle-network/agent-eval/trajectory-replay` | Re-execute a recorded shell trajectory and check whether its failure reproduces. |
 | `@tangle-network/agent-eval/traces` | Store, replay, and inspect structured traces. |
 | `@tangle-network/agent-eval/reporting` | Statistical comparisons and report rendering. |
-| `@tangle-network/agent-eval/supervisor-run` | Read recursive run directories without collapsing missing measurements to zero. |
+| `@tangle-network/agent-eval/supervisor-run` | Read recursive run directories without collapsing missing measurements to zero; `agent-eval supervisor-run report <runDir>` prints one. |
 | `@tangle-network/agent-eval/meta-eval` | Measure the grader itself: judge calibration, sentinels, and seeded known-wrong plants. |
 | `@tangle-network/agent-eval/profile-cell` | Create and validate portable agent-profile identities. |
 | `@tangle-network/agent-eval/ledger-core` | Append-only hash-chained journal with idempotent append and chain verification. |

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.174.0"
+version = "0.175.0"
 description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -53,7 +53,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.174.0"
+    __version__ = "0.175.0"
 
 __all__ = [
     "Client",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.174.0",
+  "version": "0.175.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

src/analyst/benchmark-implementation.ts

@@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 ])
 
 export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 =
-  '10c77bd9ce4d896811395f8b623216f51bd03dab9ea209cb58e1d8d7b78acb7a'
+  '4aff818466add5b179dd2308dfa7e9c3ed4b1703c4f135fa2509ac4d47e626ef'
 
 /** The published benchmark evidence was produced at this package version, by
  * the retired one-shot direct runner, before trace analysts moved to the

src/cli.ts

@@ -6,6 +6,7 @@
  *   agent-eval rpc <method>          # one request from stdin → one response on stdout
  *   agent-eval rpc-batch <method>    # JSONL stdin → JSONL stdout
  *   agent-eval analyst-benchmark ... # benchmark a real-model trace analyst
+ *   agent-eval supervisor-run report <runDir> [--format headline|markdown|json]
  *   agent-eval openapi [--out path]  # write OpenAPI spec
  *   agent-eval version
  *
@@ -16,6 +17,7 @@ import { writeFileSync } from 'node:fs'
 import { runAnalystBenchmarkCommand } from './analyst/benchmark-command'
 import { resolveCliLlmConfig } from './cli-config'
 import { runRolloutReleaseCli } from './rollout/release/hf-dataset'
+import { runSupervisorRunCommand } from './supervisor-run/report-command'
 import { handleVersion } from './wire/handlers'
 import { buildOpenApi } from './wire/openapi'
 import { runRpcBatch, runRpcOnce } from './wire/rpc'
@@ -79,6 +81,10 @@ Commands:
   analyst-benchmark --dataset <agentrx|codetracebench> --labels <path> --trace-dir <path> [--artifact-dir <path>] --out <dir> ...
         Compare an empty baseline with a real-model trace analyst on public labels.
         Run with --help for all required inputs and controls.
+  supervisor-run report <runDir> [--format headline|markdown|json]
+        Report one Runtime or loops supervisor run directory. The status comes
+        from Runtime's own result.json / failure.json; every missing measurement
+        stays named. Exits 1 when the directory cannot be read.
   version
         Print server + wire-protocol version JSON.
 
@@ -159,6 +165,10 @@ async function main(): Promise<number> {
     case 'analyst-benchmark': {
       return await runAnalystBenchmarkCommand(process.argv.slice(3))
     }
+    case 'supervisor-run': {
+      // The subcommand owns its own flag grammar; pass raw argv through.
+      return await runSupervisorRunCommand(process.argv.slice(3))
+    }
     case 'version': {
       process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\n`)
       return 0
@@ -194,7 +204,13 @@ const FLAGS_BY_COMMAND: Record<string, ReadonlySet<string>> = {
 
 function assertKnownFlags(command: string, flags: Record<string, string>): void {
   // These subcommands parse their own argv.
-  if (command === 'rollout-release' || command === 'analyst-benchmark') return
+  if (
+    command === 'rollout-release' ||
+    command === 'analyst-benchmark' ||
+    command === 'supervisor-run'
+  ) {
+    return
+  }
   const allowed = FLAGS_BY_COMMAND[command]
   if (!allowed) return
   const unknown = Object.keys(flags).filter((flag) => !allowed.has(flag))

src/supervisor-run/analyze.ts

@@ -15,6 +15,7 @@ import {
   type WorkerLogFacts,
   workerSourceKey,
 } from './source-facts'
+import { readTerminalRecord } from './terminal-record'
 import {
   type DecisionMetrics,
   type EconomicsMetrics,
@@ -82,6 +83,13 @@ export function analyzeSupervisorRunSources(
   const state = tree.state
   const result = parseJson(src.result)
   const judge = parseJson(src.judge)
+  // Runtime's settle record outranks the legacy loops documents; the record
+  // that answered is named on the report so a status never arrives unlabeled.
+  const terminal = readTerminalRecord({
+    state,
+    result,
+    failure: src.failure === undefined ? undefined : parseJson(src.failure),
+  })
   const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree
   const rootSpawn =
     rootId === null ? null : (tree.spawns.find((spawn) => spawn.id === rootId) ?? null)
@@ -829,10 +837,12 @@ export function analyzeSupervisorRunSources(
       : parsePatch(src.patch)
 
   const outcome: OutcomeMetrics = {
-    supStatus:
-      pickString(state, 'status') ??
-      pickString(result, 'sup_status') ??
-      gap('supStatus', 'no state.json / result.json status'),
+    supStatus: isUnavailable(terminal.supStatus)
+      ? gap('supStatus', terminal.supStatus.unavailable)
+      : terminal.supStatus,
+    supStatusSource: terminal.supStatusSource,
+    supReason: terminal.supReason,
+    failure: terminal.failure,
     supVerdict:
       pickString(state, 'verdict') ??
       pickString(result, 'sup_verdict') ??

src/supervisor-run/index.ts

@@ -60,13 +60,25 @@ export {
   renderSupervisorRunHeadline,
   renderSupervisorRunMarkdown,
 } from './render'
+export {
+  runSupervisorRunCommand,
+  type SupervisorRunCommandIo,
+  type SupervisorRunReportFormat,
+} from './report-command'
 export { type SupervisorRolloutOptions, supervisorRunRolloutLines } from './rollout-nodes'
 export {
   isRuntimeSupervisorRunDir,
   type RuntimeReaderOptions,
   readRuntimeSupervisorRun,
   runtimeSupervisorRunReader,
 } from './runtime-reader'
+export {
+  NO_TERMINAL_RECORD,
+  RUNTIME_FAILED_STATUS,
+  readTerminalRecord,
+  type TerminalRecord,
+  type TerminalRecordInput,
+} from './terminal-record'
 export {
   type DecisionMetrics,
   type EconomicsMetrics,
@@ -93,7 +105,9 @@ export {
   type SupervisorRunTree,
   type SupervisorRunTreeGap,
   type SupervisorRunTreeGapCode,
+  type SupervisorStatusSource,
   showMeasured,
+  type TerminalFailure,
   type Unavailable,
   unavailable,
   type WallDistribution,

src/supervisor-run/render.ts

@@ -11,6 +11,7 @@ import {
   type SupervisorRunReport,
   type SupervisorRunRollup,
   showMeasured,
+  type TerminalFailure,
 } from './types'
 
 /**
@@ -26,6 +27,8 @@ export function renderSupervisorRunHeadline(r: SupervisorRunReport): string {
       : `${o.steers} queued / ${showMeasured(o.steersDelivered)} delivered`
   return [
     `RUN-REPORT ${r.instanceId ?? '?'} [${r.arm ?? '?'}]`,
+    `  status=${showMeasured(r.outcome.supStatus)} source=${showMeasured(r.outcome.supStatusSource)}` +
+      ` reason=${showMeasured(r.outcome.supReason)} failure=${fmtFailure(r.outcome.failure)}`,
     `  steers=${steerNote}`,
     `  waves=${showMeasured(o.waves)} sizes=${isUnavailable(o.waveSizes) ? `unavailable — ${o.waveSizes.unavailable}` : `[${o.waveSizes.join(',')}]`}` +
       ` workers=${showMeasured(o.workersSpawned)} settled=${showMeasured(o.workersSettled)} cancelled=${showMeasured(o.workersCancelled)}`,
@@ -42,6 +45,20 @@ export function renderSupervisorRunHeadline(r: SupervisorRunReport): string {
   ].join('\n')
 }
 
+/**
+ * The recorded error on one line; `none recorded` and `unavailable` stay
+ * distinct, and a throw the run outlived is labeled so it is not read as the
+ * outcome of the settled result beside it.
+ */
+function fmtFailure(v: Measured<TerminalFailure | null>): string {
+  if (isUnavailable(v)) return `unavailable — ${v.unavailable}`
+  if (v === null) return 'none recorded'
+  const name = v.name ?? 'unavailable — record has no error.name'
+  const message = v.message ?? 'unavailable — record has no error.message'
+  const attempt = v.earlierAttempt ? ', earlier attempt' : ''
+  return `${name}: ${message} [${v.source}${v.at === null ? '' : ` at ${v.at}`}${attempt}]`
+}
+
 function fmtMs(v: Measured<number>): string {
   if (isUnavailable(v)) return `unavailable — ${v.unavailable}`
   if (v < 1000) return `${v}ms`
@@ -185,6 +202,9 @@ export function renderSupervisorRunMarkdown(r: SupervisorRunReport): string {
   out.push('| Metric | Value |')
   out.push('|---|---|')
   out.push(`| Supervisor status | ${showMeasured(r.outcome.supStatus)} |`)
+  out.push(`| Status source | ${showMeasured(r.outcome.supStatusSource)} |`)
+  out.push(`| Terminal reason | ${showMeasured(r.outcome.supReason)} |`)
+  out.push(`| Failure | ${fmtFailure(r.outcome.failure)} |`)
   out.push(`| Supervisor verdict | ${showMeasured(r.outcome.supVerdict)} |`)
   out.push(`| Delivered | ${showMeasured(r.outcome.delivered)} |`)
   out.push(`| Judge resolved | ${showMeasured(r.outcome.judgeResolved)} |`)

src/supervisor-run/report-command.test.ts

@@ -0,0 +1,132 @@
+/**
+ * `agent-eval supervisor-run report <runDir>` through the command driver with
+ * captured output: the bytes a terminal would show, and the exit code.
+ */
+
+import { execFile } from 'node:child_process'
+import { mkdtemp, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { describe, expect, it } from 'vitest'
+import { runSupervisorRunCommand, SUPERVISOR_RUN_USAGE } from './report-command'
+import type { SupervisorRunReport } from './types'
+
+const FIXTURES = join(process.cwd(), 'tests', 'fixtures', 'supervisor-run')
+const NO_WINNER_DIR = join(FIXTURES, 'runtime-run-r1-no-winner')
+const FAILED_DIR = join(FIXTURES, 'runtime-run-r1-failed')
+
+interface Captured {
+  readonly code: number
+  readonly stdout: string
+  readonly stderr: string
+}
+
+async function run(argv: readonly string[]): Promise<Captured> {
+  let stdout = ''
+  let stderr = ''
+  const code = await runSupervisorRunCommand(argv, {
+    stdout: (text) => {
+      stdout += text
+    },
+    stderr: (text) => {
+      stderr += text
+    },
+  })
+  return { code, stdout, stderr }
+}
+
+describe('supervisor-run report', () => {
+  it('prints the headline for a settled Runtime run and exits 0', async () => {
+    const out = await run(['report', NO_WINNER_DIR])
+    expect(out.code).toBe(0)
+    expect(out.stderr).toBe('')
+    expect(out.stdout.startsWith('RUN-REPORT meta-operator-recursion-smoke-r1')).toBe(true)
+    expect(out.stdout).toContain(
+      'status=no-winner source=runtime-result reason=all-children-down failure=TypeError:',
+    )
+    expect(out.stdout.endsWith('\n')).toBe(true)
+  })
+
+  it('prints the markdown report on --format markdown', async () => {
+    const out = await run(['report', NO_WINNER_DIR, '--format', 'markdown'])
+    expect(out.code).toBe(0)
+    expect(out.stdout).toContain('# Run report — meta-operator-recursion-smoke-r1')
+    expect(out.stdout).toContain('| Supervisor status | no-winner |')
+    expect(out.stdout).toContain('| Status source | runtime-result |')
+    expect(out.stdout).toContain('| Terminal reason | all-children-down |')
+  })
+
+  it('prints the report as JSON on --format=json', async () => {
+    const out = await run(['report', NO_WINNER_DIR, '--format=json'])
+    expect(out.code).toBe(0)
+    const report = JSON.parse(out.stdout) as SupervisorRunReport
+    expect(report.outcome.supStatus).toBe('no-winner')
+    expect(report.outcome.supStatusSource).toBe('runtime-result')
+    expect(report.outcome.failure).toMatchObject({ earlierAttempt: true, name: 'TypeError' })
+  })
+
+  it('reports a run that threw before its first spawn', async () => {
+    const out = await run(['report', FAILED_DIR])
+    expect(out.code).toBe(0)
+    expect(out.stdout.startsWith('RUN-REPORT meta-operator-recursion-smoke-r1')).toBe(true)
+    expect(out.stdout).toContain(
+      'status=failed source=runtime-failure reason=null failure=TypeError: supervise budget.deadlineMs must be a non-negative finite number [runtime-failure at 2026-09-06T05:58:29.604Z]',
+    )
+  })
+
+  it('exits 1 when the path is not a directory', async () => {
+    const missing = join(FIXTURES, 'no-such-run')
+    const out = await run(['report', missing])
+    expect(out.code).toBe(1)
+    expect(out.stdout).toBe('')
+    expect(out.stderr).toContain(`${missing} is not a directory`)
+
+    const file = join(NO_WINNER_DIR, 'result.json')
+    expect((await run(['report', file])).code).toBe(1)
+  })
+
+  it('exits 1 when a terminal record is corrupt', async () => {
+    const runDir = await mkdtemp(join(tmpdir(), 'supervisor-run-report-'))
+    await writeFile(join(runDir, 'failure.json'), '{"runId":"x"}')
+    const out = await run(['report', runDir])
+    expect(out.code).toBe(1)
+    expect(out.stdout).toBe('')
+    expect(out.stderr).toContain('Runtime failure record has no error object')
+  })
+
+  it('exits 2 on a usage error and 0 on --help', async () => {
+    expect(await run([])).toEqual({ code: 2, stdout: `${SUPERVISOR_RUN_USAGE}\n`, stderr: '' })
+    expect((await run(['--help'])).code).toBe(0)
+    expect((await run(['report', '--help'])).code).toBe(0)
+
+    const verb = await run(['inspect', NO_WINNER_DIR])
+    expect(verb.code).toBe(2)
+    expect(verb.stderr).toContain('unknown supervisor-run subcommand "inspect"')
+
+    const format = await run(['report', NO_WINNER_DIR, '--format', 'yaml'])
+    expect(format.code).toBe(2)
+    expect(format.stderr).toContain('--format expects one of headline|markdown|json')
+
+    const flag = await run(['report', NO_WINNER_DIR, '--quiet'])
+    expect(flag.code).toBe(2)
+    expect(flag.stderr).toContain('unknown flag "--quiet"')
+
+    expect((await run(['report'])).code).toBe(2)
+    expect((await run(['report', NO_WINNER_DIR, FAILED_DIR])).code).toBe(2)
+  })
+
+  it('is reachable through the agent-eval entry point', async () => {
+    // One real process through src/cli.ts proves the dispatch and the flag
+    // pass-through; the injected-io cases above cover the command itself.
+    const { stdout, stderr } = await promisify(execFile)(
+      process.execPath,
+      ['--import', 'tsx', 'src/cli.ts', 'supervisor-run', 'report', FAILED_DIR, '--format=json'],
+      { cwd: process.cwd(), env: { ...process.env, NODE_NO_WARNINGS: '1' } },
+    )
+    expect(stderr).toBe('')
+    const report = JSON.parse(stdout) as SupervisorRunReport
+    expect(report.outcome.supStatus).toBe('failed')
+    expect(report.outcome.supStatusSource).toBe('runtime-failure')
+  }, 30_000)
+})