diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 8956bb77..2ebae0c1 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -58,3 +58,4 @@ {"skill":"/handoff","ts":"2026-08-19T04:51:17Z","project":"agent-eval","target":"session 099bc10a full-arc handoff: 19 open loops, 2 live lanes, incident closed, retirement executing","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/handoff","ts":"2026-08-19T05:55:54Z","project":"agent-eval","target":"agent-eval operator arc close: retirement complete, #426 deploying, GEPA re-armed","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/agent-eval","ts":"2026-08-20T07:08:30Z","project":"agent-eval","target":"docs freshness sweep vs 0.150.1 surface (PR #663)","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-09-04T22:35:55Z","project":"agent-eval","target":"agent-eval PR #735 f3b4c993 Runtime ownedTreeRoot fix","operatorPrompt":"","durationMin":null,"verdict":"SHIP_IT","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d95de9..8bfdbdfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.173.2] — 2026-09-04 + +### Fixed + +- The Runtime supervisor reader now follows `spawned.ownedTreeRoot` when recursive child tree paths differ from child ids. + It still accepts child-id tree roots with one duplicate root marker and verifies both recorded profile identities. + +--- + ## [0.173.1] — 2026-09-01 ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 9b784fcc..327fcbdb 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.173.1" +version = "0.173.2" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index a5bc735e..3124055b 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.173.1" + __version__ = "0.173.2" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 7ed2ac7c..705ae923 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.173.1" +version = "0.173.2" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index 2645459b..4234c1f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.173.1", + "version": "0.173.2", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 1bed7614..2754d181 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '6f800044ae1736643c6834187c5378dddda1ece3e7a59b89de33e6268e1bef67' + '7747ce4ff600eb66042957aeb46d80c9af0067fe54218e101283ca63f158aec5' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/supervisor-run/runtime-reader.test.ts b/src/supervisor-run/runtime-reader.test.ts index 19dff68d..a54efb12 100644 --- a/src/supervisor-run/runtime-reader.test.ts +++ b/src/supervisor-run/runtime-reader.test.ts @@ -513,6 +513,63 @@ describe('Runtime FileRunContext supervisor reader', () => { expect(tree.gaps.filter((gap) => gap.code === 'node-role-unavailable')).toHaveLength(0) }) + it('reads Runtime-owned nested tree roots without duplicate self-spawn markers', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'owned-tree-root') + const childId = 'root:s0' + const ownedTreeRoot = `root/${childId}` + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + parent: null, + label: 'root', + identity: { profileDigest: ROOT_PROFILE }, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: childId, + parent: 'root', + ownedTreeRoot, + label: 'nested-researcher', + identity: { profileDigest: CHILD_PROFILE }, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + begin(ownedTreeRoot, 1), + event(ownedTreeRoot, { + kind: 'spawned', + id: `${childId}:s0`, + parent: childId, + label: 'leaf', + identity: { profileDigest: LEAF_PROFILE }, + runtime: 'cli', + budget: {}, + seq: 0, + at: at(2), + }), + ]) + + const source = await readRuntimeSupervisorRun(runDir, { strict: true }) + const facts = parseSupervisorTree(source) + + expect(facts.spawns).toEqual([ + expect.objectContaining({ id: 'root', role: 'supervisor', profileDigest: ROOT_PROFILE }), + expect.objectContaining({ + id: childId, + role: 'supervisor', + profileDigest: CHILD_PROFILE, + }), + expect.objectContaining({ id: `${childId}:s0`, role: 'worker', profileDigest: LEAF_PROFILE }), + ]) + }) + it('retains a current Runtime tree and classifies transport rows as unavailable only', async () => { const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(parent, 'arena-full1-ramsey-bundle-a') @@ -881,6 +938,51 @@ describe('Runtime FileRunContext supervisor reader', () => { ) }) + it('refuses a nested tree owned by more than one parent spawn', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'duplicate-owner') + const ownedTreeRoot = 'root/root:s0' + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: 'root:s0', + parent: 'root', + ownedTreeRoot, + label: 'child-a', + profileDigest: CHILD_PROFILE, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + event('root', { + kind: 'spawned', + id: 'root:s1', + parent: 'root', + ownedTreeRoot, + label: 'child-b', + profileDigest: LEAF_PROFILE, + runtime: 'driver', + budget: {}, + seq: 1, + at: at(1), + }), + begin(ownedTreeRoot, 1), + ]) + + await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow(/has 2 parent spawns/) + }) + it('refuses a Runtime result attached to a different journal root', async () => { const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(root, 'result-mismatch') diff --git a/src/supervisor-run/runtime-reader.ts b/src/supervisor-run/runtime-reader.ts index 351ba926..60a50d7f 100644 --- a/src/supervisor-run/runtime-reader.ts +++ b/src/supervisor-run/runtime-reader.ts @@ -2,13 +2,13 @@ * Reader for agent-runtime's file-backed supervision context. * * Runtime stores multiple recursive trees in one `spawn-journal.jsonl`. - * Each line is an envelope whose `root` identifies the local tree. A nested - * driver is represented twice: once as a child spawn in its parent's tree and - * once as the parentless root marker of its own tree. This reader removes only - * that duplicate root marker and preserves the remaining envelopes for the - * supervisor-run analyzer. Runtime stores profile identity below `identity` - * and does not emit Eval's role field, so this boundary projects those fields - * without changing Runtime's journal dialect. + * Each line is an envelope whose `root` identifies the local tree. A journal + * can connect a nested tree with `spawned.ownedTreeRoot`. It can also use the + * spawned child id as the nested root and repeat the spawn as a parentless + * marker. This reader accepts both forms, removes a duplicate marker, and + * preserves the other envelopes for the supervisor-run analyzer. Runtime + * stores profile identity below `identity` and does not emit Eval's role field. + * This boundary projects those fields without changing Runtime's dialect. * * The run's terminal status is Runtime's own `result.json` `kind` — `winner`, * `no-winner`, or whatever a later arm is called — read verbatim. The reader @@ -139,6 +139,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour if (begins.length === 0) throw formatError(path, 1, 'no begin record') const parentSpawnsById = new Map() + const parentSpawnsByOwnedTreeRoot = new Map() const rootMarkersByTree = new Map() for (const entry of events) { if (entry.event.kind !== 'spawned') continue @@ -148,8 +149,15 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const matches = parentSpawnsById.get(id) ?? [] matches.push(entry) parentSpawnsById.set(id, matches) + + const ownedTreeRoot = nonEmptyString(entry.event.ownedTreeRoot) + if (ownedTreeRoot !== null) { + const owners = parentSpawnsByOwnedTreeRoot.get(ownedTreeRoot) ?? [] + owners.push(entry) + parentSpawnsByOwnedTreeRoot.set(ownedTreeRoot, owners) + } } - if (entry.root === id && entry.event.parent === undefined) { + if (entry.root === id && (entry.event.parent === undefined || entry.event.parent === null)) { const markers = rootMarkersByTree.get(entry.root) ?? [] markers.push(entry) rootMarkersByTree.set(entry.root, markers) @@ -159,9 +167,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const nestedRoots = new Set() const nestedParentSpawns = new Map() for (const begin of begins) { - const parentSpawns = (parentSpawnsById.get(begin.root) ?? []).filter( - (entry) => entry.root !== begin.root, - ) + const parentSpawns = [ + ...new Set([ + ...(parentSpawnsByOwnedTreeRoot.get(begin.root) ?? []), + ...(parentSpawnsById.get(begin.root) ?? []), + ]), + ].filter((entry) => entry.root !== begin.root) if (parentSpawns.length > 1) { throw formatError( path, @@ -186,12 +197,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const top = topRoots[0] as BeginRecord for (const nestedRoot of nestedRoots) { - const marker = rootMarkersByTree.get(nestedRoot) ?? [] - if (marker.length !== 1) { + const markers = rootMarkersByTree.get(nestedRoot) ?? [] + if (markers.length > 1) { throw formatError( path, begun.get(nestedRoot)?.line ?? 1, - `nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`, + `nested tree ${JSON.stringify(nestedRoot)} contains ${markers.length} root markers`, ) } const parentSpawn = nestedParentSpawns.get(nestedRoot) @@ -202,12 +213,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour `nested tree ${JSON.stringify(nestedRoot)} has no parent spawn`, ) } - const markerDigest = profileDigest(marker[0]?.event ?? {}) + const markerDigest = profileDigest(markers[0]?.event ?? {}) const parentDigest = profileDigest(parentSpawn.event) if (markerDigest !== null && parentDigest !== null && markerDigest !== parentDigest) { throw formatError( path, - marker[0]?.line ?? 1, + markers[0]?.line ?? 1, `nested tree ${JSON.stringify(nestedRoot)} disagrees with its parent profile digest`, ) } @@ -218,7 +229,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour // Runtime's recursive atom has no supervisor/worker role field. A tree root // is a supervisor; a child without its own tree is a worker. - const supervisorIds = new Set([top.root, ...nestedRoots]) + const supervisorIds = new Set([ + top.root, + ...[...nestedParentSpawns.values()] + .map((entry) => nonEmptyString(entry.event.id)) + .filter((id): id is string => id !== null), + ]) const normalized = events .filter( (entry) => @@ -226,7 +242,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour nestedRoots.has(entry.root) && entry.event.kind === 'spawned' && entry.event.id === entry.root && - entry.event.parent === undefined + (entry.event.parent === undefined || entry.event.parent === null) ), ) .map((entry) => {