From f3b4c9931fcd5db4ee5441de6e9cadfb27496fae Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 4 Sep 2026 15:28:37 -0700 Subject: [PATCH 1/2] fix(supervisor-run): follow Runtime-owned tree roots Runtime nested begin roots can be ownership paths instead of child ids. Resolve each nested tree through exactly one parent spawn and retain child ids for supervisor identity. --- CHANGELOG.md | 9 ++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 2 +- src/analyst/benchmark-implementation.ts | 2 +- src/supervisor-run/runtime-reader.test.ts | 102 ++++++++++++++++++ src/supervisor-run/runtime-reader.ts | 52 +++++---- 8 files changed, 150 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d95de9..8bfdbdfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.173.2] — 2026-09-04 + +### Fixed + +- The Runtime supervisor reader now follows `spawned.ownedTreeRoot` when recursive child tree paths differ from child ids. + It still accepts child-id tree roots with one duplicate root marker and verifies both recorded profile identities. + +--- + ## [0.173.1] — 2026-09-01 ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 9b784fcc..327fcbdb 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.173.1" +version = "0.173.2" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index a5bc735e..3124055b 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.173.1" + __version__ = "0.173.2" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 7ed2ac7c..705ae923 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.173.1" +version = "0.173.2" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index 2645459b..4234c1f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.173.1", + "version": "0.173.2", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 1bed7614..2754d181 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '6f800044ae1736643c6834187c5378dddda1ece3e7a59b89de33e6268e1bef67' + '7747ce4ff600eb66042957aeb46d80c9af0067fe54218e101283ca63f158aec5' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/supervisor-run/runtime-reader.test.ts b/src/supervisor-run/runtime-reader.test.ts index 19dff68d..a54efb12 100644 --- a/src/supervisor-run/runtime-reader.test.ts +++ b/src/supervisor-run/runtime-reader.test.ts @@ -513,6 +513,63 @@ describe('Runtime FileRunContext supervisor reader', () => { expect(tree.gaps.filter((gap) => gap.code === 'node-role-unavailable')).toHaveLength(0) }) + it('reads Runtime-owned nested tree roots without duplicate self-spawn markers', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'owned-tree-root') + const childId = 'root:s0' + const ownedTreeRoot = `root/${childId}` + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + parent: null, + label: 'root', + identity: { profileDigest: ROOT_PROFILE }, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: childId, + parent: 'root', + ownedTreeRoot, + label: 'nested-researcher', + identity: { profileDigest: CHILD_PROFILE }, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + begin(ownedTreeRoot, 1), + event(ownedTreeRoot, { + kind: 'spawned', + id: `${childId}:s0`, + parent: childId, + label: 'leaf', + identity: { profileDigest: LEAF_PROFILE }, + runtime: 'cli', + budget: {}, + seq: 0, + at: at(2), + }), + ]) + + const source = await readRuntimeSupervisorRun(runDir, { strict: true }) + const facts = parseSupervisorTree(source) + + expect(facts.spawns).toEqual([ + expect.objectContaining({ id: 'root', role: 'supervisor', profileDigest: ROOT_PROFILE }), + expect.objectContaining({ + id: childId, + role: 'supervisor', + profileDigest: CHILD_PROFILE, + }), + expect.objectContaining({ id: `${childId}:s0`, role: 'worker', profileDigest: LEAF_PROFILE }), + ]) + }) + it('retains a current Runtime tree and classifies transport rows as unavailable only', async () => { const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(parent, 'arena-full1-ramsey-bundle-a') @@ -881,6 +938,51 @@ describe('Runtime FileRunContext supervisor reader', () => { ) }) + it('refuses a nested tree owned by more than one parent spawn', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'duplicate-owner') + const ownedTreeRoot = 'root/root:s0' + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: 'root:s0', + parent: 'root', + ownedTreeRoot, + label: 'child-a', + profileDigest: CHILD_PROFILE, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + event('root', { + kind: 'spawned', + id: 'root:s1', + parent: 'root', + ownedTreeRoot, + label: 'child-b', + profileDigest: LEAF_PROFILE, + runtime: 'driver', + budget: {}, + seq: 1, + at: at(1), + }), + begin(ownedTreeRoot, 1), + ]) + + await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow(/has 2 parent spawns/) + }) + it('refuses a Runtime result attached to a different journal root', async () => { const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(root, 'result-mismatch') diff --git a/src/supervisor-run/runtime-reader.ts b/src/supervisor-run/runtime-reader.ts index 351ba926..60a50d7f 100644 --- a/src/supervisor-run/runtime-reader.ts +++ b/src/supervisor-run/runtime-reader.ts @@ -2,13 +2,13 @@ * Reader for agent-runtime's file-backed supervision context. * * Runtime stores multiple recursive trees in one `spawn-journal.jsonl`. - * Each line is an envelope whose `root` identifies the local tree. A nested - * driver is represented twice: once as a child spawn in its parent's tree and - * once as the parentless root marker of its own tree. This reader removes only - * that duplicate root marker and preserves the remaining envelopes for the - * supervisor-run analyzer. Runtime stores profile identity below `identity` - * and does not emit Eval's role field, so this boundary projects those fields - * without changing Runtime's journal dialect. + * Each line is an envelope whose `root` identifies the local tree. A journal + * can connect a nested tree with `spawned.ownedTreeRoot`. It can also use the + * spawned child id as the nested root and repeat the spawn as a parentless + * marker. This reader accepts both forms, removes a duplicate marker, and + * preserves the other envelopes for the supervisor-run analyzer. Runtime + * stores profile identity below `identity` and does not emit Eval's role field. + * This boundary projects those fields without changing Runtime's dialect. * * The run's terminal status is Runtime's own `result.json` `kind` — `winner`, * `no-winner`, or whatever a later arm is called — read verbatim. The reader @@ -139,6 +139,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour if (begins.length === 0) throw formatError(path, 1, 'no begin record') const parentSpawnsById = new Map() + const parentSpawnsByOwnedTreeRoot = new Map() const rootMarkersByTree = new Map() for (const entry of events) { if (entry.event.kind !== 'spawned') continue @@ -148,8 +149,15 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const matches = parentSpawnsById.get(id) ?? [] matches.push(entry) parentSpawnsById.set(id, matches) + + const ownedTreeRoot = nonEmptyString(entry.event.ownedTreeRoot) + if (ownedTreeRoot !== null) { + const owners = parentSpawnsByOwnedTreeRoot.get(ownedTreeRoot) ?? [] + owners.push(entry) + parentSpawnsByOwnedTreeRoot.set(ownedTreeRoot, owners) + } } - if (entry.root === id && entry.event.parent === undefined) { + if (entry.root === id && (entry.event.parent === undefined || entry.event.parent === null)) { const markers = rootMarkersByTree.get(entry.root) ?? [] markers.push(entry) rootMarkersByTree.set(entry.root, markers) @@ -159,9 +167,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const nestedRoots = new Set() const nestedParentSpawns = new Map() for (const begin of begins) { - const parentSpawns = (parentSpawnsById.get(begin.root) ?? []).filter( - (entry) => entry.root !== begin.root, - ) + const parentSpawns = [ + ...new Set([ + ...(parentSpawnsByOwnedTreeRoot.get(begin.root) ?? []), + ...(parentSpawnsById.get(begin.root) ?? []), + ]), + ].filter((entry) => entry.root !== begin.root) if (parentSpawns.length > 1) { throw formatError( path, @@ -186,12 +197,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const top = topRoots[0] as BeginRecord for (const nestedRoot of nestedRoots) { - const marker = rootMarkersByTree.get(nestedRoot) ?? [] - if (marker.length !== 1) { + const markers = rootMarkersByTree.get(nestedRoot) ?? [] + if (markers.length > 1) { throw formatError( path, begun.get(nestedRoot)?.line ?? 1, - `nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`, + `nested tree ${JSON.stringify(nestedRoot)} contains ${markers.length} root markers`, ) } const parentSpawn = nestedParentSpawns.get(nestedRoot) @@ -202,12 +213,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour `nested tree ${JSON.stringify(nestedRoot)} has no parent spawn`, ) } - const markerDigest = profileDigest(marker[0]?.event ?? {}) + const markerDigest = profileDigest(markers[0]?.event ?? {}) const parentDigest = profileDigest(parentSpawn.event) if (markerDigest !== null && parentDigest !== null && markerDigest !== parentDigest) { throw formatError( path, - marker[0]?.line ?? 1, + markers[0]?.line ?? 1, `nested tree ${JSON.stringify(nestedRoot)} disagrees with its parent profile digest`, ) } @@ -218,7 +229,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour // Runtime's recursive atom has no supervisor/worker role field. A tree root // is a supervisor; a child without its own tree is a worker. - const supervisorIds = new Set([top.root, ...nestedRoots]) + const supervisorIds = new Set([ + top.root, + ...[...nestedParentSpawns.values()] + .map((entry) => nonEmptyString(entry.event.id)) + .filter((id): id is string => id !== null), + ]) const normalized = events .filter( (entry) => @@ -226,7 +242,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour nestedRoots.has(entry.root) && entry.event.kind === 'spawned' && entry.event.id === entry.root && - entry.event.parent === undefined + (entry.event.parent === undefined || entry.event.parent === null) ), ) .map((entry) => { From 3807b3a14a3d14b8410b78e5bd2608ab2bed82af Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 4 Sep 2026 15:37:06 -0700 Subject: [PATCH 2/2] chore(verification): record package release check --- .evolve/skill-runs.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 8956bb77..2ebae0c1 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -58,3 +58,4 @@ {"skill":"/handoff","ts":"2026-08-19T04:51:17Z","project":"agent-eval","target":"session 099bc10a full-arc handoff: 19 open loops, 2 live lanes, incident closed, retirement executing","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/handoff","ts":"2026-08-19T05:55:54Z","project":"agent-eval","target":"agent-eval operator arc close: retirement complete, #426 deploying, GEPA re-armed","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/agent-eval","ts":"2026-08-20T07:08:30Z","project":"agent-eval","target":"docs freshness sweep vs 0.150.1 surface (PR #663)","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-09-04T22:35:55Z","project":"agent-eval","target":"agent-eval PR #735 f3b4c993 Runtime ownedTreeRoot fix","operatorPrompt":"","durationMin":null,"verdict":"SHIP_IT","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null}