Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .evolve/skill-runs.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,4 @@
{"skill":"/handoff","ts":"2026-08-19T04:51:17Z","project":"agent-eval","target":"session 099bc10a full-arc handoff: 19 open loops, 2 live lanes, incident closed, retirement executing","operatorPrompt":"","durationMin":null,"verdict":"COMPLETE","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/handoff","ts":"2026-08-19T05:55:54Z","project":"agent-eval","target":"agent-eval operator arc close: retirement complete, #426 deploying, GEPA re-armed","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/agent-eval","ts":"2026-08-20T07:08:30Z","project":"agent-eval","target":"docs freshness sweep vs 0.150.1 surface (PR #663)","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/verify","ts":"2026-09-04T22:35:55Z","project":"agent-eval","target":"agent-eval PR #735 f3b4c993 Runtime ownedTreeRoot fix","operatorPrompt":"","durationMin":null,"verdict":"SHIP_IT","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null}
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-

---

## [0.173.2] — 2026-09-04

### Fixed

- The Runtime supervisor reader now follows `spawned.ownedTreeRoot` when recursive child tree paths differ from child ids.
It still accepts child-id tree roots with one duplicate root marker and verifies both recorded profile identities.

---

## [0.173.1] — 2026-09-01

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion clients/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "agent-eval-rpc"
version = "0.173.1"
version = "0.173.2"
description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
readme = "README.md"
requires-python = ">=3.10"
Expand Down
2 changes: 1 addition & 1 deletion clients/python/src/agent_eval_rpc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
try:
__version__ = version("agent-eval-rpc")
except PackageNotFoundError:
__version__ = "0.173.1"
__version__ = "0.173.2"

__all__ = [
"Client",
Expand Down
2 changes: 1 addition & 1 deletion clients/python/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tangle-network/agent-eval",
"version": "0.173.1",
"version": "0.173.2",
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
"homepage": "https://github.com/tangle-network/agent-eval#readme",
"repository": {
Expand Down
2 changes: 1 addition & 1 deletion src/analyst/benchmark-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
])

export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 =
'6f800044ae1736643c6834187c5378dddda1ece3e7a59b89de33e6268e1bef67'
'7747ce4ff600eb66042957aeb46d80c9af0067fe54218e101283ca63f158aec5'

/** The published benchmark evidence was produced at this package version, by
* the retired one-shot direct runner, before trace analysts moved to the
Expand Down
102 changes: 102 additions & 0 deletions src/supervisor-run/runtime-reader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,63 @@ describe('Runtime FileRunContext supervisor reader', () => {
expect(tree.gaps.filter((gap) => gap.code === 'node-role-unavailable')).toHaveLength(0)
})

it('reads Runtime-owned nested tree roots without duplicate self-spawn markers', async () => {
const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(parent, 'owned-tree-root')
const childId = 'root:s0'
const ownedTreeRoot = `root/${childId}`
await writeJournal(runDir, [
begin('root', 0),
event('root', {
kind: 'spawned',
id: 'root',
parent: null,
label: 'root',
identity: { profileDigest: ROOT_PROFILE },
budget: {},
seq: 0,
at: at(0),
}),
event('root', {
kind: 'spawned',
id: childId,
parent: 'root',
ownedTreeRoot,
label: 'nested-researcher',
identity: { profileDigest: CHILD_PROFILE },
runtime: 'driver',
budget: {},
seq: 0,
at: at(1),
}),
begin(ownedTreeRoot, 1),
event(ownedTreeRoot, {
kind: 'spawned',
id: `${childId}:s0`,
parent: childId,
label: 'leaf',
identity: { profileDigest: LEAF_PROFILE },
runtime: 'cli',
budget: {},
seq: 0,
at: at(2),
}),
])

const source = await readRuntimeSupervisorRun(runDir, { strict: true })
const facts = parseSupervisorTree(source)

expect(facts.spawns).toEqual([
expect.objectContaining({ id: 'root', role: 'supervisor', profileDigest: ROOT_PROFILE }),
expect.objectContaining({
id: childId,
role: 'supervisor',
profileDigest: CHILD_PROFILE,
}),
expect.objectContaining({ id: `${childId}:s0`, role: 'worker', profileDigest: LEAF_PROFILE }),
])
})

it('retains a current Runtime tree and classifies transport rows as unavailable only', async () => {
const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(parent, 'arena-full1-ramsey-bundle-a')
Expand Down Expand Up @@ -881,6 +938,51 @@ describe('Runtime FileRunContext supervisor reader', () => {
)
})

it('refuses a nested tree owned by more than one parent spawn', async () => {
const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(root, 'duplicate-owner')
const ownedTreeRoot = 'root/root:s0'
await writeJournal(runDir, [
begin('root', 0),
event('root', {
kind: 'spawned',
id: 'root',
label: 'root',
profileDigest: ROOT_PROFILE,
budget: {},
seq: 0,
at: at(0),
}),
event('root', {
kind: 'spawned',
id: 'root:s0',
parent: 'root',
ownedTreeRoot,
label: 'child-a',
profileDigest: CHILD_PROFILE,
runtime: 'driver',
budget: {},
seq: 0,
at: at(1),
}),
event('root', {
kind: 'spawned',
id: 'root:s1',
parent: 'root',
ownedTreeRoot,
label: 'child-b',
profileDigest: LEAF_PROFILE,
runtime: 'driver',
budget: {},
seq: 1,
at: at(1),
}),
begin(ownedTreeRoot, 1),
])

await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow(/has 2 parent spawns/)
})

it('refuses a Runtime result attached to a different journal root', async () => {
const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(root, 'result-mismatch')
Expand Down
52 changes: 34 additions & 18 deletions src/supervisor-run/runtime-reader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
* Reader for agent-runtime's file-backed supervision context.
*
* Runtime stores multiple recursive trees in one `spawn-journal.jsonl`.
* Each line is an envelope whose `root` identifies the local tree. A nested
* driver is represented twice: once as a child spawn in its parent's tree and
* once as the parentless root marker of its own tree. This reader removes only
* that duplicate root marker and preserves the remaining envelopes for the
* supervisor-run analyzer. Runtime stores profile identity below `identity`
* and does not emit Eval's role field, so this boundary projects those fields
* without changing Runtime's journal dialect.
* Each line is an envelope whose `root` identifies the local tree. A journal
* can connect a nested tree with `spawned.ownedTreeRoot`. It can also use the
* spawned child id as the nested root and repeat the spawn as a parentless
* marker. This reader accepts both forms, removes a duplicate marker, and
* preserves the other envelopes for the supervisor-run analyzer. Runtime
* stores profile identity below `identity` and does not emit Eval's role field.
* This boundary projects those fields without changing Runtime's dialect.
*
* The run's terminal status is Runtime's own `result.json` `kind` — `winner`,
* `no-winner`, or whatever a later arm is called — read verbatim. The reader
Expand Down Expand Up @@ -139,6 +139,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
if (begins.length === 0) throw formatError(path, 1, 'no begin record')

const parentSpawnsById = new Map<string, EventRecord[]>()
const parentSpawnsByOwnedTreeRoot = new Map<string, EventRecord[]>()
const rootMarkersByTree = new Map<string, EventRecord[]>()
for (const entry of events) {
if (entry.event.kind !== 'spawned') continue
Expand All @@ -148,8 +149,15 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
const matches = parentSpawnsById.get(id) ?? []
matches.push(entry)
parentSpawnsById.set(id, matches)

const ownedTreeRoot = nonEmptyString(entry.event.ownedTreeRoot)
if (ownedTreeRoot !== null) {
const owners = parentSpawnsByOwnedTreeRoot.get(ownedTreeRoot) ?? []
owners.push(entry)
parentSpawnsByOwnedTreeRoot.set(ownedTreeRoot, owners)
}
}
if (entry.root === id && entry.event.parent === undefined) {
if (entry.root === id && (entry.event.parent === undefined || entry.event.parent === null)) {
const markers = rootMarkersByTree.get(entry.root) ?? []
markers.push(entry)
rootMarkersByTree.set(entry.root, markers)
Expand All @@ -159,9 +167,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
const nestedRoots = new Set<string>()
const nestedParentSpawns = new Map<string, EventRecord>()
for (const begin of begins) {
const parentSpawns = (parentSpawnsById.get(begin.root) ?? []).filter(
(entry) => entry.root !== begin.root,
)
const parentSpawns = [
...new Set([
...(parentSpawnsByOwnedTreeRoot.get(begin.root) ?? []),
...(parentSpawnsById.get(begin.root) ?? []),
]),
].filter((entry) => entry.root !== begin.root)
if (parentSpawns.length > 1) {
throw formatError(
path,
Expand All @@ -186,12 +197,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
const top = topRoots[0] as BeginRecord

for (const nestedRoot of nestedRoots) {
const marker = rootMarkersByTree.get(nestedRoot) ?? []
if (marker.length !== 1) {
const markers = rootMarkersByTree.get(nestedRoot) ?? []
if (markers.length > 1) {
throw formatError(
path,
begun.get(nestedRoot)?.line ?? 1,
`nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`,
`nested tree ${JSON.stringify(nestedRoot)} contains ${markers.length} root markers`,
)
}
const parentSpawn = nestedParentSpawns.get(nestedRoot)
Expand All @@ -202,12 +213,12 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
`nested tree ${JSON.stringify(nestedRoot)} has no parent spawn`,
)
}
const markerDigest = profileDigest(marker[0]?.event ?? {})
const markerDigest = profileDigest(markers[0]?.event ?? {})
const parentDigest = profileDigest(parentSpawn.event)
if (markerDigest !== null && parentDigest !== null && markerDigest !== parentDigest) {
throw formatError(
path,
marker[0]?.line ?? 1,
markers[0]?.line ?? 1,
`nested tree ${JSON.stringify(nestedRoot)} disagrees with its parent profile digest`,
)
}
Expand All @@ -218,15 +229,20 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour

// Runtime's recursive atom has no supervisor/worker role field. A tree root
// is a supervisor; a child without its own tree is a worker.
const supervisorIds = new Set([top.root, ...nestedRoots])
const supervisorIds = new Set([
top.root,
...[...nestedParentSpawns.values()]
.map((entry) => nonEmptyString(entry.event.id))
.filter((id): id is string => id !== null),
])
const normalized = events
.filter(
(entry) =>
!(
nestedRoots.has(entry.root) &&
entry.event.kind === 'spawned' &&
entry.event.id === entry.root &&
entry.event.parent === undefined
(entry.event.parent === undefined || entry.event.parent === null)
),
)
.map((entry) => {
Expand Down