From 6c23e86025eec42b4d5f551f9cb4896f474cb16b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 5 Sep 2026 12:30:40 -0700 Subject: [PATCH] fix(learning): preserve optimizer selection and measurement integrity --- CHANGELOG.md | 31 ++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- docs/campaign-proposers.md | 42 ++ examples/agent-engine-optimizer/index.ts | 3 + examples/self-improve-optimizer/README.md | 3 + examples/self-improve-optimizer/index.ts | 3 + examples/selfimprove-quickstart/index.ts | 1 + package.json | 2 +- src/analyst/benchmark-implementation.ts | 2 +- src/bounded-process.test.ts | 40 +- src/campaign/campaign-manifest.ts | 10 +- src/campaign/coverage.ts | 23 +- src/campaign/index.ts | 1 + src/campaign/optimization-cost.ts | 144 ++++++ .../presets/compare-optimization-methods.ts | 131 ++--- src/campaign/presets/run-final-comparison.ts | 210 ++++++++ src/campaign/presets/run-improvement-loop.ts | 222 +-------- src/campaign/presets/run-optimization.test.ts | 13 +- src/campaign/presets/run-optimization.ts | 23 +- src/campaign/surface-identity.ts | 8 + src/campaign/types.ts | 2 +- src/command-runner.test.ts | 2 +- src/contract/define-agent-eval.ts | 5 + src/contract/index.ts | 5 + src/contract/self-improve-method.ts | 397 ++++++++++++++++ src/contract/self-improve-reporting.ts | 89 ++++ src/contract/self-improve.test.ts | 8 +- src/contract/self-improve.ts | 321 +++++-------- src/fuzz/explorer.ts | 2 +- src/fuzz/fuzz-agent.test.ts | 34 ++ src/index.ts | 10 +- src/rl/predictive-validity-researcher.ts | 26 +- .../compare-optimization-methods.test.ts | 78 ++- .../external-optimizer-process.test.ts | 38 +- tests/campaign/presets.test.ts | 2 +- tests/campaign/worktree.test.ts | 4 +- tests/contract-define-agent-eval.test.ts | 19 + ...ract-self-improve-method-integrity.test.ts | 447 ++++++++++++++++++ 40 files changed, 1810 insertions(+), 597 deletions(-) create mode 100644 src/campaign/optimization-cost.ts create mode 100644 src/campaign/presets/run-final-comparison.ts create mode 100644 src/contract/self-improve-method.ts create mode 100644 src/contract/self-improve-reporting.ts create mode 100644 tests/contract-self-improve-method-integrity.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bd8ed026..4efbf5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.174.0] — 2026-09-05 + +### Changed + +- `selfImprove({ method })` executes the method directly and measures its selected surface on final cases. + It returns `SelfImproveMethodResult` with `mode: 'method'`, actual `raw.method` evidence, and `tangle.method-improvement` provenance. + It does not expose native `raw.generations` or `generationsExplored`. +- `selfImprove({ proposer })` returns `SelfImproveProposerResult` with `mode: 'proposer'` and the existing native history. + `SelfImproveResult` is the union; consumers must narrow by `mode` before reading fields specific to one mode. +- Method results with deferred holdout return `baseline: null`, `winner.compositeMean: null`, and no lift. + Method `cost` includes reported search and final spending; `ledgerCost` retains the actual receipt breakdown. +- Premeasured native baselines must match the evaluator manifest. + Use `surfaceDispatchRef(baselineSurface, dispatchRef)` when creating the baseline campaign. + See [the result and cache contracts](docs/campaign-proposers.md#read-an-improvement-result). + +### Fixed + +- Complete methods can select the unchanged baseline without triggering native duplicate-candidate rejection. +- Native ranking cannot override a complete method's selected winner or repeat its train and selection evaluations. +- Method-reported spend and incomplete accounting remain in the total without duplicating metered costs. + Both improvement and method comparison reconcile each method's report against its attributed ledger receipts. +- Final method comparisons reject missing replicas before averaging surviving scores. +- Native search and final caches bind the candidate's surface content to execution identity. +- Premeasured baselines from a different judge revision are refused before candidate execution. +- Native candidate history reports `ci95: null` when uncertainty was not estimated. +- An unchanged baseline's shared campaign contributes once to result analysis, including execution count, tokens, and cost. +- `BehaviorExplorer` uses observed scores when assigning the next round's evaluations by behavior cell. + Scenario records retain their individual identities; allocation uses the pooled cell identity. + +--- + ## [0.173.3] — 2026-09-04 ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 29b300c1..bc8d9def 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.173.3" +version = "0.174.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index d3ce4f93..059f58c2 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.173.3" + __version__ = "0.174.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index c52a1db2..ed2b89b1 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.173.3" +version = "0.174.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/campaign-proposers.md b/docs/campaign-proposers.md index 33bd4040..8dbb4e8b 100644 --- a/docs/campaign-proposers.md +++ b/docs/campaign-proposers.md @@ -20,6 +20,48 @@ Use it when one surface must get better. Use it when two or more methods must be compared at equal budget. Runnable versions: [`examples/self-improve-optimizer`](../examples/self-improve-optimizer/) and [`examples/compare-optimization-methods`](../examples/compare-optimization-methods/). +## Read An Improvement Result + +`selfImprove({ method })` executes the complete method once and measures its selected surface on final cases. +The method may select the unchanged baseline; that result returns `gateDecision: 'hold'` and an empty diff. +Agent Eval does not score train and selection cases again or choose a different surface after the method finishes. + +The result type has two modes: + +| Mode | Result | Search evidence | Cost | +|---|---|---|---| +| `proposer` | `SelfImproveProposerResult` | Native `raw.generations`, `generationsExplored`, and optional `searchHistory` | Shared `cost` ledger summary | +| `method` | `SelfImproveMethodResult` | Actual `raw.method` and its optional `searchHistory` | Combined method and final `cost`; receipt breakdown in `ledgerCost` | + +Both types are exported from the package root and `/contract`. +`SelfImproveResult` is their union; branch on `result.mode` before reading mode-specific fields. +Calls with a concrete `method` or `proposer` infer the corresponding result type. +Method mode has no native generation count or fabricated native search measurements. +Its durable `method-provenance.json` uses schema `tangle.method-improvement` and records partition, measurement, and cost-receipt digests. +Proposer mode retains `LoopProvenanceRecord`. + +When method holdout is deferred, `baseline` and `winner.compositeMean` are `null`, `lift` is absent, and the decision is `hold`. +The selected surface remains available in `winner.surface`. +Method cost preserves the larger of reported search spend and newly recorded search receipts, then adds final measurements without counting receipts twice. +Underreported spending and incomplete receipts remain explicit; `raw.method.cost` retains the original report. +Inspect `cost.accountingComplete` and `cost.incompleteReasons` before treating the known subtotal as complete spending. +The shared dollar limit controls calls admitted through the cost ledger; arbitrary off-ledger callbacks must enforce their own spending limits. + +Native generation records report `ci95: null` because search does not estimate candidate uncertainty. +Final comparisons retain their independently computed statistics. +Every final case and replica must have complete execution and judge results before comparison. + +## Bind Cached Measurements To Their Evaluator + +Candidate surface content is part of native search and final measurement identity. +Pass a stable `dispatchRef` for execution behavior outside that surface, such as the worker revision and tool configuration. +Change it when that behavior changes; function names cannot identify captured state. +Set `judgeVersion` when a judge's scoring behavior changes. + +To reuse `premeasuredBaseline` in proposer mode, measure the same train cases, seed, replicas, execution revision, and judges. +The standalone campaign must use `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` from `/campaign`. +Agent Eval refuses a prior baseline whose evaluator manifest differs. + ## Adapt A Third-Party Text Optimizer `externalTextOptimizationMethod()` is the general adapter for a package that already owns text or component search. diff --git a/examples/agent-engine-optimizer/index.ts b/examples/agent-engine-optimizer/index.ts index cac34721..12a3bd98 100644 --- a/examples/agent-engine-optimizer/index.ts +++ b/examples/agent-engine-optimizer/index.ts @@ -172,6 +172,9 @@ async function main() { }) console.log(`Gate decision: ${result.gateDecision}`) + if (result.baseline === null || result.winner.compositeMean === null) { + throw new Error('This example requires a measured final comparison') + } console.log(`Baseline: ${result.baseline.compositeMean.toFixed(3)} (held-out composite)`) console.log(`Winner: ${result.winner.compositeMean.toFixed(3)} (held-out composite)`) console.log(`Lift: ${result.lift === undefined ? 'not measured' : signed(result.lift)}`) diff --git a/examples/self-improve-optimizer/README.md b/examples/self-improve-optimizer/README.md index 855daa0e..a1e611b9 100644 --- a/examples/self-improve-optimizer/README.md +++ b/examples/self-improve-optimizer/README.md @@ -72,6 +72,9 @@ Hard limits: `MAX_TOTAL_COST_USD` (default 10) caps the whole run and `GEPA_MAX_ ## Read The Result The script prints the gate decision, the held-out baseline and winner composites, the lift, the total spend, and the baseline-to-winner diff. +Method results use `mode: 'method'`; search evidence is in `raw.method` and optional `searchHistory`. +They do not contain native `raw.generations` or `generationsExplored`. +With deferred holdout, baseline and winner scores are `null` and no lift exists. Four held-out cases are wiring-scale, not statistical evidence: a `need_more_work` decision at this size is the gate refusing to claim significance, not a failure. Grow the case list and set `budget.reps` above 1 before treating the decision as a production threshold. diff --git a/examples/self-improve-optimizer/index.ts b/examples/self-improve-optimizer/index.ts index 49bad34f..6ea52872 100644 --- a/examples/self-improve-optimizer/index.ts +++ b/examples/self-improve-optimizer/index.ts @@ -217,6 +217,9 @@ async function main() { `Backend: ${MODEL} via ${BASE_URL} (${integrity.verdict}, ${records.length} calls)`, ) console.log(`Gate decision: ${result.gateDecision}`) + if (result.baseline === null || result.winner.compositeMean === null) { + throw new Error('This example requires a measured final comparison') + } console.log(`Baseline: ${result.baseline.compositeMean.toFixed(3)} (held-out composite)`) console.log(`Winner: ${result.winner.compositeMean.toFixed(3)} (held-out composite)`) console.log(`Lift: ${result.lift === undefined ? 'not measured' : signed(result.lift)}`) diff --git a/examples/selfimprove-quickstart/index.ts b/examples/selfimprove-quickstart/index.ts index d63f1a4a..e8a6952e 100644 --- a/examples/selfimprove-quickstart/index.ts +++ b/examples/selfimprove-quickstart/index.ts @@ -113,6 +113,7 @@ async function main() { }) const result = await evalKit.improve() + if (result.mode !== 'proposer') throw new Error('This example requires a native proposer result') const i = result.insight console.log('Improvement result') diff --git a/package.json b/package.json index d69a4cf3..8f41dfab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.173.3", + "version": "0.174.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 37c82bd9..45f03dac 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - 'ff7e12bb2457745febc8d525bb099365473eaae8b9bedde5dd5fbfbe6d86e470' + '10c77bd9ce4d896811395f8b623216f51bd03dab9ea209cb58e1d8d7b78acb7a' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/bounded-process.test.ts b/src/bounded-process.test.ts index a974deb8..afef0a35 100644 --- a/src/bounded-process.test.ts +++ b/src/bounded-process.test.ts @@ -76,7 +76,7 @@ describe.skipIf(!posixOnly)('runBoundedProcess kills the whole process group', ( const res = await runBoundedProcess({ command: PGID_PID_THEN_BACKGROUND_SLEEP, shell: 'bash', - timeoutMs: 100, + timeoutMs: 1000, }) const elapsed = Date.now() - started @@ -218,22 +218,36 @@ describe.skipIf(!posixOnly)('runBoundedProcess abort handling', () => { it('an abort mid-run kills the group and reports killedBySignal', async () => { const controller = new AbortController() - const timer = setTimeout(() => controller.abort(), 300) - const started = Date.now() - const res = await runBoundedProcess({ - command: PGID_PID_THEN_BACKGROUND_SLEEP, + const ready = join(dir, 'ready') + const running = runBoundedProcess({ + command: 'echo $$; sleep 60 & echo $!; : > "$BOUNDED_PROCESS_READY"; wait', + env: { BOUNDED_PROCESS_READY: ready }, shell: 'bash', signal: controller.signal, timeoutMs: 60_000, }) - clearTimeout(timer) + const settled = Promise.allSettled([running]) - expect(res.killedBySignal).toBe(true) - expect(res.killedByTimeout).toBe(false) - expect(res.exitCode).not.toBe(0) - expect(Date.now() - started).toBeLessThan(10_000) + try { + const readyDeadline = Date.now() + 5_000 + while (!existsSync(ready) && Date.now() < readyDeadline) { + await new Promise((resolve) => setTimeout(resolve, 20)) + } + expect(existsSync(ready)).toBe(true) + const abortedAt = Date.now() + controller.abort() + const res = await running - await expectTreeGone(res.stdout) + expect(res.killedBySignal).toBe(true) + expect(res.killedByTimeout).toBe(false) + expect(res.exitCode).not.toBe(0) + expect(Date.now() - abortedAt).toBeLessThan(10_000) + + await expectTreeGone(res.stdout) + } finally { + controller.abort() + await settled + } }, 30_000) }) @@ -344,7 +358,7 @@ describe('runBoundedProcess runs an argument vector with no shell', () => { const res = await runBoundedProcess({ command: 'bash', args: ['-c', PGID_PID_THEN_BACKGROUND_SLEEP], - timeoutMs: 100, + timeoutMs: 1000, }) expect(res.killedByTimeout).toBe(true) expect(res.exitCode).not.toBe(0) @@ -383,7 +397,7 @@ describe('runBoundedProcess runs an argument vector with no shell', () => { // The abort path shares `killAndDrain` with the shell form, but the flags and // the forced non-zero exit had no coverage for an argv caller. const controller = new AbortController() - setTimeout(() => controller.abort(), 150) + setTimeout(() => controller.abort(), 1000) const res = await runBoundedProcess({ command: 'bash', args: ['-c', PGID_PID_THEN_BACKGROUND_SLEEP], diff --git a/src/campaign/campaign-manifest.ts b/src/campaign/campaign-manifest.ts index 39079b71..d060edb2 100644 --- a/src/campaign/campaign-manifest.ts +++ b/src/campaign/campaign-manifest.ts @@ -6,9 +6,9 @@ import { contentHash } from '../verdict-cache' import type { DispatchFn, JudgeConfig, Scenario } from './types' -export function computeManifestHash(input: { - scenarios: Scenario[] - judges: JudgeConfig[] +export function computeManifestHash(input: { + scenarios: TScenario[] + judges: JudgeConfig[] dispatchRef: string seed: number reps: number @@ -26,7 +26,9 @@ export function computeManifestHash(input: { }) } -function judgeVersionFor(judge: JudgeConfig): string { +function judgeVersionFor( + judge: JudgeConfig, +): string { if (judge.judgeVersion !== undefined) { const version = judge.judgeVersion.trim() if (version.length === 0) { diff --git a/src/campaign/coverage.ts b/src/campaign/coverage.ts index 44139c57..35172fd0 100644 --- a/src/campaign/coverage.ts +++ b/src/campaign/coverage.ts @@ -1,5 +1,10 @@ import { contentHash } from '../verdict-cache' -import type { CampaignCellResult, CampaignScenarioIdentity, Scenario } from './types' +import type { + CampaignCellResult, + CampaignResult, + CampaignScenarioIdentity, + Scenario, +} from './types' export interface CampaignCoverage { complete: boolean @@ -199,3 +204,19 @@ function designedCellIds( } return ids } + +/** Require the complete designed denominator before a final comparison. */ +export function assertCompleteCampaign( + campaign: CampaignResult, + scenarios: readonly TScenario[], + reps: number, + requireJudgeScore: boolean, + label: string, +): void { + const coverage = campaignCoverage(campaign.cells, scenarios, reps, requireJudgeScore) + if (!coverage.complete) { + throw new Error( + `${label} is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to compare unequal results.`, + ) + } +} diff --git a/src/campaign/index.ts b/src/campaign/index.ts index b30215e1..cda2a439 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -440,6 +440,7 @@ export { componentSurfaceIdentityMaterial, renderSurfaceDiff, surfaceContentHash, + surfaceDispatchRef, surfaceHash, } from './surface-identity' export { diff --git a/src/campaign/optimization-cost.ts b/src/campaign/optimization-cost.ts new file mode 100644 index 00000000..1c56e76f --- /dev/null +++ b/src/campaign/optimization-cost.ts @@ -0,0 +1,144 @@ +import type { CostLedgerHandle, CostLedgerSummary, CostProvenance } from '../cost-ledger' + +/** Cost reported by a method or by final test scoring. */ +export interface ComparisonCost { + /** Known subtotal. Consult `costProvenance` before treating this as total spend. */ + totalCostUsd: number + /** Exact origin of the total; uncaptured means `totalCostUsd` is only a known subtotal. */ + costProvenance: CostProvenance + accountingComplete: boolean + incompleteReasons: string[] +} + +/** Attribute method calls while retaining the shared account's admission and read behavior. */ +export function createMethodCostScope(account: CostLedgerHandle, methodName: string) { + const tags = { optimizationAttempt: crypto.randomUUID() } + const ledger: CostLedgerHandle = Object.freeze({ + costCeilingUsd: account.costCeilingUsd, + runPaidCall: (input) => account.runPaidCall({ ...input, tags: { ...input.tags, ...tags } }), + summary: account.summary.bind(account), + list: account.list.bind(account), + reconcile: account.reconcile.bind(account), + markCompleted: account.markCompleted.bind(account), + costPerCompletedTask: account.costPerCompletedTask.bind(account), + ...(account.listPending ? { listPending: account.listPending.bind(account) } : {}), + ...(account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}), + }) + return { + ledger, + reconcile(reported: ComparisonCost): ComparisonCost { + const summary = account.summary({ tags }) + if (summary.pendingCalls > 0) { + throw new Error( + `optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`, + ) + } + const recorded = costFromLedgerSummary(summary) + const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd) + const roundingToleranceUsd = + Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1) + const combined = combineComparisonCosts([ + { label: 'reported', cost: reported }, + { label: 'recorded', cost: recorded }, + ]) + const incompleteReasons = [ + ...reported.incompleteReasons, + ...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`), + ...(recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd + ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] + : []), + ] + return { + totalCostUsd, + costProvenance: + combined.costProvenance.kind === 'uncaptured' + ? combined.costProvenance + : { kind: combined.costProvenance.kind, usd: totalCostUsd }, + accountingComplete: incompleteReasons.length === 0, + incompleteReasons, + } + }, + } +} + +/** Keep the cost fields a custom optimization method must report. */ +export function costFromLedgerSummary(summary: CostLedgerSummary): ComparisonCost { + const cost = { + totalCostUsd: summary.totalCostUsd, + costProvenance: structuredClone(summary.costProvenance), + accountingComplete: summary.accountingComplete, + incompleteReasons: [...summary.incompleteReasons], + } + assertComparisonCost(cost, 'cost ledger') + return cost +} + +/** Combine method costs without turning one unknown bill into a known total. */ +export function combineComparisonCosts( + entries: ReadonlyArray<{ label: string; cost: ComparisonCost }>, +): ComparisonCost { + const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0) + const costProvenance: CostProvenance = entries.some( + (entry) => entry.cost.costProvenance.kind === 'uncaptured', + ) + ? { kind: 'uncaptured', usd: null } + : entries.every((entry) => entry.cost.costProvenance.kind === 'observed') + ? { kind: 'observed', usd: totalCostUsd } + : { kind: 'estimated', usd: totalCostUsd } + const cost = { + totalCostUsd, + costProvenance, + accountingComplete: entries.every((entry) => entry.cost.accountingComplete), + incompleteReasons: entries.flatMap((entry) => + entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`), + ), + } + assertComparisonCost(cost, 'combined cost') + return cost +} + +export function assertComparisonCost(cost: ComparisonCost, label: string): void { + if (!cost || typeof cost !== 'object') { + throw new Error(`compareOptimizationMethods: ${label} returned no cost`) + } + if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) { + throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`) + } + const provenance = cost.costProvenance + if ( + !provenance || + typeof provenance !== 'object' || + (provenance.kind !== 'observed' && + provenance.kind !== 'estimated' && + provenance.kind !== 'uncaptured') || + (provenance.kind === 'uncaptured' + ? provenance.usd !== null + : !Number.isFinite(provenance.usd) || provenance.usd < 0) + ) { + throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`) + } + if (provenance.kind !== 'uncaptured' && provenance.usd !== cost.totalCostUsd) { + throw new Error( + `compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`, + ) + } + if (typeof cost.accountingComplete !== 'boolean') { + throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`) + } + if ( + !Array.isArray(cost.incompleteReasons) || + cost.incompleteReasons.some( + (reason) => typeof reason !== 'string' || reason.trim().length === 0, + ) + ) { + throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`) + } + if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) { + throw new Error( + `compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`, + ) + } + if (cost.accountingComplete && provenance.kind === 'uncaptured') { + throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`) + } +} diff --git a/src/campaign/presets/compare-optimization-methods.ts b/src/campaign/presets/compare-optimization-methods.ts index e1260e63..39dbac2f 100644 --- a/src/campaign/presets/compare-optimization-methods.ts +++ b/src/campaign/presets/compare-optimization-methods.ts @@ -7,15 +7,10 @@ import { combineAbortSignals } from '../../abort-signal' import { mapConcurrent } from '../../concurrency' -import type { - CostLedgerHandle, - CostLedgerSummary, - CostProvenance, - CostReceipt, -} from '../../cost-ledger' +import type { CostLedgerHandle, CostLedgerSummary, CostReceipt } from '../../cost-ledger' import { pairedBootstrap } from '../../statistics' import { contentHash } from '../../verdict-cache' -import { assertCampaignDesign } from '../coverage' +import { assertCampaignDesign, assertCompleteCampaign } from '../coverage' import type { ExternalOptimizerWireCounts } from '../external-optimizer-contracts' import type { ExternalOptimizerExecutionSummary, @@ -25,6 +20,20 @@ import { assertGepaCandidatePopulationSummary, type GepaCandidatePopulationSummary, } from '../gepa-candidate-population' +import { + assertComparisonCost, + type ComparisonCost, + combineComparisonCosts, + costFromLedgerSummary, + createMethodCostScope, +} from '../optimization-cost' + +export { + type ComparisonCost, + combineComparisonCosts, + costFromLedgerSummary, +} from '../optimization-cost' + import { type RunCampaignOptions, runCampaign } from '../run-campaign' import { resolveRunDir } from '../run-dir' import { campaignBreakdown } from '../score-utils' @@ -51,16 +60,6 @@ export type OptimizationMethodRunOptions 'costCeiling' | 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed' > -/** Cost reported by a method or by final test scoring. */ -export interface ComparisonCost { - /** Known subtotal. Consult `costProvenance` before treating this as total spend. */ - totalCostUsd: number - /** Exact origin of the total; uncaptured means `totalCostUsd` is only a known subtotal. */ - costProvenance: CostProvenance - accountingComplete: boolean - incompleteReasons: string[] -} - export interface OptimizationPackageSource { kind: 'package' /** Whether package identity was inspected or supplied by caller code. */ @@ -194,7 +193,7 @@ export interface OptimizationMethodScore { /** Simultaneous paired-bootstrap interval for per-scenario lift. * `low > 0` excludes zero after adjustment for all reported contrasts. */ liftCi: { low: number; high: number } - /** Optimization spend reported by the method. Excludes final test scoring. */ + /** Search spend reconciled with recorded method calls. Excludes final test scoring. */ optimizationCost: ComparisonCost /** Optimization duration reported by the method. Excludes final test scoring. */ durationMs?: number @@ -231,7 +230,7 @@ export interface OptimizationMethodComparison { /** Best vs each other method, using simultaneous paired-bootstrap intervals. */ pairwise: OptimizationMethodPairwise[] testScenarioIds: string[] - /** Sum of the costs reported by every optimization method. */ + /** Sum of method reports reconciled against each method's recorded calls. */ optimizationCost: ComparisonCost /** Baseline and distinct winner scoring on the final test partition. */ testCost: ComparisonCost @@ -340,6 +339,13 @@ export async function compareOptimizationMethods = {} for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) { byScenario[scenarioId] = composite @@ -367,6 +373,7 @@ export async function compareOptimizationMethods { try { + const methodCost = createMethodCostScope(costLedger, method.name) const out = await method.optimize( createOptimizationMethodInput( opts, @@ -374,7 +381,7 @@ export async function compareOptimizationMethods( return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || 'anonymous' } -/** Keep the cost fields a custom optimization method must report. */ -export function costFromLedgerSummary(summary: CostLedgerSummary): ComparisonCost { - const cost = { - totalCostUsd: summary.totalCostUsd, - costProvenance: structuredClone(summary.costProvenance), - accountingComplete: summary.accountingComplete, - incompleteReasons: [...summary.incompleteReasons], - } - assertComparisonCost(cost, 'cost ledger') - return cost -} - /** Preserve every optimizer token class while keeping total input and output explicit. */ export function optimizationTokenUsageFromSummary( summary: CostLedgerSummary, @@ -1122,73 +1117,3 @@ export function optimizationTokenUsageFromSummary( calls: summary.totalCalls, } } - -/** Combine method costs without turning one unknown bill into a known total. */ -export function combineComparisonCosts( - entries: ReadonlyArray<{ label: string; cost: ComparisonCost }>, -): ComparisonCost { - const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0) - const costProvenance: CostProvenance = entries.some( - (entry) => entry.cost.costProvenance.kind === 'uncaptured', - ) - ? { kind: 'uncaptured', usd: null } - : entries.every((entry) => entry.cost.costProvenance.kind === 'observed') - ? { kind: 'observed', usd: totalCostUsd } - : { kind: 'estimated', usd: totalCostUsd } - const cost = { - totalCostUsd, - costProvenance, - accountingComplete: entries.every((entry) => entry.cost.accountingComplete), - incompleteReasons: entries.flatMap((entry) => - entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`), - ), - } - assertComparisonCost(cost, 'combined cost') - return cost -} - -function assertComparisonCost(cost: ComparisonCost, label: string): void { - if (!cost || typeof cost !== 'object') { - throw new Error(`compareOptimizationMethods: ${label} returned no cost`) - } - if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) { - throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`) - } - const provenance = cost.costProvenance - if ( - !provenance || - typeof provenance !== 'object' || - (provenance.kind !== 'observed' && - provenance.kind !== 'estimated' && - provenance.kind !== 'uncaptured') || - (provenance.kind === 'uncaptured' - ? provenance.usd !== null - : !Number.isFinite(provenance.usd) || provenance.usd < 0) - ) { - throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`) - } - if (provenance.kind !== 'uncaptured' && provenance.usd !== cost.totalCostUsd) { - throw new Error( - `compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`, - ) - } - if (typeof cost.accountingComplete !== 'boolean') { - throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`) - } - if ( - !Array.isArray(cost.incompleteReasons) || - cost.incompleteReasons.some( - (reason) => typeof reason !== 'string' || reason.trim().length === 0, - ) - ) { - throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`) - } - if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) { - throw new Error( - `compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`, - ) - } - if (cost.accountingComplete && provenance.kind === 'uncaptured') { - throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`) - } -} diff --git a/src/campaign/presets/run-final-comparison.ts b/src/campaign/presets/run-final-comparison.ts new file mode 100644 index 00000000..8bde7ac2 --- /dev/null +++ b/src/campaign/presets/run-final-comparison.ts @@ -0,0 +1,210 @@ +import { assertCompleteCampaign } from '../coverage' +import { type RunCampaignOptions, runCampaign } from '../run-campaign' +import { createRunCostLedger, fsCampaignStorage } from '../storage' +import { renderSurfaceDiff, surfaceDispatchRef, surfaceHash } from '../surface-identity' +import type { CampaignResult, Gate, MutableSurface, Scenario } from '../types' + +export interface FinalComparisonOptions + extends Omit, 'dispatch'> { + baselineSurface: MutableSurface + winnerSurface: MutableSurface + dispatchWithSurface: ( + surface: MutableSurface, + scenario: TScenario, + ctx: Parameters['dispatch']>[1], + ) => Promise + gate: Gate + holdout?: 'measured' | 'deferred' + label?: string + neutralize?: (winner: MutableSurface, baseline: MutableSurface) => MutableSurface +} + +/** Compare a search-selected surface without changing the search's selection. */ +export async function runFinalComparison( + opts: FinalComparisonOptions, +) { + const storage = opts.storage ?? fsCampaignStorage() + const costLedger = + opts.costLedger ?? + createRunCostLedger({ storage, runDir: opts.runDir, costCeilingUsd: opts.costCeiling }) + const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? 600_000 + const costPhase = (phase: string) => (opts.costPhase ? `${opts.costPhase}.${phase}` : phase) + const baselineSurface = structuredClone(opts.baselineSurface) + const winnerSurface = structuredClone(opts.winnerSurface) + const finalScenarios = structuredClone(opts.scenarios) + + // An unchanged selection has nothing to promote, regardless of measurement noise. + const winnerIsBaseline = surfaceHash(winnerSurface) === surfaceHash(baselineSurface) + const holdoutDeferred = (opts.holdout ?? 'measured') === 'deferred' + + // An empty campaign records deferred measurement without dispatching final cases. + const baselineOnHoldout = holdoutDeferred + ? await runCampaign({ + ...opts, + labeledStore: 'off', + costLedger, + costPhase: costPhase('holdout.deferred'), + dispatchTimeoutMs, + scenarios: [], + dispatch: async () => { + throw new Error('runFinalComparison: unreachable dispatch — holdout is deferred') + }, + runDir: `${opts.runDir}/holdout-deferred`, + }) + : await runCampaign({ + ...opts, + labeledStore: 'off', + costLedger, + costPhase: costPhase('holdout.baseline'), + dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef), + dispatchTimeoutMs, + scenarios: structuredClone(finalScenarios), + dispatch: (scenario, ctx) => + opts.dispatchWithSurface(structuredClone(baselineSurface), scenario, ctx), + runDir: `${opts.runDir}/holdout-baseline`, + }) + + // Reuse unchanged or deferred measurements; neither can justify promotion. + const winnerOnHoldout = + winnerIsBaseline || holdoutDeferred + ? baselineOnHoldout + : await runCampaign({ + ...opts, + labeledStore: 'off', + costLedger, + costPhase: costPhase('holdout.winner'), + dispatchRef: surfaceDispatchRef(winnerSurface, opts.dispatchRef), + dispatchTimeoutMs, + scenarios: structuredClone(finalScenarios), + dispatch: (scenario, ctx) => + opts.dispatchWithSurface(structuredClone(winnerSurface), scenario, ctx), + runDir: `${opts.runDir}/holdout-winner`, + }) + + // Missing replicas or judges must not improve an arm by reducing its denominator. + const requireJudgeScore = (opts.judges?.length ?? 0) > 0 + const reps = opts.reps ?? 1 + const assertCompleteHoldout = ( + arm: string, + campaign: CampaignResult, + ): void => { + assertCompleteCampaign( + campaign, + finalScenarios, + reps, + requireJudgeScore, + `${opts.label ?? 'runImprovementLoop'}: ${arm} holdout`, + ) + } + if (!holdoutDeferred) { + assertCompleteHoldout('baseline', baselineOnHoldout) + assertCompleteHoldout('winner', winnerOnHoldout) + } + + // Both arms share cell identifiers, so their scores need separate maps. + type ScoreMap = Map< + string, + Record; notes: string }> + > + const candidateArtifacts = new Map() + const baselineArtifacts = new Map() + const judgeScores: ScoreMap = new Map() + const baselineJudgeScores: ScoreMap = new Map() + for (const cell of winnerOnHoldout.cells) { + candidateArtifacts.set(cell.cellId, cell.artifact) + judgeScores.set(cell.cellId, cell.judgeScores) + } + for (const cell of baselineOnHoldout.cells) { + baselineArtifacts.set(cell.cellId, cell.artifact) + baselineJudgeScores.set(cell.cellId, cell.judgeScores) + } + + // The optional control measures whether lift survives removal of the selected content. + let neutralizedArtifacts: Map | undefined + let neutralizedJudgeScores: ScoreMap | undefined + let neutralizedOnHoldout: CampaignResult | undefined + let neutralizedSurface: MutableSurface | undefined + if (opts.neutralize && !winnerIsBaseline && !holdoutDeferred) { + const surface = opts.neutralize( + structuredClone(winnerSurface), + structuredClone(baselineSurface), + ) + neutralizedSurface = surface + neutralizedOnHoldout = await runCampaign({ + ...opts, + labeledStore: 'off', + costLedger, + costPhase: costPhase('holdout.neutralized'), + dispatchRef: surfaceDispatchRef(surface, opts.dispatchRef), + dispatchTimeoutMs, + scenarios: structuredClone(finalScenarios), + dispatch: (scenario, ctx) => + opts.dispatchWithSurface(structuredClone(surface), scenario, ctx), + runDir: `${opts.runDir}/holdout-neutralized`, + }) + assertCompleteHoldout('neutralized', neutralizedOnHoldout) + neutralizedArtifacts = new Map() + neutralizedJudgeScores = new Map() + for (const cell of neutralizedOnHoldout.cells) { + neutralizedArtifacts.set(cell.cellId, cell.artifact) + neutralizedJudgeScores.set(cell.cellId, cell.judgeScores) + } + } + + // Deferred measurement has no observed delta. An unchanged selection has zero change. + const gateResult = holdoutDeferred + ? { + decision: 'hold' as const, + reasons: [ + 'holdout deferred — improvement-set search completed without a held-out measurement; nothing to promote from this run', + ], + contributingGates: [ + { + name: 'holdout-deferred', + status: 'not_evaluated' as const, + detail: { holdout: 'deferred' }, + }, + ], + } + : winnerIsBaseline + ? { + decision: 'hold' as const, + reasons: ['selected surface equals the baseline (empty diff); nothing to promote'], + contributingGates: [ + { name: 'no-op-guard', status: 'fail' as const, detail: { winnerIsBaseline: true } }, + ], + delta: 0, + } + : await opts.gate.decide({ + candidateArtifacts, + baselineArtifacts, + judgeScores, + baselineJudgeScores, + neutralizedArtifacts, + neutralizedJudgeScores, + scenarios: structuredClone(finalScenarios), + cost: { + candidate: winnerOnHoldout.aggregates.cost.totalCostUsd, + baseline: baselineOnHoldout.aggregates.cost.totalCostUsd, + }, + costLedger, + costPhase: costPhase('promotion.gate'), + signal: opts.signal ?? new AbortController().signal, + }) + + const promotedDiff = + surfaceHash(winnerSurface) === surfaceHash(baselineSurface) + ? '' + : renderSurfaceDiff(winnerSurface, baselineSurface) + + return { + baselineOnHoldout, + winnerOnHoldout, + ...(neutralizedOnHoldout && neutralizedSurface + ? { neutralizedOnHoldout, neutralizedSurface } + : {}), + ...(holdoutDeferred ? { holdout: 'deferred' as const } : {}), + gateResult, + promotedDiff, + } +} diff --git a/src/campaign/presets/run-improvement-loop.ts b/src/campaign/presets/run-improvement-loop.ts index 2a7a5765..dfabc00f 100644 --- a/src/campaign/presets/run-improvement-loop.ts +++ b/src/campaign/presets/run-improvement-loop.ts @@ -5,12 +5,10 @@ */ import { openAutoPr } from '../auto-pr' -import { campaignCoverage, formatCoverageFailures } from '../coverage' -import { runCampaign } from '../run-campaign' import { resolveRunDir } from '../run-dir' import { createRunCostLedger, fsCampaignStorage } from '../storage' -import { renderSurfaceDiff, surfaceHash } from '../surface-identity' import type { CampaignResult, Gate, MutableSurface, Scenario } from '../types' +import { runFinalComparison } from './run-final-comparison' import type { RunOptimizationOptions, RunOptimizationResult } from './run-optimization' import { runOptimization } from './run-optimization' @@ -137,206 +135,15 @@ export async function runImprovementLoop( // ── (1) optimization loop produces a winner ──────────────────────── const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger }) - const baselineSurface = optimization.baselineSurface - - // No candidate beat the training baseline ⇒ the "winner" IS the baseline - // (empty diff). Re-scoring the baseline against ITSELF on the holdout and - // gating the resulting model noise as "lift" is a false positive — it - // promotes nothing and reports run-to-run variance as an improvement. Detect - // it up front: skip the redundant winner-holdout pass and force a `hold`. - const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(baselineSurface) - - // ── (2) baseline + winner re-scored on the holdout set ───────────── - const holdoutDeferred = (opts.holdout ?? 'measured') === 'deferred' - - // Deferred holdout: the held-out comparison happens in a separate later run, - // so dispatch ZERO holdout cells here. One empty-scenario campaign (shared by - // both arms) keeps every identity field (manifestHash, splitDigest, seed, - // aggregates) real without inventing a synthetic CampaignResult. - const baselineOnHoldout = holdoutDeferred - ? await runCampaign({ - ...opts, - labeledStore: 'off', - costLedger, - costPhase: 'holdout.deferred', - dispatchTimeoutMs, - scenarios: [], - dispatch: async () => { - throw new Error('runImprovementLoop: unreachable dispatch — holdout is deferred') - }, - runDir: `${opts.runDir}/holdout-deferred`, - }) - : await runCampaign({ - ...opts, - labeledStore: 'off', - costLedger, - costPhase: 'holdout.baseline', - dispatchTimeoutMs, - scenarios: opts.holdoutScenarios, - dispatch: (scenario, ctx) => opts.dispatchWithSurface(baselineSurface, scenario, ctx), - runDir: `${opts.runDir}/holdout-baseline`, - }) - - // When the winner == baseline, scoring it again would just be a second noisy - // sample of the same surface. Reuse the baseline holdout — the gate is forced - // to `hold` below regardless, and we save a full campaign. Deferred mode - // reuses the shared empty campaign for the same reason. - const winnerOnHoldout = - winnerIsBaseline || holdoutDeferred - ? baselineOnHoldout - : await runCampaign({ - ...opts, - labeledStore: 'off', - costLedger, - costPhase: 'holdout.winner', - dispatchTimeoutMs, - scenarios: opts.holdoutScenarios, - dispatch: (scenario, ctx) => - opts.dispatchWithSurface(optimization.winnerSurface, scenario, ctx), - runDir: `${opts.runDir}/holdout-winner`, - }) - - // A final comparison is valid only when both arms scored every designed - // (scenario × rep) cell with the same complete judge set. Otherwise an arm - // can appear to improve by silently dropping its hardest cell or failed - // judge. This is the same exact-denominator check used during optimization. - const requireJudgeScore = (opts.judges?.length ?? 0) > 0 - const reps = opts.reps ?? 1 - const assertCompleteHoldout = ( - arm: string, - campaign: CampaignResult, - ): void => { - const coverage = campaignCoverage( - campaign.cells, - opts.holdoutScenarios, - reps, - requireJudgeScore, - ) - if (!coverage.complete) { - throw new Error( - `runImprovementLoop: ${arm} holdout is incomplete ` + - `(${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ` + - `${formatCoverageFailures(coverage)}. Refusing to compare unequal holdout results.`, - ) - } - } - if (!holdoutDeferred) { - assertCompleteHoldout('baseline', baselineOnHoldout) - assertCompleteHoldout('winner', winnerOnHoldout) - } - - // ── (3) gate verdict ─────────────────────────────────────────────── - // Candidate + baseline share cellIds (same holdout scenarios), so their - // judge scores MUST stay in separate maps — merging them collapses the - // holdout delta to zero and the gate can never ship a real improvement. - type ScoreMap = Map< - string, - Record; notes: string }> - > - const candidateArtifacts = new Map() - const baselineArtifacts = new Map() - const judgeScores: ScoreMap = new Map() - const baselineJudgeScores: ScoreMap = new Map() - for (const cell of winnerOnHoldout.cells) { - candidateArtifacts.set(cell.cellId, cell.artifact) - judgeScores.set(cell.cellId, cell.judgeScores) - } - for (const cell of baselineOnHoldout.cells) { - baselineArtifacts.set(cell.cellId, cell.artifact) - baselineJudgeScores.set(cell.cellId, cell.judgeScores) - } - - // ── (3a) placebo arm ─────────────────────────────────────────────── - // When a `neutralize` fn is wired and the winner actually changed something, - // score a third holdout arm: the winner surface with its content - // footprint-matched-blanked. A `neutralizationGate` reads these scores to - // reject a win whose lift survives blanking the content (decorative — driven - // by the added footprint, not the content). Skipped for a no-op winner (there - // is no content to blank) and when no `neutralize` is supplied. - let neutralizedArtifacts: Map | undefined - let neutralizedJudgeScores: ScoreMap | undefined - let neutralizedOnHoldout: CampaignResult | undefined - let neutralizedSurface: MutableSurface | undefined - if (opts.neutralize && !winnerIsBaseline && !holdoutDeferred) { - const surface = opts.neutralize(optimization.winnerSurface, baselineSurface) - neutralizedSurface = surface - neutralizedOnHoldout = await runCampaign({ - ...opts, - labeledStore: 'off', - costLedger, - costPhase: 'holdout.neutralized', - dispatchTimeoutMs, - scenarios: opts.holdoutScenarios, - dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx), - runDir: `${opts.runDir}/holdout-neutralized`, - }) - assertCompleteHoldout('neutralized', neutralizedOnHoldout) - neutralizedArtifacts = new Map() - neutralizedJudgeScores = new Map() - for (const cell of neutralizedOnHoldout.cells) { - neutralizedArtifacts.set(cell.cellId, cell.artifact) - neutralizedJudgeScores.set(cell.cellId, cell.judgeScores) - } - } - - // No-op guard: a winner identical to the baseline has nothing to promote, so - // it never reaches the gate — otherwise the gate scores baseline-vs-itself, - // sees model noise as a delta, and can "ship" an empty diff (the observed - // false positive: a +4 held-out "lift" with `diff: ''`). Force `hold`. - // Deferred holdout forces `hold` WITHOUT consulting the gate: there is no - // held-out measurement to decide on, so any decision other than `hold` would - // be ungrounded. No `delta` is recorded — a 0 here would read as a measured - // no-lift, which is exactly the meaningless number this mode exists to avoid. - const gateResult = holdoutDeferred - ? { - decision: 'hold' as const, - reasons: [ - 'holdout deferred — improvement-set search completed without a held-out measurement; nothing to promote from this run', - ], - contributingGates: [ - { - name: 'holdout-deferred', - status: 'not_evaluated' as const, - detail: { holdout: 'deferred' }, - }, - ], - } - : winnerIsBaseline - ? { - decision: 'hold' as const, - reasons: [ - 'no candidate beat the training baseline — winner == baseline (empty diff); nothing to promote', - ], - contributingGates: [ - { name: 'no-op-guard', status: 'fail' as const, detail: { winnerIsBaseline: true } }, - ], - delta: 0, - } - : await opts.gate.decide({ - candidateArtifacts, - baselineArtifacts, - judgeScores, - baselineJudgeScores, - neutralizedArtifacts, - neutralizedJudgeScores, - scenarios: opts.holdoutScenarios, - cost: { - candidate: winnerOnHoldout.aggregates.cost.totalCostUsd, - baseline: baselineOnHoldout.aggregates.cost.totalCostUsd, - }, - costLedger, - costPhase: 'promotion.gate', - signal: new AbortController().signal, - }) - - // ── (4) baseline→winner diff (always) + auto-PR when gate ships ──── - // The diff is computed UNCONDITIONALLY — it's the human-auditable record of - // what the loop actually changed, needed for the provenance artifact whether - // or not a PR is opened. winner == baseline ⇒ empty diff (nothing changed). - const promotedDiff = - optimization.winnerSurfaceHash === surfaceHash(baselineSurface) - ? '' - : renderSurfaceDiff(optimization.winnerSurface, baselineSurface) + const comparison = await runFinalComparison({ + ...opts, + costLedger, + dispatchTimeoutMs, + baselineSurface: optimization.baselineSurface, + winnerSurface: optimization.winnerSurface, + scenarios: opts.holdoutScenarios, + }) + const { winnerOnHoldout, gateResult, promotedDiff } = comparison let prResult: ReturnType | undefined if (opts.autoOnPromote === 'pr' && gateResult.decision === 'ship') { @@ -351,14 +158,7 @@ export async function runImprovementLoop( return { ...optimization, - baselineOnHoldout, - winnerOnHoldout, - ...(neutralizedOnHoldout && neutralizedSurface - ? { neutralizedOnHoldout, neutralizedSurface } - : {}), - ...(holdoutDeferred ? { holdout: 'deferred' as const } : {}), - gateResult, - promotedDiff, + ...comparison, prResult, cost: costLedger.summary(), } diff --git a/src/campaign/presets/run-optimization.test.ts b/src/campaign/presets/run-optimization.test.ts index fec80e03..7522afc2 100644 --- a/src/campaign/presets/run-optimization.test.ts +++ b/src/campaign/presets/run-optimization.test.ts @@ -3,7 +3,7 @@ import type { ParentSelector } from '../parent-selection' import { runCampaign } from '../run-campaign' import { campaignMeanComposite, compareRankKeys } from '../score-utils' import { type CampaignStorage, inMemoryCampaignStorage } from '../storage' -import { surfaceHash } from '../surface-identity' +import { surfaceDispatchRef, surfaceHash } from '../surface-identity' import type { CampaignResult, JudgeConfig, @@ -43,7 +43,7 @@ async function measureBaseline( await ctx.artifacts.write('baseline.txt', 'exact baseline artifact') return { surface: 'BASELINE' } }, - dispatchRef: 'test:premeasured-baseline', + dispatchRef: surfaceDispatchRef('BASELINE', 'test:premeasured-baseline'), judges: [qualityJudge], seed: 7, reps: 1, @@ -105,6 +105,7 @@ describe('runOptimization premeasured baseline', () => { const result = await runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: baseline, @@ -114,7 +115,6 @@ describe('runOptimization premeasured baseline', () => { dispatches.push(surface) return { surface: String(surface) } }, - dispatchRef: 'test:continuation', judges: [qualityJudge], proposer: proposer(), populationSize: 1, @@ -177,6 +177,7 @@ describe('runOptimization premeasured baseline', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: { ...baseline, cells: [] }, @@ -214,6 +215,7 @@ describe('runOptimization premeasured baseline', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: withoutJudgeScores, @@ -240,6 +242,7 @@ describe('runOptimization premeasured baseline', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: baseline, @@ -266,6 +269,7 @@ describe('runOptimization premeasured baseline', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('OTHER'), campaign: baseline, @@ -292,6 +296,7 @@ describe('runOptimization premeasured baseline', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: baseline, @@ -610,6 +615,7 @@ describe('runOptimization candidate concurrency', () => { await expect( runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: await measureBaseline(failFastScenarios), @@ -671,6 +677,7 @@ describe('runOptimization candidate concurrency', () => { const pending = runOptimization({ baselineSurface: 'BASELINE', + dispatchRef: 'test:premeasured-baseline', premeasuredBaseline: { surfaceHash: surfaceHash('BASELINE'), campaign: await measureBaseline(), diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index 6aad06b8..dfb5969c 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -16,6 +16,7 @@ import type { ProposalFinding } from '../../analyst/types' import { mapConcurrent } from '../../concurrency' import type { CostLedgerHandle, CostLedgerSummary } from '../../cost-ledger' import { type Objective, paretoFrontier } from '../../pareto' +import { computeManifestHash } from '../campaign-manifest' import { assertCampaignSplitIdentity, type CampaignCoverage, @@ -36,7 +37,7 @@ import { import type { SearchHistoryReceipt } from '../search-history-receipt' import { type SearchLedgerBinding, SearchRecorder } from '../search-ledger-recording' import { createRunCostLedger, fsCampaignStorage } from '../storage' -import { surfaceHash, surfaceHashMatches } from '../surface-identity' +import { surfaceDispatchRef, surfaceHash, surfaceHashMatches } from '../surface-identity' import { type CampaignResult, type GenerationRecord, @@ -235,11 +236,14 @@ export async function runOptimization( scenarios: opts.scenarios, reps, seed: opts.seed ?? 42, + judges: opts.judges ?? [], + dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef), }) : await runCampaign({ ...opts, costLedger, costPhase: 'search.baseline', + dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef), dispatch: (scenario, ctx) => opts.dispatchWithSurface(baselineSurface, scenario, ctx), runDir: `${opts.runDir}/baseline`, }) @@ -456,6 +460,7 @@ export async function runOptimization( signal, costLedger, costPhase: 'search.candidate', + dispatchRef: surfaceDispatchRef(surface, opts.dispatchRef), dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx), runDir: `${opts.runDir}/gen-${gen}/candidate-${i}`, }) @@ -550,7 +555,7 @@ export async function runOptimization( const candidate: GenerationRecord['candidates'][number] = { surfaceHash: s.surfaceHash, composite: s.composite, - ci95: s.composite === null ? null : [s.composite, s.composite], + ci95: null, parentSurfaceHash, parentComposite, ...(s.coverage.complete @@ -691,6 +696,8 @@ function validatedPremeasuredBaseline(arg scenarios: TScenario[] reps: number seed: number + judges: NonNullable['judges']> + dispatchRef: string }): CampaignResult { const { input } = args if (!surfaceHashMatches(args.baselineSurface, input.surfaceHash)) { @@ -723,6 +730,18 @@ function validatedPremeasuredBaseline(arg 'runOptimization: premeasured baseline split does not match the requested scenarios', ) } + const expectedManifest = computeManifestHash({ + scenarios: args.scenarios, + judges: args.judges, + dispatchRef: args.dispatchRef, + seed: args.seed, + reps: args.reps, + }) + if (campaign.manifestHash !== expectedManifest) { + throw new Error( + 'runOptimization: premeasured baseline evaluator identity does not match the requested dispatch and judges', + ) + } return campaign } diff --git a/src/campaign/surface-identity.ts b/src/campaign/surface-identity.ts index 98674e07..be15915b 100644 --- a/src/campaign/surface-identity.ts +++ b/src/campaign/surface-identity.ts @@ -206,3 +206,11 @@ export function renderSurfaceDiff( return `--- baseline\n${describe(baselineSurface)}\n+++ winner\n${describe(winnerSurface)}` } + +/** Bind a campaign cache entry to the exact surface and caller-owned execution revision. */ +export function surfaceDispatchRef(surface: MutableSurface, executionRef = 'anonymous'): string { + if (!executionRef.trim() || executionRef.trim() !== executionRef) { + throw new Error('surfaceDispatchRef: executionRef must be trimmed and non-empty') + } + return `surface:${executionRef}:${surfaceContentHash(surface)}` +} diff --git a/src/campaign/types.ts b/src/campaign/types.ts index ac69561c..0dd7c792 100644 --- a/src/campaign/types.ts +++ b/src/campaign/types.ts @@ -666,7 +666,7 @@ export interface GenerationCandidate { surfaceHash: string /** Mean over complete task-quality scores, or null when none were produced. */ composite: number | null - /** Descriptive interval for `composite`, or null when no score exists. */ + /** Estimated interval for `composite`, or null when uncertainty was not estimated. */ ci95: [number, number] | null /** Exact surface this candidate mutated. */ parentSurfaceHash?: string diff --git a/src/command-runner.test.ts b/src/command-runner.test.ts index 03baf813..3060e274 100644 --- a/src/command-runner.test.ts +++ b/src/command-runner.test.ts @@ -205,7 +205,7 @@ describe('localCommandRunner on runBoundedProcess', () => { const r = await localCommandRunner.run({ cmd: 'sh', argv: ['-c', 'sleep 60 & echo $!; wait'], - capMs: 400, + capMs: 1000, }) expect(r.timedOut).toBe(true) expect(r.status).toBeNull() diff --git a/src/contract/define-agent-eval.ts b/src/contract/define-agent-eval.ts index 356193f9..10ce472e 100644 --- a/src/contract/define-agent-eval.ts +++ b/src/contract/define-agent-eval.ts @@ -1,6 +1,7 @@ import type { RunEvalOptions } from '../campaign/presets/run-eval' import { runEval } from '../campaign/presets/run-eval' import { inMemoryCampaignStorage } from '../campaign/storage' +import { surfaceDispatchRef } from '../campaign/surface-identity' import type { CampaignResult, DispatchContext, @@ -109,6 +110,10 @@ export function defineAgentEval( runDir: selectedRunDir, scenarios: scenarios ?? defaults.scenarios, dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx), + dispatchRef: surfaceDispatchRef( + selectedSurface, + campaignOpts.dispatchRef ?? defaults.dispatchRef, + ), judges: evaluateJudges(judges, judge ?? defaults.judge), } if (evalOptions.reps !== undefined) diff --git a/src/contract/index.ts b/src/contract/index.ts index 0dc4bf70..6c93a200 100644 --- a/src/contract/index.ts +++ b/src/contract/index.ts @@ -217,8 +217,13 @@ export { } from './profile-measured-comparison' export { type SelfImproveBudget, + type SelfImproveMethodOptions, + type SelfImproveMethodProvenance, + type SelfImproveMethodResult, type SelfImproveOptions, type SelfImproveProgressEvent, + type SelfImproveProposerOptions, + type SelfImproveProposerResult, type SelfImproveResult, SelfImproveRunError, selfImprove, diff --git a/src/contract/self-improve-method.ts b/src/contract/self-improve-method.ts new file mode 100644 index 00000000..0f122c07 --- /dev/null +++ b/src/contract/self-improve-method.ts @@ -0,0 +1,397 @@ +import { openAutoPr } from '../campaign/auto-pr' +import { campaignSplitDigest } from '../campaign/coverage' +import { defaultProductionGate } from '../campaign/gates/default-production-gate' +import { createMethodCostScope } from '../campaign/optimization-cost' +import { + assertOptimizationResult, + type ComparisonCost, + combineComparisonCosts, + costFromLedgerSummary, + type OptimizationMethodResult, +} from '../campaign/presets/compare-optimization-methods' +import { runFinalComparison } from '../campaign/presets/run-final-comparison' +import { campaignMeasurementDigest, canonicalDigest } from '../campaign/provenance' +import { + campaignCellExecutionEvidence, + campaignCellJudgeDimensions, + campaignCellTaskScore, +} from '../campaign/run-record' +import type { CampaignStorage } from '../campaign/storage' +import { surfaceContentHash, surfaceHash } from '../campaign/surface-identity' +import type { Scenario } from '../campaign/types' +import type { CostLedgerHandle, CostLedgerSummary } from '../cost-ledger' +import { createHostedClient } from '../hosted/client' +import type { EvalRunGenerationSnapshot } from '../hosted/types' +import { analyzeRuns } from './analyze-runs' +import type { + SelfImproveMethodOptions, + SelfImproveOptions, + SelfImproveProposerResult, +} from './self-improve' +import { cellsToRunRecords, meanComposite } from './self-improve-reporting' + +export interface SelfImproveMethodProvenance { + schema: 'tangle.method-improvement' + recordDigest: `sha256:${string}` + runId: string + runDir: string + timestamp: string + baselineContentHash: string + winnerContentHash: string + diff: string + optimizationMethod: NonNullable['optimization']> + evidence: { + trainSplitDigest: `sha256:${string}` + selectionSplitDigest: `sha256:${string}` + holdoutSplitDigest: `sha256:${string}` + baselineCampaignDigest: `sha256:${string}` + winnerCampaignDigest: `sha256:${string}` + costReceiptsDigest: `sha256:${string}` + neutralizedCampaignDigest?: `sha256:${string}` + } + gate: Awaited>['gateResult'] + holdout?: 'deferred' + baselineHoldoutComposite?: number + winnerHoldoutComposite?: number + heldOutLift?: number + cost: ComparisonCost + totalCostUsd: number + totalDurationMs: number +} + +export interface SelfImproveMethodResult + extends Omit< + SelfImproveProposerResult, + | 'mode' + | 'baseline' + | 'winner' + | 'provenance' + | 'generationsExplored' + | 'cost' + | 'raw' + | 'optimization' + | 'insight' + > { + mode: 'method' + /** No final measurement exists when holdout is deferred. */ + baseline: SelfImproveProposerResult['baseline'] | null + winner: Omit['winner'], 'compositeMean'> & { + compositeMean: number | null + } + provenance: SelfImproveMethodProvenance + optimization: NonNullable['optimization']> + /** Includes the method's reported cost and final measurements without double counting receipts. */ + cost: ComparisonCost + /** Only calls actually recorded by the shared ledger have token and channel breakdowns. */ + ledgerCost: CostLedgerSummary + insight?: SelfImproveProposerResult['insight'] + raw: Awaited>> & { + kind: 'method' + baselineSurface: SelfImproveProposerResult['raw']['baselineSurface'] + winnerSurface: SelfImproveProposerResult['raw']['winnerSurface'] + winnerSurfaceHash: string + method: OptimizationMethodResult + cost: ComparisonCost + prResult?: ReturnType + } +} + +/** Complete methods own search and selection; this path only measures their final choice. */ +export async function runSelfImproveMethod(args: { + opts: SelfImproveOptions & + Pick, 'method'> + train: TScenario[] + selection: TScenario[] + holdout: TScenario[] + costLedger: CostLedgerHandle + storage: CampaignStorage + runDir: string + startedAt: number +}): Promise> { + const { opts, train, selection, holdout, costLedger, storage, runDir, startedAt } = args + const budget = opts.budget ?? {} + if (opts.autoOnPromote === 'pr' && (!opts.ghOwner || !opts.ghRepo)) { + throw new Error("selfImprove: autoOnPromote='pr' requires ghOwner + ghRepo") + } + const baselineSurface = structuredClone(opts.baselineSurface) + const judge = { + ...opts.judge, + dimensions: opts.judge.dimensions.map((dimension) => Object.freeze({ ...dimension })), + } + Object.freeze(judge.dimensions) + Object.freeze(judge) + const methodCostScope = createMethodCostScope(costLedger, opts.method.name) + const method = await opts.method.optimize( + Object.freeze({ + baselineSurface: structuredClone(baselineSurface), + trainScenarios: Object.freeze(train.map((scenario) => structuredClone(scenario))), + selectionScenarios: Object.freeze(selection.map((scenario) => structuredClone(scenario))), + dispatchWithSurface: opts.agent, + judges: Object.freeze([judge]), + runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, '_')}`, + seed: 42, + runOptions: Object.freeze({ + storage, + maxConcurrency: budget.maxConcurrency ?? 2, + reps: budget.reps, + dispatchRef: opts.dispatchRef, + dispatchTimeoutMs: opts.dispatchTimeoutMs, + cellRetry: opts.cellRetry, + cellPlacement: opts.cellPlacement, + labeledStore: opts.labeledStore, + captureSource: opts.captureSource, + expectUsage: opts.expectUsage ?? 'assert', + }), + costLedger: methodCostScope.ledger, + }), + ) + assertOptimizationResult(opts.method.name, method) + const selected = structuredClone(method) + const methodCost = methodCostScope.reconcile(selected.cost) + const finalPhase = 'selfImprove.method-final' + const finalCost = (): ComparisonCost => + combineComparisonCosts( + ['holdout.baseline', 'holdout.winner', 'holdout.neutralized', 'promotion.gate'].map( + (phase) => ({ + label: phase, + cost: costFromLedgerSummary(costLedger.summary({ phase: `${finalPhase}.${phase}` })), + }), + ), + ) + const totalCost = (): ComparisonCost => + combineComparisonCosts([ + { label: opts.method.name, cost: methodCost }, + { label: 'final comparison', cost: finalCost() }, + ]) + const gate = + opts.gate ?? + defaultProductionGate({ + holdoutScenarios: holdout, + deltaThreshold: 0.05, + }) + const comparison = await runFinalComparison({ + baselineSurface, + winnerSurface: selected.winnerSurface, + scenarios: holdout, + dispatchWithSurface: opts.agent, + dispatchRef: opts.dispatchRef, + judges: [judge], + holdout: budget.holdout, + neutralize: opts.neutralize, + gate: { + ...gate, + async decide(context) { + const cost = totalCost() + if ( + budget.dollars !== undefined && + (!cost.accountingComplete || cost.totalCostUsd > budget.dollars) + ) { + return { + decision: 'hold' as const, + reasons: ['complete method spend cannot satisfy the declared run budget'], + contributingGates: [ + { + name: 'budget', + status: 'fail' as const, + detail: { cost, budgetUsd: budget.dollars }, + }, + ], + } + } + return gate.decide(context) + }, + }, + runDir, + storage, + costLedger, + costPhase: finalPhase, + reps: budget.reps, + maxConcurrency: budget.maxConcurrency ?? 2, + dispatchTimeoutMs: opts.dispatchTimeoutMs, + cellRetry: opts.cellRetry, + cellPlacement: opts.cellPlacement, + expectUsage: opts.expectUsage ?? 'assert', + label: 'selfImprove', + }) + const deferred = comparison.holdout === 'deferred' + const baseline = deferred + ? null + : meanComposite(comparison.baselineOnHoldout.aggregates.byScenario) + const winner = deferred ? null : meanComposite(comparison.winnerOnHoldout.aggregates.byScenario) + const lift = baseline && winner ? winner.compositeMean - baseline.compositeMean : undefined + const insight = deferred + ? undefined + : await analyzeRuns({ + runs: [ + ...cellsToRunRecords( + comparison.baselineOnHoldout.cells, + 'baseline', + runDir, + baselineSurface, + 'holdout', + opts.model, + ), + ...(comparison.winnerOnHoldout === comparison.baselineOnHoldout + ? [] + : cellsToRunRecords( + comparison.winnerOnHoldout.cells, + 'winner', + runDir, + selected.winnerSurface, + 'holdout', + opts.model, + )), + ], + baselineCandidateId: 'baseline', + ...(comparison.winnerOnHoldout === comparison.baselineOnHoldout + ? {} + : { candidateCandidateId: 'winner' }), + }) + const cost = totalCost() + const optimization = { + name: opts.method.name, + cost: methodCost, + ...(selected.durationMs === undefined ? {} : { durationMs: selected.durationMs }), + ...(selected.provenance === undefined ? {} : { provenance: selected.provenance }), + } + const durationMs = Date.now() - startedAt + const receipts = costLedger.list() + const record = { + schema: 'tangle.method-improvement' as const, + runId: `${runDir}#${startedAt}`, + runDir, + timestamp: new Date(startedAt).toISOString(), + baselineContentHash: surfaceContentHash(baselineSurface), + winnerContentHash: surfaceContentHash(selected.winnerSurface), + diff: comparison.promotedDiff, + optimizationMethod: optimization, + evidence: { + trainSplitDigest: campaignSplitDigest(train, budget.reps ?? 1), + selectionSplitDigest: campaignSplitDigest(selection, budget.reps ?? 1), + holdoutSplitDigest: campaignSplitDigest(holdout, budget.reps ?? 1), + baselineCampaignDigest: campaignMeasurementDigest(comparison.baselineOnHoldout), + winnerCampaignDigest: campaignMeasurementDigest(comparison.winnerOnHoldout), + costReceiptsDigest: canonicalDigest(receipts), + ...(comparison.neutralizedOnHoldout + ? { neutralizedCampaignDigest: campaignMeasurementDigest(comparison.neutralizedOnHoldout) } + : {}), + }, + gate: comparison.gateResult, + ...(deferred ? { holdout: 'deferred' as const } : {}), + ...(baseline && winner + ? { + baselineHoldoutComposite: baseline.compositeMean, + winnerHoldoutComposite: winner.compositeMean, + heldOutLift: lift, + } + : {}), + cost, + totalCostUsd: cost.totalCostUsd, + totalDurationMs: durationMs, + } + // The persisted JSON and returned record must have the same canonical content. + const serialized: Omit = JSON.parse( + JSON.stringify(record), + ) + const provenance: SelfImproveMethodProvenance = { + ...serialized, + recordDigest: canonicalDigest(serialized), + } + storage.ensureDir(runDir) + storage.write(`${runDir}/method-provenance.json`, JSON.stringify(provenance, null, 2)) + opts.onProvenance?.(provenance) + opts.onProgress?.({ + kind: 'gate.decided', + decision: comparison.gateResult.decision, + ...(lift === undefined ? {} : { lift }), + }) + const prResult = + opts.autoOnPromote === 'pr' && comparison.gateResult.decision === 'ship' + ? openAutoPr({ + result: comparison.winnerOnHoldout, + gate: comparison.gateResult, + promotedDiff: comparison.promotedDiff, + ghOwner: opts.ghOwner!, + ghRepo: opts.ghRepo!, + }) + : undefined + const result: SelfImproveMethodResult = { + mode: 'method', + baseline, + winner: { + surface: selected.winnerSurface, + compositeMean: winner?.compositeMean ?? null, + perScenario: winner?.perScenario ?? {}, + label: opts.method.name, + }, + ...(lift === undefined ? {} : { lift }), + diff: comparison.promotedDiff, + gateDecision: comparison.gateResult.decision, + provenance, + optimization, + cost, + ledgerCost: costLedger.summary(), + totalCostUsd: cost.totalCostUsd, + durationMs, + receipts, + ...(insight ? { insight } : {}), + ...(selected.searchHistory ? { searchHistory: selected.searchHistory } : {}), + raw: { + ...comparison, + kind: 'method', + baselineSurface, + winnerSurface: selected.winnerSurface, + winnerSurfaceHash: surfaceHash(selected.winnerSurface), + method: selected, + cost, + ...(prResult ? { prResult } : {}), + }, + } + if (opts.hostedTenant) { + const snapshot = ( + index: number, + surface: typeof baselineSurface, + campaign: typeof comparison.baselineOnHoldout, + ): EvalRunGenerationSnapshot => ({ + index, + surfaceHash: surfaceHash(surface), + surface, + cells: campaign.cells.map((cell) => { + const execution = campaignCellExecutionEvidence(cell) + return { + scenarioId: cell.scenarioId, + rep: cell.rep, + compositeMean: campaignCellTaskScore(cell) ?? null, + dimensions: campaignCellJudgeDimensions(cell), + terminalOutcome: execution.terminalOutcome, + executionErrorCount: execution.executionErrorCount ?? null, + ...(cell.error ? { errorMessage: cell.error } : {}), + } + }), + compositeMean: deferred ? null : meanComposite(campaign.aggregates.byScenario).compositeMean, + costUsd: campaign.aggregates.cost.totalCostUsd, + durationMs: campaign.durationMs, + }) + try { + await createHostedClient(opts.hostedTenant).ingestEvalRun({ + runId: provenance.runId, + runDir, + timestamp: provenance.timestamp, + status: 'finished', + labels: { ...opts.hostedLabels, mode: 'method' }, + baseline: snapshot(0, baselineSurface, comparison.baselineOnHoldout), + generations: [snapshot(1, selected.winnerSurface, comparison.winnerOnHoldout)], + gateDecision: result.gateDecision, + ...(lift === undefined ? {} : { holdoutLift: lift }), + totalCostUsd: result.totalCostUsd, + totalDurationMs: durationMs, + ...(insight ? { insightReport: insight } : {}), + }) + } catch (error) { + console.warn( + `[agent-eval] hosted ingest failed (continuing): ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + return result +} diff --git a/src/contract/self-improve-reporting.ts b/src/contract/self-improve-reporting.ts new file mode 100644 index 00000000..3290d337 --- /dev/null +++ b/src/contract/self-improve-reporting.ts @@ -0,0 +1,89 @@ +import { campaignCellToRunRecord } from '../campaign/run-record' +import { surfaceContentHash } from '../campaign/surface-identity' +import type { CampaignCellResult, MutableSurface } from '../campaign/types' +import { ValidationError } from '../errors' +import { modelHasSnapshot, type RunRecord, type RunSplitTag } from '../run-record' + +export function meanComposite(byScenario: Record): { + compositeMean: number + perScenario: Record +} { + const perScenario: Record = {} + const values: number[] = [] + for (const [id, agg] of Object.entries(byScenario)) { + perScenario[id] = agg.meanComposite + values.push(agg.meanComposite) + } + return { + compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length, + perScenario, + } +} + +/** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key. + * + * Frozen: the key names a persisted cell, so a change orphans every cell + * already written. This is a cell-key function, not a general-purpose hash. */ +function hashString(s: string): string { + let h = 2166136261 >>> 0 + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i) + h = Math.imul(h, 16777619) >>> 0 + } + return h.toString(16).padStart(8, '0') +} + +/** + * Adapt campaign cells into the `RunRecord` shape `analyzeRuns()` consumes. + * Each cell becomes one run; `candidateId` is the caller-supplied label so + * baseline + winner pair cleanly on `(experimentId, scenarioId, seed)`. + * + * `promptHash` identifies the executed surface; `configHash` identifies the candidate label. + */ +export function cellsToRunRecords( + cells: ReadonlyArray>, + candidateId: 'baseline' | 'winner', + runId: string, + surface: MutableSurface, + splitTag: RunSplitTag, + fallbackModel?: string, +): RunRecord[] { + const promptHash = surfaceContentHash(surface) + const configHash = surfaceContentHash(candidateId) + return cells.map((cell) => { + const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []) + if (receiptModels.length > 1) { + throw new ValidationError( + `selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(', ')}`, + ) + } + const model = receiptModels[0] ?? fallbackModel + if (!model) { + throw new ValidationError( + `selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`, + ) + } + if (!modelHasSnapshot(model)) { + throw new ValidationError( + `selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`, + ) + } + return campaignCellToRunRecord(cell, { + runId: `${runId}::${candidateId}::${cell.cellId}`, + experimentId: runId, + candidateId, + // scenarioId is explicit; seed keeps repeated runs distinct. + seed: + cell.rep * 1_000_000 + + hashString(cell.scenarioId) + .slice(0, 6) + .split('') + .reduce((a, c) => (a * 31 + c.charCodeAt(0)) >>> 0, 0), + model, + promptHash, + configHash, + commitSha: 'cell', + splitTag, + }) + }) +} diff --git a/src/contract/self-improve.test.ts b/src/contract/self-improve.test.ts index 4a70e6c1..f0a220bc 100644 --- a/src/contract/self-improve.test.ts +++ b/src/contract/self-improve.test.ts @@ -11,7 +11,7 @@ import { describe, expect, it } from 'vitest' import type { OptimizationMethod } from '../campaign/presets/compare-optimization-methods' import { runCampaign } from '../campaign/run-campaign' import { inMemoryCampaignStorage } from '../campaign/storage' -import { surfaceHash } from '../campaign/surface-identity' +import { surfaceDispatchRef, surfaceHash } from '../campaign/surface-identity' import type { CodeSurface, Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' import { CostLedger, type CostLedgerHandle } from '../cost-ledger' import type { DispatchContext, Scenario } from './index' @@ -258,8 +258,6 @@ describe('selfImprove — complete optimization methods', () => { storage: inMemoryCampaignStorage(), expectUsage: 'off', budget: { - generations: 1, - populationSize: 1, holdoutScenarios: finalCases, }, }) @@ -348,7 +346,7 @@ describe('selfImprove — complete optimization methods', () => { expectUsage: 'off', selectParent: ({ frontier }) => frontier[0]!, }), - ).rejects.toThrow('selectParent apply only to proposer mode') + ).rejects.toThrow('searchLedger apply only to proposer mode') }) }) @@ -695,7 +693,7 @@ describe('selfImprove — premeasured baseline passthrough', () => { const premeasured = await runCampaign({ scenarios: trainScenarios, dispatch: (scenario, ctx) => stubAgent('BASE', scenario, ctx), - dispatchRef: 'test:selfimprove-premeasured', + dispatchRef: surfaceDispatchRef('BASE'), judges: [winJudge], seed: 42, reps: 1, diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 3c1cb5c1..00a5c6f4 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -10,11 +10,10 @@ import type { ProposalFinding } from '../analyst/types' import { defaultProductionGate } from '../campaign/gates/default-production-gate' import { type PowerPreflight, powerPreflight } from '../campaign/gates/power-preflight' -import { - assertOptimizationResult, - type OptimizationMethod, - type OptimizationMethodProvenance, - type OptimizationMethodResult, +import type { + OptimizationMethod, + OptimizationMethodProvenance, + OptimizationMethodResult, } from '../campaign/presets/compare-optimization-methods' import { type RunImprovementLoopResult, @@ -35,7 +34,6 @@ import { campaignCellExecutionEvidence, campaignCellJudgeDimensions, campaignCellTaskScore, - campaignCellToRunRecord, } from '../campaign/run-record' import type { SearchHistoryReceipt } from '../campaign/search-history-receipt' import { @@ -44,9 +42,8 @@ import { fsCampaignStorage, inMemoryCampaignStorage, } from '../campaign/storage' -import { surfaceContentHash, surfaceHash } from '../campaign/surface-identity' +import { surfaceHash } from '../campaign/surface-identity' import type { - CampaignCellResult, DispatchContext, Gate, JudgeConfig, @@ -56,12 +53,19 @@ import type { SurfaceProposer, } from '../campaign/types' import type { CostLedgerHandle, CostLedgerSummary, CostReceipt } from '../cost-ledger' -import { ValidationError } from '../errors' import { createHostedClient, type HostedTenant } from '../hosted/client' import type { EvalRunCellScore, EvalRunEvent, EvalRunGenerationSnapshot } from '../hosted/types' -import { modelHasSnapshot, type RunRecord, type RunSplitTag } from '../run-record' +import type { RunSplitTag } from '../run-record' import { analyzeRuns } from './analyze-runs' import type { InsightReport } from './insight-report' +import { + runSelfImproveMethod, + type SelfImproveMethodProvenance, + type SelfImproveMethodResult, +} from './self-improve-method' +import { cellsToRunRecords, meanComposite } from './self-improve-reporting' + +export type { SelfImproveMethodProvenance, SelfImproveMethodResult } from './self-improve-method' export interface SelfImproveBudget { /** Hard spend cap across the full run. Each paid call reserves its enforced @@ -138,6 +142,8 @@ export interface SelfImproveOptions { * Omit this when every cell reports its concrete model in a receipt. */ model?: string + /** Version of execution behavior outside the candidate surface, used by measurement caches. */ + dispatchRef?: string /** Scenarios to evaluate against. Train/holdout split is computed from * these unless `budget.holdoutScenarios` is set explicitly. */ @@ -156,13 +162,14 @@ export interface SelfImproveOptions { /** * Complete prior measurement of `baselineSurface` over the TRAIN split. * Forwarded to the loop body, which validates its surface hash, scenario - * split, seed (42), reps, and coverage, then skips the baseline search + * split, seed (42), reps, evaluator manifest, and coverage, then skips the baseline search * campaign entirely — no baseline dispatch, no resumability lookup. The * train split is `scenarios` minus the holdout split, so premeasure with * exactly that scenario set (explicit `budget.holdoutScenarios`, or * `budget.holdout: 'deferred'` with no reserved set, makes the train split * deterministic). Prior spend stays in the imported campaign aggregates and - * is not re-added to this run's cost ledger. + * is not re-added to this run's cost ledger. Premeasure with + * `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` and the same judge revision. */ premeasuredBaseline?: PremeasuredOptimizationBaseline @@ -206,9 +213,9 @@ export interface SelfImproveOptions { * `.agent-eval/runs/self-improve-`. */ runDir?: string - /** Fires once the durable provenance record + OTel spans are emitted. + /** Fires once the durable provenance record is written. * Receives the structured record for inline assertions / custom routing. */ - onProvenance?: (record: LoopProvenanceRecord) => void + onProvenance?: (record: LoopProvenanceRecord | SelfImproveMethodProvenance) => void /** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`. * Returns an opaque placement key the substrate forwards to your agent @@ -314,7 +321,8 @@ export interface SelfImproveOptions { searchLedger?: RunOptimizationOptions['searchLedger'] } -export interface SelfImproveResult { +export interface SelfImproveProposerResult { + mode: 'proposer' /** Composite mean across all scenarios, baseline run. When * `budget.holdout === 'deferred'` this is measured on the improvement * (search) split — no holdout campaign ran. */ @@ -393,6 +401,27 @@ export interface SelfImproveResult { raw: RunImprovementLoopResult } +export type SelfImproveResult = + | SelfImproveProposerResult + | SelfImproveMethodResult + +export type SelfImproveMethodOptions = Omit< + SelfImproveOptions, + 'proposer' | 'onProvenance' +> & { + method: OptimizationMethod + proposer?: never + onProvenance?: (record: SelfImproveMethodProvenance) => void +} + +export type SelfImproveProposerOptions = Omit< + SelfImproveOptions, + 'method' | 'onProvenance' +> & { + method?: never + onProvenance?: (record: LoopProvenanceRecord) => void +} + /** Failed self-improvement run with an immutable receipt snapshot. */ export class SelfImproveRunError extends Error { readonly cost: CostLedgerSummary @@ -428,12 +457,14 @@ function assertSelfImproveSearchMode( throw new Error('selfImprove: method must have a trimmed name and optimize(input)') } const budget = opts.budget - if (budget?.generations !== undefined && budget.generations !== 1) { - throw new Error('selfImprove: method owns its rounds; budget.generations must be 1 when set') + if (budget?.generations !== undefined) { + throw new Error( + 'selfImprove: method owns its rounds; budget.generations applies only to proposer mode', + ) } - if (budget?.populationSize !== undefined && budget.populationSize !== 1) { + if (budget?.populationSize !== undefined) { throw new Error( - 'selfImprove: method owns its candidates; budget.populationSize must be 1 when set', + 'selfImprove: method owns its candidates; budget.populationSize applies only to proposer mode', ) } if ( @@ -441,10 +472,13 @@ function assertSelfImproveSearchMode( budget?.maxImprovementShots !== undefined || opts.analyzeGeneration !== undefined || opts.findings !== undefined || - opts.selectParent !== undefined + opts.selectParent !== undefined || + opts.premeasuredBaseline !== undefined || + opts.selectionRankKey !== undefined || + opts.searchLedger !== undefined ) { throw new Error( - 'selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, and selectParent apply only to proposer mode', + 'selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, selectParent, premeasuredBaseline, selectionRankKey, and searchLedger apply only to proposer mode', ) } } @@ -502,10 +536,6 @@ function splitMethodPartitions( } } -function safeRunComponent(value: string): string { - return value.replace(/[^a-zA-Z0-9._-]/g, '_') -} - /** 32-bit FNV-1a over raw UTF-16 code units, read as an unsigned int and used * only to order scenarios deterministically. * @@ -538,22 +568,6 @@ function splitTrainHoldout( } } -function meanComposite(byScenario: Record): { - compositeMean: number - perScenario: Record -} { - const perScenario: Record = {} - const values: number[] = [] - for (const [id, agg] of Object.entries(byScenario)) { - perScenario[id] = agg.meanComposite - values.push(agg.meanComposite) - } - return { - compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length, - perScenario, - } -} - /** * Latest search campaign measured for the winner surface; the baseline search * campaign when the winner IS the baseline. Used by the deferred-holdout @@ -597,8 +611,20 @@ function winnerSearchCampaign( * budget: { maxConcurrency: 12 }, * }) */ -export async function selfImprove( +export function selfImprove( + opts: SelfImproveMethodOptions, +): Promise> +export function selfImprove( + opts: SelfImproveProposerOptions, +): Promise> +export function selfImprove( opts: SelfImproveOptions, +): Promise> +export async function selfImprove( + opts: + | SelfImproveOptions + | SelfImproveMethodOptions + | SelfImproveProposerOptions, ): Promise> { const startedAt = Date.now() const requestedRunDir = @@ -613,7 +639,13 @@ export async function selfImprove( costCeilingUsd: opts.budget?.dollars, }) try { - return await runSelfImprove(opts, costLedger, startedAt, runDir, storage) + return await runSelfImprove( + opts as SelfImproveOptions, + costLedger, + startedAt, + runDir, + storage, + ) } catch (error) { throw new SelfImproveRunError(error, costLedger) } @@ -628,8 +660,6 @@ async function runSelfImprove( ): Promise> { const budget = opts.budget ?? {} assertSelfImproveSearchMode(opts) - const generations = opts.method ? 1 : (budget.generations ?? 3) - const populationSize = opts.method ? 1 : (budget.populationSize ?? 2) const maxConcurrency = budget.maxConcurrency ?? 2 const holdoutFraction = budget.holdoutFraction ?? 0.25 const holdoutMode = budget.holdout ?? 'measured' @@ -659,60 +689,34 @@ async function runSelfImprove( throw new Error('selfImprove: holdout split is empty. Pass more scenarios.') } - if (generations > 0 && !opts.proposer && !opts.method) { + if (opts.method) { + const partitions = splitMethodPartitions( + train, + opts.selectionScenarios, + budget.selectionFraction ?? 0.25, + ) + return runSelfImproveMethod({ + opts: { ...opts, method: opts.method }, + train: partitions.train, + selection: partitions.selection, + holdout, + costLedger, + storage, + runDir, + startedAt, + }) + } + const generations = budget.generations ?? 3 + const populationSize = budget.populationSize ?? 2 + if (generations > 0 && !opts.proposer) { throw new Error( 'selfImprove: method or proposer is required when budget.generations is greater than zero', ) } - let optimizationResult: OptimizationMethodResult | undefined - const methodPartitions = opts.method - ? splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? 0.25) - : undefined - const proposer: SurfaceProposer = opts.method - ? { - kind: `method:${opts.method.name}`, - propose: async (context) => { - if (context.generation > 0) return [] - const result = await opts.method!.optimize( - Object.freeze({ - baselineSurface: structuredClone(context.currentSurface), - trainScenarios: Object.freeze( - methodPartitions!.train.map((scenario) => structuredClone(scenario)), - ), - selectionScenarios: Object.freeze( - methodPartitions!.selection.map((scenario) => structuredClone(scenario)), - ), - dispatchWithSurface: opts.agent, - judges: Object.freeze([opts.judge]), - runDir: `${runDir}/optimization/${safeRunComponent(opts.method!.name)}`, - seed: 42, - runOptions: Object.freeze({ - storage, - maxConcurrency, - reps: budget.reps, - dispatchTimeoutMs: opts.dispatchTimeoutMs, - cellRetry: opts.cellRetry, - expectUsage, - costCeiling: budget.dollars, - }), - costLedger, - }), - ) - assertOptimizationResult(opts.method!.name, result) - optimizationResult = structuredClone(result) - return [ - { - surface: structuredClone(result.winnerSurface), - label: opts.method!.name, - rationale: `${opts.method!.name} selected this surface without final cases.`, - }, - ] - }, - } - : (opts.proposer ?? { - kind: 'baseline-only', - propose: async () => [], - }) + const proposer: SurfaceProposer = opts.proposer ?? { + kind: 'baseline-only', + propose: async () => [], + } const gate: Gate = opts.gate ?? @@ -730,6 +734,7 @@ async function runSelfImprove( baselineSurface: opts.baselineSurface, premeasuredBaseline: opts.premeasuredBaseline, dispatchWithSurface: opts.agent, + dispatchRef: opts.dispatchRef, proposer, judges: [opts.judge], populationSize, @@ -842,17 +847,19 @@ async function runSelfImprove( reportSplit, opts.model, ), - ...cellsToRunRecords( - reportWinnerCampaign.cells, - 'winner', - runDir, - result.winnerSurface, - reportSplit, - opts.model, - ), + ...(reportWinnerCampaign === reportBaselineCampaign + ? [] + : cellsToRunRecords( + reportWinnerCampaign.cells, + 'winner', + runDir, + result.winnerSurface, + reportSplit, + opts.model, + )), ], baselineCandidateId: 'baseline', - candidateCandidateId: 'winner', + ...(reportWinnerCampaign === reportBaselineCampaign ? {} : { candidateCandidateId: 'winner' }), }) // ── Durable provenance: candidate→cell→gate→promote chain + rationale + @@ -869,26 +876,13 @@ async function runSelfImprove( totalCostUsd: totalCost, totalDurationMs: durationMs, }), - ...(optimizationResult - ? { - optimizationMethod: { - name: opts.method!.name, - cost: structuredClone(optimizationResult.cost), - ...(optimizationResult.durationMs === undefined - ? {} - : { durationMs: optimizationResult.durationMs }), - ...(optimizationResult.provenance === undefined - ? {} - : { provenance: structuredClone(optimizationResult.provenance) }), - }, - } - : {}), storage, hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : undefined, }) if (opts.onProvenance) opts.onProvenance(provenance) - const summary: SelfImproveResult = { + const summary: SelfImproveProposerResult = { + mode: 'proposer', baseline, winner: { ...winnerStats, @@ -905,20 +899,6 @@ async function runSelfImprove( totalCostUsd: totalCost, cost, receipts: costLedger.list(), - ...(optimizationResult - ? { - optimization: { - name: opts.method!.name, - cost: structuredClone(optimizationResult.cost), - ...(optimizationResult.durationMs === undefined - ? {} - : { durationMs: optimizationResult.durationMs }), - ...(optimizationResult.provenance === undefined - ? {} - : { provenance: structuredClone(optimizationResult.provenance) }), - }, - } - : {}), ...(result.searchHistory ? { searchHistory: result.searchHistory } : {}), insight, ...(power ? { power } : {}), @@ -943,7 +923,7 @@ async function runSelfImprove( async function shipEvalRunToHosted( tenant: HostedTenant, opts: SelfImproveOptions, - summary: SelfImproveResult, + summary: SelfImproveProposerResult, raw: RunImprovementLoopResult, runDir: string, ): Promise { @@ -1029,76 +1009,3 @@ function averageComposite( const aggs = Object.values(campaign.aggregates.byScenario) return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length } - -/** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key. - * - * Frozen: the key names a persisted cell, so a change orphans every cell - * already written. Same loop as `stableScenarioHash` above but a different - * return form; neither is a general-purpose hash. */ -function hashString(s: string): string { - let h = 2166136261 >>> 0 - for (let i = 0; i < s.length; i++) { - h ^= s.charCodeAt(i) - h = Math.imul(h, 16777619) >>> 0 - } - return h.toString(16).padStart(8, '0') -} - -/** - * Adapt campaign cells into the `RunRecord` shape `analyzeRuns()` consumes. - * Each cell becomes one run; `candidateId` is the caller-supplied label so - * baseline + winner pair cleanly on `(experimentId, scenarioId, seed)`. - * - * `promptHash` is the REAL sha256 content hash of the surface this cell ran - * (baseline vs winner are byte-distinguishable + byte-identical-verifiable); - * `configHash` is the sha256 of the candidate label so the two candidates' - * config rows differ. Both were previously the literal `'sha256:cell'`, which - * made baseline and winner indistinguishable in every downstream record. - */ -function cellsToRunRecords( - cells: ReadonlyArray>, - candidateId: 'baseline' | 'winner', - runId: string, - surface: MutableSurface, - splitTag: RunSplitTag, - fallbackModel?: string, -): RunRecord[] { - const promptHash = surfaceContentHash(surface) - const configHash = surfaceContentHash(candidateId) - return cells.map((cell) => { - const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []) - if (receiptModels.length > 1) { - throw new ValidationError( - `selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(', ')}`, - ) - } - const model = receiptModels[0] ?? fallbackModel - if (!model) { - throw new ValidationError( - `selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`, - ) - } - if (!modelHasSnapshot(model)) { - throw new ValidationError( - `selfImprove model "${model}" lacks a snapshot version for cell ${cell.cellId}`, - ) - } - return campaignCellToRunRecord(cell, { - runId: `${runId}::${candidateId}::${cell.cellId}`, - experimentId: runId, - candidateId, - // scenarioId is explicit; seed keeps repeated runs distinct. - seed: - cell.rep * 1_000_000 + - hashString(cell.scenarioId) - .slice(0, 6) - .split('') - .reduce((a, c) => (a * 31 + c.charCodeAt(0)) >>> 0, 0), - model, - promptHash, - configHash, - commitSha: 'cell', - splitTag, - }) - }) -} diff --git a/src/fuzz/explorer.ts b/src/fuzz/explorer.ts index 81de3a93..2f10e619 100644 --- a/src/fuzz/explorer.ts +++ b/src/fuzz/explorer.ts @@ -141,7 +141,7 @@ export class BehaviorExplorer { return varianceBasedCurriculum( this.log.map((r) => ({ variantId: r.cell.id, - scenarioId: r.scenarioId, + scenarioId: '*', score: r.ev.score, pass: r.ev.valid && r.ev.score >= 0.5, })), diff --git a/src/fuzz/fuzz-agent.test.ts b/src/fuzz/fuzz-agent.test.ts index 5b6749e5..3c1d86f1 100644 --- a/src/fuzz/fuzz-agent.test.ts +++ b/src/fuzz/fuzz-agent.test.ts @@ -195,6 +195,40 @@ describe('fuzzAgent (adversarial preset)', () => { }) describe('BehaviorExplorer session + tools', () => { + it('uses observed cell variance to change allocation in the second round', async () => { + const allocations: Array<{ cellId: string; count: number }> = [] + const executedIds: string[] = [] + let nextId = 0 + const explorer = new BehaviorExplorer<{ id: string; score: number }>({ + target: 'curriculum-feedback', + space: { axes: [{ name: 'kind', values: ['stable', 'noisy'] }] }, + budget: 80, + floorPerCell: 2, + allocation: 'variance', + seedsFor: () => [], + proposer: ({ cell, count }) => + Array.from({ length: count }, (_, index) => ({ + id: `scenario-${nextId++}`, + score: cell.coords.kind === 'stable' ? 0.5 : index % 2, + })), + evaluate: async (scenario) => { + executedIds.push(scenario.id) + return { valid: true, score: scenario.score } + }, + scenarioId: (scenario) => scenario.id, + onProgress: (event) => { + if (event.type === 'cell-allocated') + allocations.push({ cellId: event.cell.id, count: event.count }) + }, + }) + await explorer.step() + expect(allocations.map((allocation) => allocation.count)).toEqual([10, 10]) + await explorer.step() + expect(allocations.slice(2).map((allocation) => allocation.count)).toEqual([8, 12]) + expect(executedIds).toHaveLength(40) + expect(new Set(executedIds).size).toBe(40) + }) + it('step() makes incremental progress an agent can drive', async () => { const explorer = new BehaviorExplorer(base) const first = await explorer.step() diff --git a/src/index.ts b/src/index.ts index c43a6884..054daddf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,7 +47,15 @@ export { defineAgentEval } from './contract/define-agent-eval' export type { InsightReport } from './contract/insight-report' -export type { SelfImproveOptions, SelfImproveResult } from './contract/self-improve' +export type { + SelfImproveMethodOptions, + SelfImproveMethodProvenance, + SelfImproveMethodResult, + SelfImproveOptions, + SelfImproveProposerOptions, + SelfImproveProposerResult, + SelfImproveResult, +} from './contract/self-improve' export { selfImprove } from './contract/self-improve' export type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset' diff --git a/src/rl/predictive-validity-researcher.ts b/src/rl/predictive-validity-researcher.ts index b41c3d4f..2a174d4b 100644 --- a/src/rl/predictive-validity-researcher.ts +++ b/src/rl/predictive-validity-researcher.ts @@ -1,22 +1,16 @@ /** - * `PredictiveValidityResearcher` — concrete `Researcher` implementation - * that drives selection from outcome-anchored predictive validity. + * `PredictiveValidityResearcher` reports failures and recommends rubric changes + * from supplied scores and observed outcomes. * - * Each method: + * `inspectFailures` groups runs below the configured score threshold. + * `runValidityCheck` stores a correlation report for subsequent recommendations. + * `proposeChange` recommends rubric changes from that report. + * `applyChange` appends those recommendations to an experiment plan. + * `evaluateChange` returns no runs and declines promotion because this class + * does not execute plans. * - * - `inspectFailures(runs)` — synthesizes failure modes from the - * bottom-quartile of `RunRecord`s on the configured proxy reward. - * - `proposeChange(failures)` — proposes steering changes that target - * the rubrics with the lowest predictive validity (decorative ones). - * Either reduce their weight in the composite, or recalibrate them. - * - `applyChange(changes, baseline)` — merges the proposed steering - * into the experiment plan. - * - `evaluateChange(plan)` — re-runs the predictive-validity check on - * the post-change runs and reports the delta. - * - * The result is a closed loop: the rubric weights drift toward the ones - * that actually predict deployment outcomes, automatically. Pair with - * `runRLCampaign` for the full auto-research story. + * Callers apply rubric changes, execute the experiment, and supply fresh results + * for the next validity check. */ import type { GateDecision, SplitCoverage } from '../held-out-gate' diff --git a/tests/campaign/compare-optimization-methods.test.ts b/tests/campaign/compare-optimization-methods.test.ts index 0f83fe19..9557c03e 100644 --- a/tests/campaign/compare-optimization-methods.test.ts +++ b/tests/campaign/compare-optimization-methods.test.ts @@ -332,6 +332,78 @@ describe('compareOptimizationMethods', () => { expect(result.testCost.incompleteReasons).toEqual([]) }) + it('reconciles concurrent methods independently from prior spending and incomplete reports', async () => { + const costLedger = new CostLedger() + const prior = await costLedger.runPaidCall({ + channel: 'optimizer', + phase: 'prior', + actor: 'prior', + model: 'fixture', + execute: async () => undefined, + receipt: () => ({ model: 'fixture', inputTokens: 1, outputTokens: 1, actualCostUsd: 11 }), + }) + if (!prior.succeeded) throw prior.error + let started = 0 + let release!: () => void + const bothStarted = new Promise((resolve) => { + release = resolve + }) + const metered = ( + name: string, + usd: number, + report: number, + usageUnknown: boolean, + ): OptimizationMethod => ({ + name, + async optimize(input) { + const paid = await input.costLedger.runPaidCall({ + channel: 'optimizer', + phase: 'search', + actor: name, + model: 'fixture', + execute: async () => { + if (++started === 2) release() + await bothStarted + return 'SOLVE_h1' + }, + receipt: () => ({ + model: 'fixture', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: usd, + usageUnknown, + }), + }) + if (!paid.succeeded) throw paid.error + return { winnerSurface: paid.value, cost: completeCost(report) } + }, + }) + const result = await compareOptimizationMethods({ + methods: [metered('underreported', 3, 0, false), metered('unknown-usage', 5, 5, true)], + optimizationConcurrency: 2, + baselineSurface: 'nothing', + ...PARTITIONS, + dispatchWithSurface: async (surface) => ({ text: String(surface) }), + judges: [judge], + runDir, + costLedger, + expectUsage: 'off', + }) + expect(result.optimizationCost.totalCostUsd).toBe(8) + expect(result.totalCost.totalCostUsd).toBe(8) + expect(costLedger.summary().totalCostUsd).toBe(19) + const costs = Object.fromEntries( + result.scores.map((score) => [score.name, score.optimizationCost]), + ) + expect(costs.underreported?.totalCostUsd).toBe(3) + expect(costs['unknown-usage']?.totalCostUsd).toBe(5) + expect(costs.underreported?.incompleteReasons.join(' ')).toContain( + 'reported 0 USD below recorded 3 USD', + ) + expect(costs['unknown-usage']?.incompleteReasons.join(' ')).toContain('token usage unknown') + expect(result.optimizationCost.accountingComplete).toBe(false) + }) + it('applies one cost ceiling across baseline and winner test scoring', async () => { let paidCalls = 0 await expect( @@ -348,7 +420,7 @@ describe('compareOptimizationMethods', () => { costCeiling: 0.05, expectUsage: 'assert', }), - ).rejects.toThrow(/produced no test score/) + ).rejects.toThrow(/final comparison is incomplete/) expect(paidCalls).toBe(5) }) @@ -552,7 +624,9 @@ describe('compareOptimizationMethods', () => { runDir, expectUsage: 'off', }), - ).rejects.toThrow(/compareOptimizationMethods: baseline produced no test score.*h3/) + ).rejects.toThrow( + /compareOptimizationMethods: test\/baseline final comparison is incomplete.*h3/, + ) }) const invalidControls: Array<[string, Partial>, RegExp]> = diff --git a/tests/campaign/external-optimizer-process.test.ts b/tests/campaign/external-optimizer-process.test.ts index d870a559..0f5cac12 100644 --- a/tests/campaign/external-optimizer-process.test.ts +++ b/tests/campaign/external-optimizer-process.test.ts @@ -647,32 +647,32 @@ describe('external optimizer process', () => { const marker = join(dir, 'descendant-survived.txt') const descendant = [ "const { writeFileSync } = require('node:fs')", + `writeFileSync(${JSON.stringify(ready)}, 'ready')`, `setTimeout(() => writeFileSync(${JSON.stringify(marker)}, 'survived'), 1_000)`, 'setInterval(() => {}, 1_000)', ].join(';') const parent = [ "const { spawn } = require('node:child_process')", - "const { writeFileSync } = require('node:fs')", `spawn(process.execPath, ['-e', ${JSON.stringify(descendant)}], { stdio: 'ignore' })`, - `writeFileSync(${JSON.stringify(ready)}, 'ready')`, 'setInterval(() => {}, 1_000)', ].join(';') const owner = new AbortController() + const running = runExternalOptimizerProcess({ + label: 'aborted optimizer', + tempPrefix: 'agent-eval-aborted-', + module: 'unused', + input: {}, + runner: { + command: process.execPath, + args: ['-e', parent, '--'], + }, + timeoutMs: 10_000, + signal: owner.signal, + }) + const settled = Promise.allSettled([running]) try { - const running = runExternalOptimizerProcess({ - label: 'aborted optimizer', - tempPrefix: 'agent-eval-aborted-', - module: 'unused', - input: {}, - runner: { - command: process.execPath, - args: ['-e', parent, '--'], - }, - timeoutMs: 10_000, - signal: owner.signal, - }) - await waitFor(() => existsSync(ready)) + await waitFor(() => existsSync(ready), 5_000) const abortedAt = performance.now() owner.abort(new Error('owner cancelled optimizer')) @@ -682,6 +682,7 @@ describe('external optimizer process', () => { expect(existsSync(marker)).toBe(false) } finally { owner.abort(new Error('test cleanup')) + await settled await rm(dir, { recursive: true, force: true }) } }, 10_000) @@ -2264,11 +2265,12 @@ function postResponses( }) } -async function waitFor(predicate: () => boolean): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { +async function waitFor(predicate: () => boolean, timeoutMs = 500): Promise { + const deadline = performance.now() + timeoutMs + do { if (predicate()) return await new Promise((resolve) => setTimeout(resolve, 5)) - } + } while (performance.now() < deadline) throw new Error('condition was not met') } diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index 3aa2a67a..640eafaa 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -1740,7 +1740,7 @@ describe('runImprovementLoop — no-op guard (empty-diff false-ship killer)', () seed: 7, }) expect(result.gateResult.decision).toBe('hold') - expect(result.gateResult.reasons.join(' ')).toMatch(/winner == baseline/) + expect(result.gateResult.reasons.join(' ')).toMatch(/selected surface equals the baseline/) expect(result.promotedDiff).toBe('') // The winner surface is byte-identical to the baseline. expect(String(result.winnerSurface)).toBe(STRONG) diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 69f1992a..a75a29fc 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -50,7 +50,7 @@ beforeEach(() => { }) afterEach(() => rmSync(repoRoot, { recursive: true, force: true })) -describe('gitWorktreeAdapter — real git worktrees', () => { +describe('gitWorktreeAdapter — real git worktrees', { timeout: 30_000 }, () => { it.each(['repository', 'worktree directory'])( 'returns a canonical worktree path when the %s is a symbolic link', async (linkedOption) => { @@ -654,7 +654,7 @@ describe('gitWorktreeAdapter — real git worktrees', () => { }) }) -describe('resolveWorktreePath', () => { +describe('resolveWorktreePath', { timeout: 30_000 }, () => { it('returns a verified absolute worktree path', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const wt = await adapter.create({ baseRef: 'main', label: 'resolve' }) diff --git a/tests/contract-define-agent-eval.test.ts b/tests/contract-define-agent-eval.test.ts index c304b523..73e1a943 100644 --- a/tests/contract-define-agent-eval.test.ts +++ b/tests/contract-define-agent-eval.test.ts @@ -2,6 +2,7 @@ import { mkdtempSync, readdirSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { inMemoryCampaignStorage } from '../src/campaign/storage' import { defineAgentEval, type JudgeConfig, @@ -131,6 +132,24 @@ describe('defineAgentEval', () => { expect(result.aggregates.byJudge.quality).toBeUndefined() }) + it('does not reuse baseline scores for another surface in the same run directory', async () => { + const evalKit = defineAgentEval({ + scenarios, + agent, + judge, + baselineSurface: 'base', + expectUsage: 'off', + storage: inMemoryCampaignStorage(), + runDir: 'mem://evaluate-surface-identity', + }) + const baseline = await evalKit.evaluate() + const candidate = await evalKit.evaluate({ surface: 'candidate better' }) + expect(baseline.aggregates.byJudge.quality?.mean).toBeCloseTo(0.3) + expect(candidate.cells.every((cell) => cell.artifact.surface === 'candidate better')).toBe(true) + expect(candidate.aggregates.byJudge.quality?.mean).toBeCloseTo(0.9) + expect(candidate.manifestHash).not.toBe(baseline.manifestHash) + }) + it('fails loudly when evaluate callers pass an empty judge list', async () => { const evalKit = defineAgentEval({ scenarios, diff --git a/tests/contract-self-improve-method-integrity.test.ts b/tests/contract-self-improve-method-integrity.test.ts new file mode 100644 index 00000000..2d16c3d1 --- /dev/null +++ b/tests/contract-self-improve-method-integrity.test.ts @@ -0,0 +1,447 @@ +import { describe, expect, it } from 'vitest' +import { compareOptimizationMethods } from '../src/campaign/presets/compare-optimization-methods' +import { runOptimization } from '../src/campaign/presets/run-optimization' +import { runCampaign } from '../src/campaign/run-campaign' +import { createRunCostLedger, inMemoryCampaignStorage } from '../src/campaign/storage' +import { surfaceDispatchRef, surfaceHash } from '../src/campaign/surface-identity' +import type { Gate, JudgeConfig, Scenario } from '../src/campaign/types' +import { selfImprove } from '../src/contract' + +interface Artifact { + quality: number +} +const scenarios = ['t1', 't2', 't3', 's1', 'h1', 'h2'].map((id) => ({ id, kind: 'fixture' })) +const judge: JudgeConfig = { + name: 'quality', + dimensions: [{ key: 'quality', description: 'task quality' }], + score: ({ artifact }) => ({ + composite: artifact.quality, + dimensions: { quality: artifact.quality }, + notes: '', + }), +} +const gate: Gate = { + name: 'fixture', + decide: async () => ({ decision: 'ship', reasons: [], contributingGates: [] }), +} +const freeCost = { + totalCostUsd: 0, + costProvenance: { kind: 'observed' as const, usd: 0 }, + accountingComplete: true, + incompleteReasons: [], +} +const common = () => ({ + scenarios, + baselineSurface: 'BASE', + judge, + gate, + model: 'deterministic@2026-07-25', + expectUsage: 'off' as const, + storage: inMemoryCampaignStorage(), + runDir: 'mem://method-integrity', + budget: { holdoutScenarios: scenarios.slice(4) }, +}) + +describe('complete method measurement integrity', () => { + it('accepts an unchanged selected baseline without inventing candidate history', async () => { + const executed: string[] = [] + const result = await selfImprove({ + ...common(), + agent: async (_surface, scenario) => { + executed.push(scenario.id) + return { quality: 0.5 } + }, + method: { + name: 'unchanged', + optimize: async (input) => ({ winnerSurface: input.baselineSurface, cost: freeCost }), + }, + }) + expect(result.mode).toBe('method') + expect(result.winner.surface).toBe('BASE') + expect(result.gateDecision).toBe('hold') + expect(result.lift).toBe(0) + expect(executed.sort()).toEqual(['h1', 'h2']) + expect(result.raw.winnerOnHoldout).toBe(result.raw.baselineOnHoldout) + expect(result.raw).not.toHaveProperty('generations') + expect(result.provenance).not.toHaveProperty('baselineSearchComposite') + expect(result.provenance.schema).toBe('tangle.method-improvement') + }) + + it.each(['method', 'proposer'] as const)( + 'counts a shared unchanged campaign once in %s insight', + async (mode) => { + const options = common() + const result = await selfImprove({ + ...options, + budget: { ...options.budget, ...(mode === 'proposer' ? { generations: 1 } : {}) }, + agent: async (surface, _scenario, context) => { + const paid = await context.cost.runPaidCall({ + actor: 'worker', + model: 'fixture@2026-07-25', + execute: async () => ({ quality: surface === 'BASE' ? 1 : 0 }), + receipt: () => ({ + model: 'fixture@2026-07-25', + inputTokens: 10, + outputTokens: 5, + actualCostUsd: 1, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + }, + ...(mode === 'method' + ? { + method: { + name: 'unchanged', + optimize: async () => ({ winnerSurface: 'BASE', cost: freeCost }), + }, + } + : { proposer: { kind: 'fixed', propose: async () => ['BAD'] } }), + }) + expect(result.raw.winnerOnHoldout).toBe(result.raw.baselineOnHoldout) + expect(result.raw.baselineOnHoldout.cells).toHaveLength(2) + expect(result.insight?.n).toBe(2) + expect(result.insight?.costQuality.provenance?.observed).toEqual({ n: 2, totalUsd: 2 }) + expect(result.insight?.execution.tokenUsage.totals.input).toBe(20) + expect(result.insight?.lift).toBeUndefined() + expect(result.lift).toBe(0) + expect(result.gateDecision).toBe('hold') + }, + ) + + it('tests the method-selected winner directly without re-ranking on training cases', async () => { + const executed: Array<{ surface: string; id: string }> = [] + const result = await selfImprove({ + ...common(), + selectionScenarios: [scenarios[3]!], + agent: async (surface, scenario) => { + executed.push({ surface: String(surface), id: scenario.id }) + return { quality: surface === 'BASE' ? 0.6 : scenario.id.startsWith('t') ? 0 : 1 } + }, + method: { + name: 'selection-winner', + optimize: async (input) => { + expect(input.trainScenarios.map((scenario) => scenario.id)).toEqual(['t1', 't2', 't3']) + expect(input.selectionScenarios.map((scenario) => scenario.id)).toEqual(['s1']) + return { winnerSurface: 'WIN', cost: freeCost } + }, + }, + }) + expect(result.winner.surface).toBe('WIN') + expect(result.lift).toBeCloseTo(0.4) + expect(result.gateDecision).toBe('ship') + expect(executed).toHaveLength(4) + expect(executed.every((cell) => cell.id.startsWith('h'))).toBe(true) + expect(result.raw.method.winnerSurface).toBe('WIN') + }) + + it('retains external method spend and incomplete accounting in the total', async () => { + const result = await selfImprove({ + ...common(), + agent: async (surface) => ({ quality: surface === 'BASE' ? 0 : 1 }), + method: { + name: 'external', + optimize: async () => ({ + winnerSurface: 'WIN', + cost: { + totalCostUsd: 17, + costProvenance: { kind: 'estimated', usd: 17 }, + accountingComplete: false, + incompleteReasons: ['optimizer calls were not captured'], + }, + }), + }, + }) + expect(result.totalCostUsd).toBe(17) + expect(result.cost.accountingComplete).toBe(false) + expect(result.cost.incompleteReasons.join(' ')).toContain('optimizer calls were not captured') + expect(result.provenance.totalCostUsd).toBe(17) + expect(result.ledgerCost.totalCostUsd).toBe(0) + }) + + it('counts metered method and final spending exactly once', async () => { + const result = await selfImprove({ + ...common(), + agent: async (surface, _scenario, context) => { + const paid = await context.cost.runPaidCall({ + actor: 'worker', + model: 'fixture@2026-07-25', + execute: async () => ({ quality: surface === 'BASE' ? 0 : 1 }), + receipt: () => ({ + model: 'fixture@2026-07-25', + inputTokens: 10, + outputTokens: 5, + actualCostUsd: 1, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + }, + method: { + name: 'metered', + optimize: async (input) => { + const paid = await input.costLedger.runPaidCall({ + actor: 'optimizer', + model: 'fixture', + phase: 'method-search', + channel: 'driver', + execute: async () => 'WIN', + receipt: () => ({ + model: 'fixture', + inputTokens: 20, + outputTokens: 10, + actualCostUsd: 3, + }), + }) + if (!paid.succeeded) throw paid.error + return { + winnerSurface: paid.value, + cost: { + ...freeCost, + totalCostUsd: 3, + costProvenance: { kind: 'observed', usd: 3 }, + }, + } + }, + }, + }) + expect(result.totalCostUsd).toBe(7) + expect(result.ledgerCost.totalCostUsd).toBe(7) + expect(result.cost.accountingComplete).toBe(true) + expect(result.receipts).toHaveLength(5) + }) + + it.each([false, true])( + 'retains newly metered spend when a method reports zero (unknown usage: %s)', + async (usageUnknown) => { + const options = common() + const priorLedger = createRunCostLedger({ storage: options.storage, runDir: options.runDir }) + const prior = await priorLedger.runPaidCall({ + actor: 'prior-work', + model: 'fixture', + phase: 'prior', + channel: 'driver', + execute: async () => undefined, + receipt: () => ({ model: 'fixture', inputTokens: 1, outputTokens: 1, actualCostUsd: 11 }), + }) + if (!prior.succeeded) throw prior.error + const result = await selfImprove({ + ...options, + agent: async (surface) => ({ quality: surface === 'BASE' ? 0 : 1 }), + method: { + name: 'underreported', + optimize: async (input) => { + const paid = await input.costLedger.runPaidCall({ + actor: 'optimizer', + model: 'fixture', + phase: 'method-search', + channel: 'driver', + execute: async () => 'WIN', + receipt: () => ({ + model: 'fixture', + inputTokens: 20, + outputTokens: 10, + actualCostUsd: 3, + usageUnknown, + }), + }) + if (!paid.succeeded) throw paid.error + return { winnerSurface: paid.value, cost: freeCost } + }, + }, + }) + expect(result.totalCostUsd).toBe(3) + expect(result.ledgerCost.totalCostUsd).toBe(14) + expect(result.raw.method.cost.totalCostUsd).toBe(0) + expect(result.cost.accountingComplete).toBe(false) + expect(result.cost.incompleteReasons.join(' ')).toContain( + 'reported 0 USD below recorded 3 USD', + ) + if (usageUnknown) { + expect(result.cost.incompleteReasons.join(' ')).toContain('token usage unknown') + } + }, + ) + + it('does not mark equivalent floating-point cost sums as underreported', async () => { + const result = await selfImprove({ + ...common(), + agent: async (surface) => ({ quality: surface === 'BASE' ? 0 : 1 }), + method: { + name: 'rounded-sum', + optimize: async (input) => { + for (const usd of [0.1, 0.2, 0.3]) { + const paid = await input.costLedger.runPaidCall({ + actor: 'optimizer', + model: 'fixture', + phase: 'search', + channel: 'driver', + execute: async () => undefined, + receipt: () => ({ + model: 'fixture', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: usd, + }), + }) + if (!paid.succeeded) throw paid.error + } + return { + winnerSurface: 'WIN', + cost: { + ...freeCost, + totalCostUsd: 0.6, + costProvenance: { kind: 'observed', usd: 0.6 }, + }, + } + }, + }, + }) + expect(result.totalCostUsd).toBeCloseTo(0.6) + expect(result.cost.accountingComplete).toBe(true) + expect(result.cost.incompleteReasons).toEqual([]) + }) + + it('keeps deferred final measurements absent without executing search again', async () => { + let executions = 0 + const result = await selfImprove({ + ...common(), + budget: { holdout: 'deferred' }, + agent: async () => { + executions++ + return { quality: 1 } + }, + method: { + name: 'selected', + optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }), + }, + }) + expect(executions).toBe(0) + expect(result.baseline).toBeNull() + expect(result.winner.compositeMean).toBeNull() + expect(result.winner.surface).toBe('WIN') + expect(result.lift).toBeUndefined() + expect(result.gateDecision).toBe('hold') + expect(result.provenance.heldOutLift).toBeUndefined() + }) + + it('rejects a final comparison that loses one replica of each candidate case', async () => { + let failedCells = 0 + await expect( + compareOptimizationMethods({ + methods: [ + { name: 'partial', optimize: async () => ({ winnerSurface: 'WIN', cost: freeCost }) }, + ], + baselineSurface: 'BASE', + trainScenarios: scenarios.slice(0, 3), + selectionScenarios: [scenarios[3]!], + testScenarios: scenarios.slice(4), + judges: [judge], + reps: 2, + expectUsage: 'off', + storage: inMemoryCampaignStorage(), + runDir: 'mem://partial-replicas', + dispatchWithSurface: async (surface, _scenario, context) => { + if (surface === 'WIN' && context.rep === 1) { + failedCells++ + throw new Error('dispatch failed') + } + return { quality: surface === 'WIN' ? 1 : 0.5 } + }, + }), + ).rejects.toThrow(/final comparison is incomplete \(2\/4 designed cells scorable\)/) + expect(failedCells).toBe(2) + }) + + it('cannot ship a new native candidate using a previous candidate cache', async () => { + const options = common() + let executions = 0 + const agent = async (surface: unknown) => { + executions++ + return { quality: surface === 'GOOD' ? 1 : 0 } + } + const first = await selfImprove({ + ...options, + agent, + budget: { ...options.budget, generations: 1 }, + proposer: { kind: 'fixed', propose: async () => ['GOOD'] }, + }) + const beforeSecond = executions + const second = await selfImprove({ + ...options, + agent, + budget: { ...options.budget, generations: 1 }, + proposer: { kind: 'fixed', propose: async () => ['BAD'] }, + }) + expect(first.winner.surface).toBe('GOOD') + expect(second.winner.surface).toBe('BASE') + expect(second.gateDecision).toBe('hold') + expect(second.lift).toBe(0) + expect(executions - beforeSecond).toBe(4) + expect( + second.raw.generations[0]!.surfaces[0]!.campaign.cells.every((cell) => cell.cached === false), + ).toBe(true) + }) + + it('rejects a premeasured baseline from a different judge revision', async () => { + const train = scenarios.slice(0, 3) + const oldBaseline = await runCampaign({ + scenarios: train, + dispatch: async () => ({ quality: 1 }), + dispatchRef: surfaceDispatchRef('BASE'), + judges: [ + { + ...judge, + judgeVersion: 'old', + score: () => ({ composite: 0, dimensions: { quality: 0 }, notes: '' }), + }, + ], + expectUsage: 'off', + storage: inMemoryCampaignStorage(), + runDir: 'mem://old-judge', + }) + let executions = 0 + await expect( + runOptimization({ + scenarios: train, + baselineSurface: 'BASE', + premeasuredBaseline: { surfaceHash: surfaceHash('BASE'), campaign: oldBaseline }, + judges: [{ ...judge, judgeVersion: 'new' }], + proposer: { kind: 'fixed', propose: async () => ['WIN'] }, + dispatchWithSurface: async () => { + executions++ + return { quality: 0.5 } + }, + populationSize: 1, + maxGenerations: 1, + expectUsage: 'off', + storage: inMemoryCampaignStorage(), + runDir: 'mem://new-judge', + }), + ).rejects.toThrow(/premeasured baseline evaluator identity does not match/) + expect(executions).toBe(0) + }) + + it('does not label a native candidate mean as an estimated confidence interval', async () => { + const options = common() + const records: Array<{ composite: number | null; ci95: [number, number] | null }> = [] + const result = await selfImprove({ + ...options, + budget: { ...options.budget, generations: 2 }, + agent: async (surface, scenario) => ({ + quality: surface === 'BASE' ? 0 : scenario.id === 't1' ? 1 : 0, + }), + proposer: { + kind: 'fixed', + propose: async () => ['WIN'], + decide: ({ history }) => { + for (const generation of history) records.push(...generation.candidates) + return { stop: history.length > 0 } + }, + }, + }) + expect(result.raw.generations).toHaveLength(1) + expect(records).toHaveLength(1) + expect(records[0]).toMatchObject({ composite: 0.25, ci95: null }) + }) +})