fix(learning): preserve optimizer selection and measurement integrity

CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.174.0] — 2026-09-05
+
+### Changed
+
+- `selfImprove({ method })` executes the method directly and measures its selected surface on final cases.
+  It returns `SelfImproveMethodResult` with `mode: 'method'`, actual `raw.method` evidence, and `tangle.method-improvement` provenance.
+  It does not expose native `raw.generations` or `generationsExplored`.
+- `selfImprove({ proposer })` returns `SelfImproveProposerResult` with `mode: 'proposer'` and the existing native history.
+  `SelfImproveResult` is the union; consumers must narrow by `mode` before reading fields specific to one mode.
+- Method results with deferred holdout return `baseline: null`, `winner.compositeMean: null`, and no lift.
+  Method `cost` includes reported search and final spending; `ledgerCost` retains the actual receipt breakdown.
+- Premeasured native baselines must match the evaluator manifest.
+  Use `surfaceDispatchRef(baselineSurface, dispatchRef)` when creating the baseline campaign.
+  See [the result and cache contracts](docs/campaign-proposers.md#read-an-improvement-result).
+
+### Fixed
+
+- Complete methods can select the unchanged baseline without triggering native duplicate-candidate rejection.
+- Native ranking cannot override a complete method's selected winner or repeat its train and selection evaluations.
+- Method-reported spend and incomplete accounting remain in the total without duplicating metered costs.
+  Both improvement and method comparison reconcile each method's report against its attributed ledger receipts.
+- Final method comparisons reject missing replicas before averaging surviving scores.
+- Native search and final caches bind the candidate's surface content to execution identity.
+- Premeasured baselines from a different judge revision are refused before candidate execution.
+- Native candidate history reports `ci95: null` when uncertainty was not estimated.
+- An unchanged baseline's shared campaign contributes once to result analysis, including execution count, tokens, and cost.
+- `BehaviorExplorer` uses observed scores when assigning the next round's evaluations by behavior cell.
+  Scenario records retain their individual identities; allocation uses the pooled cell identity.
+
+---
+
 ## [0.173.3] — 2026-09-04
 
 ### Fixed

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.173.3"
+version = "0.174.0"
 description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -53,7 +53,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.173.3"
+    __version__ = "0.174.0"
 
 __all__ = [
     "Client",

docs/campaign-proposers.md

@@ -20,6 +20,48 @@ Use it when one surface must get better.
 Use it when two or more methods must be compared at equal budget.
 Runnable versions: [`examples/self-improve-optimizer`](../examples/self-improve-optimizer/) and [`examples/compare-optimization-methods`](../examples/compare-optimization-methods/).
 
+## Read An Improvement Result
+
+`selfImprove({ method })` executes the complete method once and measures its selected surface on final cases.
+The method may select the unchanged baseline; that result returns `gateDecision: 'hold'` and an empty diff.
+Agent Eval does not score train and selection cases again or choose a different surface after the method finishes.
+
+The result type has two modes:
+
+| Mode | Result | Search evidence | Cost |
+|---|---|---|---|
+| `proposer` | `SelfImproveProposerResult` | Native `raw.generations`, `generationsExplored`, and optional `searchHistory` | Shared `cost` ledger summary |
+| `method` | `SelfImproveMethodResult` | Actual `raw.method` and its optional `searchHistory` | Combined method and final `cost`; receipt breakdown in `ledgerCost` |
+
+Both types are exported from the package root and `/contract`.
+`SelfImproveResult` is their union; branch on `result.mode` before reading mode-specific fields.
+Calls with a concrete `method` or `proposer` infer the corresponding result type.
+Method mode has no native generation count or fabricated native search measurements.
+Its durable `method-provenance.json` uses schema `tangle.method-improvement` and records partition, measurement, and cost-receipt digests.
+Proposer mode retains `LoopProvenanceRecord`.
+
+When method holdout is deferred, `baseline` and `winner.compositeMean` are `null`, `lift` is absent, and the decision is `hold`.
+The selected surface remains available in `winner.surface`.
+Method cost preserves the larger of reported search spend and newly recorded search receipts, then adds final measurements without counting receipts twice.
+Underreported spending and incomplete receipts remain explicit; `raw.method.cost` retains the original report.
+Inspect `cost.accountingComplete` and `cost.incompleteReasons` before treating the known subtotal as complete spending.
+The shared dollar limit controls calls admitted through the cost ledger; arbitrary off-ledger callbacks must enforce their own spending limits.
+
+Native generation records report `ci95: null` because search does not estimate candidate uncertainty.
+Final comparisons retain their independently computed statistics.
+Every final case and replica must have complete execution and judge results before comparison.
+
+## Bind Cached Measurements To Their Evaluator
+
+Candidate surface content is part of native search and final measurement identity.
+Pass a stable `dispatchRef` for execution behavior outside that surface, such as the worker revision and tool configuration.
+Change it when that behavior changes; function names cannot identify captured state.
+Set `judgeVersion` when a judge's scoring behavior changes.
+
+To reuse `premeasuredBaseline` in proposer mode, measure the same train cases, seed, replicas, execution revision, and judges.
+The standalone campaign must use `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` from `/campaign`.
+Agent Eval refuses a prior baseline whose evaluator manifest differs.
+
 ## Adapt A Third-Party Text Optimizer
 
 `externalTextOptimizationMethod()` is the general adapter for a package that already owns text or component search.

examples/agent-engine-optimizer/index.ts

@@ -172,6 +172,9 @@ async function main() {
   })
 
   console.log(`Gate decision: ${result.gateDecision}`)
+  if (result.baseline === null || result.winner.compositeMean === null) {
+    throw new Error('This example requires a measured final comparison')
+  }
   console.log(`Baseline:      ${result.baseline.compositeMean.toFixed(3)} (held-out composite)`)
   console.log(`Winner:        ${result.winner.compositeMean.toFixed(3)} (held-out composite)`)
   console.log(`Lift:          ${result.lift === undefined ? 'not measured' : signed(result.lift)}`)

examples/self-improve-optimizer/README.md

@@ -72,6 +72,9 @@ Hard limits: `MAX_TOTAL_COST_USD` (default 10) caps the whole run and `GEPA_MAX_
 ## Read The Result
 
 The script prints the gate decision, the held-out baseline and winner composites, the lift, the total spend, and the baseline-to-winner diff.
+Method results use `mode: 'method'`; search evidence is in `raw.method` and optional `searchHistory`.
+They do not contain native `raw.generations` or `generationsExplored`.
+With deferred holdout, baseline and winner scores are `null` and no lift exists.
 Four held-out cases are wiring-scale, not statistical evidence: a `need_more_work` decision at this size is the gate refusing to claim significance, not a failure.
 Grow the case list and set `budget.reps` above 1 before treating the decision as a production threshold.
 

examples/self-improve-optimizer/index.ts

@@ -217,6 +217,9 @@ async function main() {
     `Backend:       ${MODEL} via ${BASE_URL} (${integrity.verdict}, ${records.length} calls)`,
   )
   console.log(`Gate decision: ${result.gateDecision}`)
+  if (result.baseline === null || result.winner.compositeMean === null) {
+    throw new Error('This example requires a measured final comparison')
+  }
   console.log(`Baseline:      ${result.baseline.compositeMean.toFixed(3)} (held-out composite)`)
   console.log(`Winner:        ${result.winner.compositeMean.toFixed(3)} (held-out composite)`)
   console.log(`Lift:          ${result.lift === undefined ? 'not measured' : signed(result.lift)}`)

examples/selfimprove-quickstart/index.ts

@@ -113,6 +113,7 @@ async function main() {
   })
 
   const result = await evalKit.improve()
+  if (result.mode !== 'proposer') throw new Error('This example requires a native proposer result')
 
   const i = result.insight
   console.log('Improvement result')

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.173.3",
+  "version": "0.174.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

src/analyst/benchmark-implementation.ts

@@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 ])
 
 export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 =
-  'ff7e12bb2457745febc8d525bb099365473eaae8b9bedde5dd5fbfbe6d86e470'
+  '10c77bd9ce4d896811395f8b623216f51bd03dab9ea209cb58e1d8d7b78acb7a'
 
 /** The published benchmark evidence was produced at this package version, by
  * the retired one-shot direct runner, before trace analysts moved to the

src/bounded-process.test.ts

@@ -76,7 +76,7 @@ describe.skipIf(!posixOnly)('runBoundedProcess kills the whole process group', (
     const res = await runBoundedProcess({
       command: PGID_PID_THEN_BACKGROUND_SLEEP,
       shell: 'bash',
-      timeoutMs: 100,
+      timeoutMs: 1000,
     })
     const elapsed = Date.now() - started
 
@@ -218,22 +218,36 @@ describe.skipIf(!posixOnly)('runBoundedProcess abort handling', () => {
 
   it('an abort mid-run kills the group and reports killedBySignal', async () => {
     const controller = new AbortController()
-    const timer = setTimeout(() => controller.abort(), 300)
-    const started = Date.now()
-    const res = await runBoundedProcess({
-      command: PGID_PID_THEN_BACKGROUND_SLEEP,
+    const ready = join(dir, 'ready')
+    const running = runBoundedProcess({
+      command: 'echo $$; sleep 60 & echo $!; : > "$BOUNDED_PROCESS_READY"; wait',
+      env: { BOUNDED_PROCESS_READY: ready },
       shell: 'bash',
       signal: controller.signal,
       timeoutMs: 60_000,
     })
-    clearTimeout(timer)
+    const settled = Promise.allSettled([running])
 
-    expect(res.killedBySignal).toBe(true)
-    expect(res.killedByTimeout).toBe(false)
-    expect(res.exitCode).not.toBe(0)
-    expect(Date.now() - started).toBeLessThan(10_000)
+    try {
+      const readyDeadline = Date.now() + 5_000
+      while (!existsSync(ready) && Date.now() < readyDeadline) {
+        await new Promise((resolve) => setTimeout(resolve, 20))
+      }
+      expect(existsSync(ready)).toBe(true)
+      const abortedAt = Date.now()
+      controller.abort()
+      const res = await running
 
-    await expectTreeGone(res.stdout)
+      expect(res.killedBySignal).toBe(true)
+      expect(res.killedByTimeout).toBe(false)
+      expect(res.exitCode).not.toBe(0)
+      expect(Date.now() - abortedAt).toBeLessThan(10_000)
+
+      await expectTreeGone(res.stdout)
+    } finally {
+      controller.abort()
+      await settled
+    }
   }, 30_000)
 })
 
@@ -344,7 +358,7 @@ describe('runBoundedProcess runs an argument vector with no shell', () => {
       const res = await runBoundedProcess({
         command: 'bash',
         args: ['-c', PGID_PID_THEN_BACKGROUND_SLEEP],
-        timeoutMs: 100,
+        timeoutMs: 1000,
       })
       expect(res.killedByTimeout).toBe(true)
       expect(res.exitCode).not.toBe(0)
@@ -383,7 +397,7 @@ describe('runBoundedProcess runs an argument vector with no shell', () => {
       // The abort path shares `killAndDrain` with the shell form, but the flags and
       // the forced non-zero exit had no coverage for an argv caller.
       const controller = new AbortController()
-      setTimeout(() => controller.abort(), 150)
+      setTimeout(() => controller.abort(), 1000)
       const res = await runBoundedProcess({
         command: 'bash',
         args: ['-c', PGID_PID_THEN_BACKGROUND_SLEEP],

src/campaign/campaign-manifest.ts

@@ -6,9 +6,9 @@
 import { contentHash } from '../verdict-cache'
 import type { DispatchFn, JudgeConfig, Scenario } from './types'
 
-export function computeManifestHash(input: {
-  scenarios: Scenario[]
-  judges: JudgeConfig<unknown>[]
+export function computeManifestHash<TScenario extends Scenario, TArtifact>(input: {
+  scenarios: TScenario[]
+  judges: JudgeConfig<TArtifact, TScenario>[]
   dispatchRef: string
   seed: number
   reps: number
@@ -26,7 +26,9 @@ export function computeManifestHash(input: {
   })
 }
 
-function judgeVersionFor(judge: JudgeConfig<unknown>): string {
+function judgeVersionFor<TScenario extends Scenario, TArtifact>(
+  judge: JudgeConfig<TArtifact, TScenario>,
+): string {
   if (judge.judgeVersion !== undefined) {
     const version = judge.judgeVersion.trim()
     if (version.length === 0) {

src/campaign/coverage.ts

@@ -1,5 +1,10 @@
 import { contentHash } from '../verdict-cache'
-import type { CampaignCellResult, CampaignScenarioIdentity, Scenario } from './types'
+import type {
+  CampaignCellResult,
+  CampaignResult,
+  CampaignScenarioIdentity,
+  Scenario,
+} from './types'
 
 export interface CampaignCoverage {
   complete: boolean
@@ -199,3 +204,19 @@ function designedCellIds<TScenario extends Scenario>(
   }
   return ids
 }
+
+/** Require the complete designed denominator before a final comparison. */
+export function assertCompleteCampaign<TArtifact, TScenario extends Scenario>(
+  campaign: CampaignResult<TArtifact, TScenario>,
+  scenarios: readonly TScenario[],
+  reps: number,
+  requireJudgeScore: boolean,
+  label: string,
+): void {
+  const coverage = campaignCoverage(campaign.cells, scenarios, reps, requireJudgeScore)
+  if (!coverage.complete) {
+    throw new Error(
+      `${label} is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to compare unequal results.`,
+    )
+  }
+}

src/campaign/index.ts

@@ -440,6 +440,7 @@ export {
   componentSurfaceIdentityMaterial,
   renderSurfaceDiff,
   surfaceContentHash,
+  surfaceDispatchRef,
   surfaceHash,
 } from './surface-identity'
 export {