From a76659fcb8792ddf55e8c827cb7ddbf93c3fbf64 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 04:13:36 -0400 Subject: [PATCH 01/94] Integrate stochastic histories with sequential density weights --- docs/uncertainty/implementation-progress.md | 2 + .../stochastic-future-integration.md | 34 +++++ .../code_sim_learning/inference_sequential.py | 115 ++++++++++++++++ .../test_inference_sequential.py | 130 ++++++++++++++++++ 4 files changed, 281 insertions(+) create mode 100644 predicators/code_sim_learning/inference_sequential.py create mode 100644 tests/code_sim_learning/test_inference_sequential.py diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2163030ba..53928b2c6 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -9,6 +9,8 @@ Forty functional tests and focused static checks pass. Its first native Balloons diagnostic reproduces the reference trajectory but fails all four numerical comparisons: eight and 64 complete paths remain dominated by a single contribution. This is an unresolved integration problem, so these scores are not used to compare estimators or change agent behavior. A native factor audit reproduces ten selected paths exactly and attributes their output-score variation to the box and attached balloon positions; robot factors remain invariant. +The subsequent offline sequential integrator retains complete histories and block density normalizers, with explicit ancestry diagnostics. +Forty-four functional tests and focused static checks pass; four native Balloons integrations are running on compute nodes to assess whether it reduces the observed concentration. The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. diff --git a/docs/uncertainty/stochastic-future-integration.md b/docs/uncertainty/stochastic-future-integration.md index 091c7f3a8..9a5ba566a 100644 --- a/docs/uncertainty/stochastic-future-integration.md +++ b/docs/uncertainty/stochastic-future-integration.md @@ -92,3 +92,37 @@ Evaluate a position-guided or sequential integration method with explicit densit Any proposed method must preserve the joint future law, exact observations, native replay contract and numerical failure reporting. A reliable conditional-path integral still needs integration over an assessed joint parameter and prefix-state posterior. Matched legacy predictions, initial-state and prior ablations, saved planning decisions and the live-agent validation gates remain required before replacing the incumbent. + +## Sequential history integration + +The offline `inference_sequential` component extends complete histories through fixed observation blocks and performs multinomial resampling between blocks. +Each extension samples a normalized conditional proposal and returns the incremental target/proposal factor for that block, including its exact-observation density factors. +The product of block-average weights estimates the complete future density. +This retains temporal dependence; it is not a product of separately fitted marginal forecasts. +No resampling is performed after the last block, so the terminal histories retain their normalized weights. + +The implementation records every block normalizer, effective contribution count and surviving original ancestry count. +Resampled terminal histories are correlated, so their spread cannot be used as an iid error estimate of the normalizer. +A high terminal effective count does not undo earlier ancestry loss. +Independent complete integrations and budget comparisons remain necessary for numerical assessment. +Zero-support extensions retain zero weight, and complete sampled-support loss returns an explicit unavailable result. +Replay exceptions and nonfinite factors abort the computation instead of deleting or retrying individual extensions. + +Callbacks receive private copies of their parent histories, and returned histories are copied before retention to avoid mutation across siblings. +History payloads describe reconstructible paths; they must not contain live simulator handles. +The Balloons adapter reconstructs a fresh native world from the same validated prefix for every extension, preserving hidden dynamics and the cached-link observation phase. +This has a substantial cost which must be measured rather than hidden by counting only newly extended actions. + +The component reference enumerates all histories of a two-state Markov model to check both the joint observation density and terminal posterior probability across independent integration runs. +Additional tests check an observation-guided proposal with its retained density correction, exact-event zero weights, mutable-parent isolation, ancestry collapse, numerical overflow and interrupted replay. + +Compute job `22654085` passed all 44 functional tests, focused mypy and pylint, and pinned formatter checks. +The initial check found missing type annotations and a reused variable name; the next found three test-only lint issues. +All attempts and final source hashes are retained in `logs/uncertainty_sequential_checks_v3_20260912` and its preceding check bundles. + +Native array `22654086` evaluates both the generated701 and recorded 32-action futures with independent seeds 911 and 912, using 32 histories and eight four-action blocks. +The plan, worker and overlays are frozen in `logs/uncertainty_balloons_sequential_future_20260912`. +Each run first reproduces the full 235-action reference factor, and every path extension must reproduce the original 64-action prefix exactly. +The terminal audit repeats one retained full history and checks that its sum of block factors equals its direct whole-future factor. +Each completed run requires 21,419 native actions, including reconstruction and terminal checks, rather than just its 1,024 newly sampled future actions. +These pilots remain numerical diagnostics of a fixed full-training-selected witness, not prefix-only inference, new agent results or a comparison of model quality. diff --git a/predicators/code_sim_learning/inference_sequential.py b/predicators/code_sim_learning/inference_sequential.py new file mode 100644 index 000000000..f3f51b7b6 --- /dev/null +++ b/predicators/code_sim_learning/inference_sequential.py @@ -0,0 +1,115 @@ +"""Sequential integration of complete stochastic histories, for offline use. + +Fixed-size multinomial resampling operates between declared blocks. Each +block retains its conditional target/proposal density ratio, including +exact observation factors. The accumulated normalizer estimates a joint +history density; terminal weights alone do not provide that density or +certify a usable posterior. Independent runs are needed to assess error. +""" +from __future__ import annotations + +import copy +import math +from dataclasses import dataclass +from typing import Callable, Generic, List, Literal, Optional, Tuple, TypeVar + +import numpy as np + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError + +History = TypeVar("History") + + +@dataclass(frozen=True) +class SequentialPathIntegral(Generic[History]): + """A joint density estimate with concentration and ancestry diagnostics. + + Histories are owned by this result, but their payloads may be + mutable. Within-run paths share ancestors and are not independent + estimates. A large terminal ESS cannot erase past collapse or + missing support. + """ + log_density: float + log_increments: Tuple[float, ...] + effective_terms: Tuple[float, ...] + surviving_ancestors: Tuple[int, ...] + histories: Tuple[History, ...] + weights: Tuple[float, ...] + status: Literal["finite_estimate", "no_sample_support"] + + +def integrate_sequential_paths(advance: Callable[ + [Optional[History], int, np.random.Generator], Tuple[History, float]], + count: int, stages: int, + seed: int) -> SequentialPathIntegral[History]: + """Extend histories and multiply block normalizer estimates. + + advance(None, 0, rng) starts from the same fixed supported prefix. + Later calls receive a deep copy of a resampled parent history and + the zero-based block index. A callback must preserve that history, + draw its extension from a normalized conditional proposal and return + only the new block's log target/proposal factor. Deterministic + history reconstruction may use a native simulator; live simulator + handles are not suitable history payloads. + + Resampling occurs at every nonfinal boundary under the normalized + block factors, including zero-support paths with zero probability. + Independent random draws extend repeated parents. Exceptions and + invalid factors abort, never discard or retry a failed callback. + No sampled support does not prove a model is inconsistent. + """ + for name, argument, minimum in (("count", count, 2), ("stages", stages, 1), + ("seed", seed, 0)): + if not isinstance(argument, int) or isinstance(argument, bool) or \ + argument < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + rng = np.random.default_rng(seed) + parents: Tuple[Optional[History], ...] = (None, ) * count + ancestors = np.arange(count) + increments: List[float] = [] + effective: List[float] = [] + lineage: List[int] = [] + for stage in range(stages): + children = [] + factors = [] + for parent in parents: + child, factor = advance(copy.deepcopy(parent), stage, rng) + factor = float(factor) + if math.isnan(factor) or factor == math.inf: + raise ConditioningNumericalError("Invalid block log factor") + children.append(copy.deepcopy(child)) + factors.append(factor) + peak = max(factors) + if peak == -math.inf: + return SequentialPathIntegral(-math.inf, + tuple(increments + [-math.inf]), + tuple(effective + [0.]), + tuple(lineage + [0]), (), (), + "no_sample_support") + scaled = [math.exp(factor - peak) for factor in factors] + total = math.fsum(scaled) + weights = np.asarray(scaled) / total + # Explicit final normalization for the categorical sampling API. + weights /= weights.sum() + increment = peak + math.log(total / count) + if not math.isfinite(increment): + raise ConditioningNumericalError("Block normalizer overflow") + increments.append(increment) + effective.append(total * total / math.fsum(value * value + for value in scaled)) + lineage.append(len(set(ancestors[weights > 0].tolist()))) + if stage + 1 < stages: + indices = rng.choice(count, size=count, p=weights) + parents = tuple(children[i] for i in indices) + ancestors = ancestors[indices] + try: + value = math.fsum(increments) + except OverflowError as exc: + raise ConditioningNumericalError("Joint normalizer overflow") from exc + if not math.isfinite(value): + raise ConditioningNumericalError("Joint normalizer overflow") + return SequentialPathIntegral(value, tuple(increments), tuple(effective), + tuple(lineage), tuple(children), + tuple(float(w) for w in weights), + "finite_estimate") diff --git a/tests/code_sim_learning/test_inference_sequential.py b/tests/code_sim_learning/test_inference_sequential.py new file mode 100644 index 000000000..14511f94c --- /dev/null +++ b/tests/code_sim_learning/test_inference_sequential.py @@ -0,0 +1,130 @@ +"""Sequential density integration versus exact joint history references.""" +import itertools +import math + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_path_integral import \ + summarize_path_integral +from predicators.code_sim_learning.inference_sequential import \ + integrate_sequential_paths + + +def test_joint_markov_density_and_terminal_measure(): + """Enumeration checks dependence, evidence and terminal weighted paths.""" + observations = (0, 1, 1, 0, 1, 0) + transition = np.array([[.85, .15], [.3, .7]]) + emission = np.array([[.8, .2], [.1, .9]]) + evidence = 0. + final_one = 0. + for path in itertools.product((0, 1), repeat=len(observations)): + density = .5 + for step, (state, reading) in enumerate(zip(path, observations)): + density *= emission[state, reading] + if step: + density *= transition[path[step - 1], state] + evidence += density + final_one += density * path[-1] + + def advance(parent, stage, rng): + probability = .5 if parent is None else transition[parent[-1], 1] + state = int(rng.random() < probability) + history = (() if parent is None else parent) + (state, ) + return history, math.log(emission[state, observations[stage]]) + + results = [ + integrate_sequential_paths(advance, 512, len(observations), seed) + for seed in range(24) + ] + summary = summarize_path_integral(tuple(r.log_density for r in results)) + assert math.exp(summary.log_density) == pytest.approx(evidence, rel=.02) + means = [ + sum(w * history[-1] for history, w in zip(r.histories, r.weights)) + for r in results + ] + assert np.mean(means) == pytest.approx(final_one / evidence, abs=.015) + assert results[0] == integrate_sequential_paths(advance, 512, + len(observations), 0) + assert all(len(h) == len(observations) for h in results[0].histories) + + +def test_proposal_correction_and_zero_paths(): + """A guided proposal retains its normalizer and zero event outcomes.""" + target, proposal = .2, .7 + + def advance(parent, stage, rng): + del parent + state = int(rng.random() < proposal) + # Exact event observes state=1 in each of three independent blocks. + return (stage, state), math.log(target / proposal) if state else \ + -math.inf + + estimates = [ + integrate_sequential_paths(advance, 1000, 3, seed).log_density + for seed in range(12) + ] + estimate = summarize_path_integral(tuple(estimates)) + assert math.exp(estimate.log_density) == pytest.approx(target**3, rel=.03) + + +def test_ancestry_and_mutable_parent_isolation(): + """Sibling extensions cannot mutate a parent or conceal prior collapse.""" + calls = [0, 0] + + def advance(parent, stage, rng): + del rng + index = calls[stage] + calls[stage] += 1 + if stage == 0: + return [index], 0. if index == 0 else -math.inf + assert parent == [0] + parent.append(index) + return parent, 10000. + + result = integrate_sequential_paths(advance, 8, 2, 10) + assert result.log_density == pytest.approx(10000. - math.log(8)) + assert result.effective_terms == (1., 8.) + assert result.surviving_ancestors == (1, 1) + assert result.histories == tuple([0, i] for i in range(8)) + assert result.weights == (.125, ) * 8 + + +def test_no_support_and_numerical_errors(): + """Unseen support is distinct from an invalid or interrupted replay.""" + + def absent(parent, stage, rng): + del parent, stage + return (), 0. if rng.random() < 1e-12 else -math.inf + + result = integrate_sequential_paths(absent, 8, 4, 10) + assert result.status == "no_sample_support" + assert not result.histories and not result.weights + assert result.log_increments == (-math.inf, ) + for invalid in (math.inf, math.nan): + with pytest.raises(ConditioningNumericalError): + integrate_sequential_paths(lambda *args, value=invalid: + ((), value), + 8, + 2, + 0) + calls = [] + + def interrupted(parent, stage, rng): + del parent, stage, rng + calls.append(1) + if len(calls) == 3: + raise RuntimeError("native replay failed") + return (), 0. + + with pytest.raises(RuntimeError, match="native replay failed"): + integrate_sequential_paths(interrupted, 8, 2, 0) + assert len(calls) == 3 + with pytest.raises(ConditioningNumericalError, match="overflow"): + integrate_sequential_paths(lambda *args: ((), 1e308), 2, 2, 0) + for count, stages, seed in ((1, 1, 0), (2, 0, 0), (2, 1, -1), (True, 1, + 0)): + with pytest.raises(ValueError): + integrate_sequential_paths(absent, count, stages, seed) From e4335d3b4a701e22498aa745c3d4225d3c0affb5 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 04:18:58 -0400 Subject: [PATCH 02/94] Record failed density diagnostics and continued inference runs --- .../balloons-composed-inference.md | 12 ++++++++ docs/uncertainty/domino-joint-inference.md | 29 +++++++++++++++++-- docs/uncertainty/fan-joint-inference.md | 10 +++++++ docs/uncertainty/implementation-progress.md | 7 +++-- .../stochastic-future-integration.md | 16 ++++++++++ 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/uncertainty/balloons-composed-inference.md b/docs/uncertainty/balloons-composed-inference.md index e12f555ad..90a811326 100644 --- a/docs/uncertainty/balloons-composed-inference.md +++ b/docs/uncertainty/balloons-composed-inference.md @@ -204,6 +204,18 @@ The original geometry normalizer and fixed initial robot-output factors cancel f Their numerical availability remains unevaluated until the separate assessment is complete. No resulting parameter distribution is routed to the acting agent by this experiment. +## Checkpoint continuation, September 13 + +Both original array tasks `22649168_0` and `22649168_1` reached their eight-hour Slurm limits with terminal `TIMEOUT` states. +Numerical seeds 300 and 301 retained complete-stage checkpoints at stages 14 and 13, after 7,922 and 7,601 evaluations respectively. +Their old progress reports still say running because the scheduler terminated the process; those reports do not override the terminal job states. + +Array `22671041` continues those same numerical seeds from the saved sampler populations and random states, using the identical frozen worker, target, proposal and 32,896-evaluation cap. +Each continuation has another eight-hour allocation on node1391; the additional allocation is part of the total inference cost and is not a new experiment seed. +The original checkpoints were copied and hashed before continuation in `logs/uncertainty_balloons_joint_pilot_20260912/continuation-20260913`. +The first continuation has confirmed `resumed=true` with 7,922 saved evaluations; the other task is queued for resources at this update. +No completed Balloons posterior or numerical adequacy result is available from these fits yet. + ## Unconditional future-generation audit Compute job `22652531` validates the generation half of the stochastic forecast using the same mapped support witness and frozen learned program as the joint sampler preflight. diff --git a/docs/uncertainty/domino-joint-inference.md b/docs/uncertainty/domino-joint-inference.md index 344fd2b6f..426bc6030 100644 --- a/docs/uncertainty/domino-joint-inference.md +++ b/docs/uncertainty/domino-joint-inference.md @@ -114,16 +114,41 @@ It fits the same 64-action development prefix under fixed priors and an identifi Run `22637359_100` reached its four-hour Slurm limit before producing a completed sampler result. Its last saved progress report recorded 9,536 target calls; the scheduler reports `TIMEOUT`, not an agent outcome or a completed posterior. No sampler checkpoint existed in that frozen worker, so its best candidate cannot serve as a continuation state. -The other numerical seed, `22637359_101`, remains a separate running fit with an eight-hour allocation. +The other numerical seed, `22637359_101`, used an eight-hour allocation and subsequently completed. Replacement `22649657_100` restarts numerical seed 100 from the original prior with the same data, priors, temperature schedule and evaluation budget. It adds the tested scalar likelihood optimization, [stage checkpoints](sampler-checkpoints.md) and an eight-hour allocation on the same AMD EPYC 7542 worker node. The runtime identity changes to identify those source changes; the statistical model and sampler configuration do not change. Its startup checks compare complete old/new likelihoods on two replayed candidates before fitting. One pair retains zero support, and the finite pair matches exactly at `8141.576094195281` on this AMD runtime. -The retry is running and has saved its initialized population checkpoint after 64 evaluations. +The retry saved its initialized population after 64 evaluations and subsequently completed all 32 stages. All sixty recorded finite-initial-base entries also match the timed-out run's entries exactly. The [retry manifest](../../logs/uncertainty_domino_conditioned_checkpoint_20260912/plan.json) retains the timeout reason and hashes the old likelihood source used for the paired check. Attempt files retain per-attempt counters; the sampler result and checkpoint retain the cumulative numerical evaluation count. This recovery does not establish posterior adequacy, convergence or an improvement in agent performance. + +### Completed numerical pilots, September 13 + +Both conditioned-base fits now have terminal `COMPLETED` job states and complete, structurally valid 64-row weighted sampler results at temperature 1. +Numerical seed 100 used 14,318 target evaluations in its replacement attempt, and seed 101 used 14,038. +Their completed attempts took approximately 5 hours 8 minutes and 5 hours 32 minutes; seed 100 also incurred the earlier four-hour timed-out attempt. +Both retained only one original ancestor after 11 resampling events. +The minimum recorded effective sample sizes were 18.32 and 20.33, respectively. + +Their empirical parameter summaries disagree substantially: + +| Parameter | Seed 100: 5th / 50th / 95th percentile | Seed 101: 5th / 50th / 95th percentile | +| --- | --- | --- | +| Lateral friction | 0.11496 / 0.17337 / 0.21346 | 0.42817 / 0.65932 / 0.87769 | +| Restitution | 0.25945 / 0.41405 / 0.55124 | 0.72363 / 0.87345 / 0.89240 | +| Rolling friction | 0.003084 / 0.003496 / 0.006734 | 0.000129 / 0.000129 / 0.002509 | +| Spinning friction | 0.01137 / 0.02051 / 0.02760 | 0.03731 / 0.04631 / 0.06521 | +| Mass | 0.30506 / 0.40014 / 0.53333 | 0.49608 / 0.64470 / 0.78762 | + +These are summaries of the numerical populations, not validated credible intervals. +Shared data, sensor, program, prior and sampler configuration were verified, along with complete finite samples and normalized nonnegative weights. +The runtime-source difference and its earlier exact likelihood parity checks remain part of the comparison's provenance. +Completion does not resolve the disagreement or establish trustworthy posterior coverage. +Predictive stability and a defensible exploration of the joint parameter/initial-state distribution remain required before any posterior publication or legacy comparison claim. +The source hashes and completion checks are retained in `logs/uncertainty_domino_conditioned_checkpoint_20260912/completion-comparison-20260913.json`. diff --git a/docs/uncertainty/fan-joint-inference.md b/docs/uncertainty/fan-joint-inference.md index 127b85909..95c4a618b 100644 --- a/docs/uncertainty/fan-joint-inference.md +++ b/docs/uncertainty/fan-joint-inference.md @@ -141,3 +141,13 @@ The initially queued unguarded recovery submissions `22650769` and `22650770` we The recovery manifest, gate checks and authoritative scheduler evidence are in the same bundle. Original jobs remain live and unchanged; no full replacement is running concurrently. Original attempts, initialization checks and any eventual recovery all belong to the same two numerical seeds, not additional agent outcomes or independent posterior replications. + +### Recovery activation, September 13 + +Slurm now confirms both original tasks `22643258_0` and `22643258_1` timed out at their eight-hour limits. +Their guarded recovery jobs `22650786_0` and `22650787_1` activated after those terminal failures and are running on node1412. +They restart the same numerical seeds from the verified stage-zero populations after 64 evaluations; the original eight-hour workers did not save their later sampler populations. +The recovery allocations are twelve hours each, and original-attempt costs remain part of the total. +They are not additional independent fits or agent outcomes. +The earlier descriptions of live original jobs and pending recoveries above record the state when the safeguards were implemented. +No completed or numerically assessed Fan posterior is claimed by this update. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 53928b2c6..14e183d59 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -1,6 +1,6 @@ # Uncertainty simplification: implementation progress -Updated September 12, 2026. +Updated September 13, 2026. This tracks implementation of the [simplification proposal](simplification-proposal.md). The incumbent estimator remains the production default. @@ -10,7 +10,10 @@ Its first native Balloons diagnostic reproduces the reference trajectory but fai This is an unresolved integration problem, so these scores are not used to compare estimators or change agent behavior. A native factor audit reproduces ten selected paths exactly and attributes their output-score variation to the box and attached balloon positions; robot factors remain invariant. The subsequent offline sequential integrator retains complete histories and block density normalizers, with explicit ancestry diagnostics. -Forty-four functional tests and focused static checks pass; four native Balloons integrations are running on compute nodes to assess whether it reduces the observed concentration. +Forty-four functional tests and focused static checks pass. +All four native sequential integrations completed with exact replay and factor accounting, but both independent-run comparisons fail the declared density-stability diagnostic and retain only one or two original ancestors. +The two [Domino conditioned-base pilots](domino-joint-inference.md#completed-numerical-pilots-september-13) also completed, with strongly different parameter summaries and one original ancestor each; numerical adequacy remains unestablished. +Fan recovery jobs are active after confirmed allocation timeouts, and the two Balloons fits have been submitted for continuation from their saved complete-stage checkpoints under the same numerical budgets. The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. diff --git a/docs/uncertainty/stochastic-future-integration.md b/docs/uncertainty/stochastic-future-integration.md index 9a5ba566a..37e37d1b3 100644 --- a/docs/uncertainty/stochastic-future-integration.md +++ b/docs/uncertainty/stochastic-future-integration.md @@ -126,3 +126,19 @@ Each run first reproduces the full 235-action reference factor, and every path e The terminal audit repeats one retained full history and checks that its sum of block factors equals its direct whole-future factor. Each completed run requires 21,419 native actions, including reconstruction and terminal checks, rather than just its 1,024 newly sampled future actions. These pilots remain numerical diagnostics of a fixed full-training-selected witness, not prefix-only inference, new agent results or a comparison of model quality. + +All four tasks completed and passed the full-prefix, terminal replay, block-factor decomposition and artifact checks. +The numerical consistency result is negative: + +| Evaluated future | Seed 911 log density | Seed 912 log density | Between-seed difference | Final original ancestors, seeds 911 / 912 | +| --- | ---: | ---: | ---: | --- | +| Generated701 | 4950.40744 | 4960.59650 | 10.18906 | 1 / 2 | +| Recorded | -2573.78777 | -2548.09901 | 25.68876 | 1 / 1 | + +Both comparisons fail the predeclared 0.2 log-density-difference diagnostic. +Individual block effective counts sometimes improve, but that does not establish an adequate complete-history integral. +The retained ancestry and independent-run disagreement show why terminal particle counts alone are insufficient. +The four jobs took approximately 6.5, 10.2, 8.6 and 6.6 allocated minutes and performed 85,676 native actions in total. +This pilot uses more replay work per integration than the earlier whole-path diagnostic; it does not demonstrate either accuracy or cost superiority. +Verified outputs and a repeatable artifact checker are in the native bundle's `verification.json` and `verify_report.py`. +Position-guided proposals or another demonstrated variance reduction remain necessary before claiming a reliable stochastic forecast score. From 34a914bab8929441a55beef1bc52e6f065acc66c Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 04:36:54 -0400 Subject: [PATCH 03/94] Add corrected defensive proposals for conditional velocity directions --- docs/uncertainty/implementation-progress.md | 2 + .../stochastic-future-integration.md | 44 +++++++ .../code_sim_learning/inference_guidance.py | 111 ++++++++++++++++ .../test_inference_guidance.py | 118 ++++++++++++++++++ 4 files changed, 275 insertions(+) create mode 100644 predicators/code_sim_learning/inference_guidance.py create mode 100644 tests/code_sim_learning/test_inference_guidance.py diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 14e183d59..313465d84 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,8 @@ A native factor audit reproduces ten selected paths exactly and attributes their The subsequent offline sequential integrator retains complete histories and block density normalizers, with explicit ancestry diagnostics. Forty-four functional tests and focused static checks pass. All four native sequential integrations completed with exact replay and factor accounting, but both independent-run comparisons fail the declared density-stability diagnostic and retain only one or two original ancestors. +The next offline proposal mixes original and position-guided velocity directions while retaining the full mixture correction, so it targets the same probability model. +Fifty-one functional tests and focused static checks pass; its matched native Balloons pilot is submitted for numerical evaluation. The two [Domino conditioned-base pilots](domino-joint-inference.md#completed-numerical-pilots-september-13) also completed, with strongly different parameter summaries and one original ancestor each; numerical adequacy remains unestablished. Fan recovery jobs are active after confirmed allocation timeouts, and the two Balloons fits have been submitted for continuation from their saved complete-stage checkpoints under the same numerical budgets. diff --git a/docs/uncertainty/stochastic-future-integration.md b/docs/uncertainty/stochastic-future-integration.md index 37e37d1b3..8d84aaf79 100644 --- a/docs/uncertainty/stochastic-future-integration.md +++ b/docs/uncertainty/stochastic-future-integration.md @@ -142,3 +142,47 @@ The four jobs took approximately 6.5, 10.2, 8.6 and 6.6 allocated minutes and pe This pilot uses more replay work per integration than the earlier whole-path diagnostic; it does not demonstrate either accuracy or cost superiority. Verified outputs and a repeatable artifact checker are in the native bundle's `verification.json` and `verify_report.py`. Position-guided proposals or another demonstrated variance reduction remain necessary before claiming a reliable stochastic forecast score. + +## Defensive position guidance + +The offline `inference_guidance` component proposes a velocity direction conditional on the same exact speed using a mixture of the original direction law and a guided law. +Both components are normalized conditional Gaussian direction distributions on the speed sphere. +The guide changes the proposal mean, not the original velocity model or its discrepancy scale. + +For original radial density `p(s)`, original conditional direction density `p(d | s)`, guided density `g(d | s)` and guide probability `a`, the retained factor is: + +```text +p(s) * p(d | s) / ((1 - a) * p(d | s) + a * g(d | s)) +``` + +The denominator is the complete mixture density, including both components regardless of which generated the draw. +The original component has positive probability, so the directional importance ratio cannot exceed `1 / (1 - a)`. +This bounds a single direction correction; it does not bound the variance of a complete history or establish adequate sampling. +Zero observed speed retains the original rest atom and does not introduce direction coordinates. +Disabled guidance and identical proposal means preserve the original conditional draw and radial factor exactly. + +The native proposal uses the next noisy Cartesian readings of the box and currently attached, unpopped balloons. +For each axis it adds `velocity_sigma² * action_dt * sum((next_reading - current_prediction) / sensor_variance)` to the predicted box velocity. +This is the direction proposal obtained from an approximate rigid translation over one action with isotropic position sensors. +Contacts, rotation and forces can invalidate that approximation, so the native transition and complete observation model still determine the corrected target weight. +The declared action duration comes from the simulator's actual fixed time step and configured substeps. +The pilot uses guide probability 0.8 and disables guidance at the last evaluated step, which has no subsequent position reading. + +Guidance is only used while evaluating the density of an observed future. +Its observation-guided paths must never enter unconditional forecast generation, prefix parameter fitting or the acting agent's state. +The scored future is already fixed when the proposal is constructed, and every proposal factor is retained exactly once. +Three independent uniform coordinates per moving extension preserve both the component selection and direction for deterministic history reconstruction. + +The density calculation uses a centered Gaussian log ratio and a compensated sum of radial terms. +A direct API reproduction exposed cancellation in the first draft: with equal-strength opposite means, large common radial terms erased the direction correction. +The corrected implementation matches the independently derived direction ratio for both selected components in those cases. +The old behavior, corrected values and reference values are retained in `logs/uncertainty_guidance_checks_v2_20260913/cancellation-reproduction.json`. +Component tests also compare mixture corrections with independent Gaussian and noncentral-chi densities, and compare guided downstream observation integrals with one-dimensional integration under the original sphere law. + +Compute job `22671599` passed 51 functional tests, focused type and lint checks, and pinned formatting checks. +Source hashes and verification artifacts are retained in `logs/uncertainty_guidance_checks_v2_20260913`. +Native array `22671657` uses the same two evaluated futures, independent seeds 911 and 912, 32 histories and eight four-action blocks as the preceding sequential pilot. +Its frozen plan and worker are in `logs/uncertainty_balloons_guided_future_20260913`. +Each moving extension retains its component selector, direction coordinates, proposal mean, correction and separately recomputed original speed factor. +The worker verifies their density decomposition and preserves the full-reference, prefix and terminal replay checks. +These checks establish accounting and replay consistency; numerical stability of the complete density estimate remains to be evaluated from the pilot results. diff --git a/predicators/code_sim_learning/inference_guidance.py b/predicators/code_sim_learning/inference_guidance.py new file mode 100644 index 000000000..2e759fbd7 --- /dev/null +++ b/predicators/code_sim_learning/inference_guidance.py @@ -0,0 +1,111 @@ +"""Defensive direction proposals for exact-speed conditional integration. + +Guidance changes the sampling law only. The returned factor includes the +original radial density and the original-direction/mixture-proposal +ratio. It is not a replacement velocity law or a forecast generator. +""" +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Literal, Tuple + +import numpy as np + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_discrepancy import \ + VelocityDiscrepancy + + +@dataclass(frozen=True) +class GuidedVelocity: + """An importance draw, including the complete proposal correction.""" + velocity: Tuple[float, float, float] + log_factor: float + log_proposal_correction: float + component: Literal["original", "guided", "rest"] + speed_residual: float + + +def condition_guided_velocity(law: VelocityDiscrepancy, + predicted: Tuple[float, float, float], + speed: float, proposal_mean: Tuple[float, float, + float], + guided_probability: float, + unit: Tuple[float, ...]) -> GuidedVelocity: + """Draw from a mixture of original and guided conditional directions. + + At positive speed, three independent unit uniforms select a mixture + component and its two direction coordinates. Both components are + normalized conditional Gaussian direction laws at the same speed. + The original component has strictly positive mixture probability, + bounding the direction importance ratio by 1/(1-guided_probability). + The mixture density, not the selected component density, determines + the correction. A zero speed uses the original rest mass and no + proposal coordinates. + + proposal_mean may depend on the scored observations and parent + history. The caller must keep it fixed while drawing this extension, + preserve its reconstruction, and retain this full factor exactly + once. It must never use these observation-guided draws as generated + forecasts or reinterpret the factors as posterior probabilities. + """ + if any( + len(mean) != 3 or any(not math.isfinite(v) for v in mean) + for mean in (predicted, proposal_mean)): + raise ValueError( + "Original and proposal means need three finite values") + if not math.isfinite(guided_probability) or \ + not 0 <= guided_probability < 1: + raise ValueError("Guided probability must lie in [0, 1)") + if speed == 0: + if unit: + raise ValueError("Rest has no proposal coordinates") + rest = law.condition_on_speed(predicted, speed) + return GuidedVelocity(rest.velocity, rest.log_observation_factor, 0., + "rest", rest.speed_residual) + if len(unit) != 3 or any(not math.isfinite(v) or not 0 <= v <= 1 + for v in unit): + raise ValueError("Positive speed requires three unit uniforms") + if guided_probability == 0 or predicted == proposal_mean: + original = law.condition_on_speed(predicted, speed, unit[1:]) + return GuidedVelocity(original.velocity, + original.log_observation_factor, 0., "original", + original.speed_residual) + # Moving-only radial laws avoid subtracting two -inf masses when the + # physical model assigns probability one to rest. + moving = VelocityDiscrepancy(0., law.sigma) + original = moving.condition_on_speed(predicted, speed, unit[1:]) + guided = moving.condition_on_speed(proposal_mean, speed, unit[1:]) + selected = guided if unit[0] < guided_probability else original + scaled_shift = tuple( + (b - a) / law.sigma for a, b in zip(predicted, proposal_mean)) + scaled_offset = tuple( + (v - (.5 * a + .5 * b)) / law.sigma + for v, a, b in zip(selected.velocity, predicted, proposal_mean)) + terms = tuple(d * offset for d, offset in zip(scaled_shift, scaled_offset)) + if any(not math.isfinite(v) for v in terms): + raise ConditioningNumericalError("Guidance density ratio overflow") + try: + # q(direction)/p(direction) = N_q(v)/N_p(v) * radial_p/radial_q. + relative = math.fsum(terms + (original.log_observation_factor, + -guided.log_observation_factor)) + except OverflowError as exc: + raise ConditioningNumericalError("Guidance density ratio overflow") \ + from exc + if not math.isfinite(relative): + raise ConditioningNumericalError("Guidance density ratio overflow") + correction = -float( + np.logaddexp(math.log1p(-guided_probability), + math.log(guided_probability) + relative)) + radial = (-math.inf if law.rest_probability == 1 else + math.log1p(-law.rest_probability) + + original.log_observation_factor) + factor = radial + correction + if math.isnan(factor) or factor == math.inf or \ + (math.isfinite(radial) and not math.isfinite(factor)): + raise ConditioningNumericalError("Guided factor overflow") + return GuidedVelocity(selected.velocity, factor, correction, + "guided" if selected is guided else "original", + selected.speed_residual) diff --git a/tests/code_sim_learning/test_inference_guidance.py b/tests/code_sim_learning/test_inference_guidance.py new file mode 100644 index 000000000..30cdfd250 --- /dev/null +++ b/tests/code_sim_learning/test_inference_guidance.py @@ -0,0 +1,118 @@ +"""Guided conditional integration retains the original probability model.""" +import math + +import numpy as np +import pytest +from scipy.integrate import quad +from scipy.stats import multivariate_normal, ncx2 + +from predicators.code_sim_learning.inference_discrepancy import \ + VelocityDiscrepancy +from predicators.code_sim_learning.inference_guidance import \ + condition_guided_velocity +from predicators.code_sim_learning.inference_path_integral import \ + integrate_conditional_paths + + +@pytest.mark.parametrize("selector", [.1, .95]) +def test_full_mixture_density_matches_independent_reference(selector): + """Both selected components must use the same full mixture density.""" + law = VelocityDiscrepancy(.2, .3) + original, proposal, speed = (0., 0., .3), (.5, .2, -.4), .4 + probability = .8 + result = condition_guided_velocity(law, original, speed, proposal, + probability, (selector, .4, .6)) + + def radial(mean): + return ncx2.pdf((speed / law.sigma)**2, 3, + np.dot(mean, mean) / law.sigma**2) * \ + 2 * speed / law.sigma**2 + + original_radial, guided_radial = radial(original), radial(proposal) + original_pdf = multivariate_normal.pdf(result.velocity, original, + np.eye(3) * law.sigma**2) + guided_pdf = multivariate_normal.pdf(result.velocity, proposal, + np.eye(3) * law.sigma**2) + relative = guided_pdf / original_pdf * original_radial / guided_radial + correction = -math.log(1 - probability + probability * relative) + assert result.log_proposal_correction == pytest.approx(correction, + abs=1e-12) + assert result.log_factor == pytest.approx(math.log(.8 * original_radial) + + correction, + abs=1e-12) + assert result.log_proposal_correction <= -math.log1p(-probability) + assert result.component == ("guided" + if selector < probability else "original") + assert math.hypot(*result.velocity) == pytest.approx(speed, abs=1e-15) + + +@pytest.mark.parametrize("probability", [.2, .8]) +def test_guided_future_integral_matches_original_sphere_quadrature( + probability): + """Off-axis guidance leaves the original downstream integral unchanged.""" + law = VelocityDiscrepancy(.2, .3) + original, proposal, speed = (0., 0., .3), (.25, .1, .1), .4 + reading, noise = .05, .2 + + def integrand(rng): + result = condition_guided_velocity(law, original, speed, proposal, + probability, tuple(rng.random(3))) + output = -.5 * ((reading - result.velocity[2]) / noise)**2 - \ + math.log(noise * math.sqrt(2 * math.pi)) + return result.log_factor + output + + result = integrate_conditional_paths(integrand, 12000, 97) + concentration = speed * original[2] / law.sigma**2 + + def reference(cosine): + direction = concentration * math.exp(concentration * cosine) / \ + (2 * math.sinh(concentration)) + output = math.exp(-.5 * ((reading - speed * cosine) / noise)**2) / \ + (noise * math.sqrt(2 * math.pi)) + return direction * output + + radial = math.exp( + law.condition_on_speed(original, speed, + (.5, .5)).log_observation_factor) + expected = radial * quad(reference, -1, 1, epsabs=1e-12)[0] + assert math.exp(result.log_density) == pytest.approx(expected, rel=.03) + assert result.relative_standard_error is not None + assert result.relative_standard_error < .02 + + +def test_no_guidance_rest_and_zero_physical_support(): + """Disabled guidance is exact, and rest retains its original atom.""" + law = VelocityDiscrepancy(.2, .3) + mean, other, units = (0., 0., .3), (.2, .1, .1), (.4, .3, .7) + original = law.condition_on_speed(mean, .4, units[1:]) + for proposal, probability in ((other, 0.), (mean, .8)): + result = condition_guided_velocity(law, mean, .4, proposal, + probability, units) + assert result.velocity == original.velocity + assert result.log_factor == original.log_observation_factor + assert result.log_proposal_correction == 0. + rest = condition_guided_velocity(law, mean, 0., other, .8, ()) + assert rest.log_factor == math.log(.2) + assert rest.velocity == (0., 0., 0.) and rest.component == "rest" + zero = condition_guided_velocity(VelocityDiscrepancy(1., .3), mean, .4, + other, .8, units) + assert zero.log_factor == -math.inf + for probability in (-.1, 1., math.nan): + with pytest.raises(ValueError): + condition_guided_velocity(law, mean, .4, other, probability, units) + for invalid_units in ((), (.1, .2), (.1, .2, math.nan)): + with pytest.raises(ValueError): + condition_guided_velocity(law, mean, .4, other, .8, invalid_units) + + +@pytest.mark.parametrize("selector", [.1, .95]) +def test_opposite_concentrated_means_preserve_direction_ratio(selector): + """Cancel shared radial constants without losing directional evidence.""" + law = VelocityDiscrepancy(.2, 1.) + result = condition_guided_velocity(law, (0., 0., 1e150), 1., + (0., 0., -1e150), .8, + (selector, .5, .5)) + # Equal concentration and opposite axes cancel vMF normalizers exactly. + relative = -2e150 * result.velocity[2] + expected = -float(np.logaddexp(math.log(.2), math.log(.8) + relative)) + assert result.log_proposal_correction == pytest.approx(expected, rel=1e-15) From 79e6f68c49703f6a71e5f3e0e6128a20b219975f Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 04:52:44 -0400 Subject: [PATCH 04/94] Record guided integration and Domino prediction stability results --- docs/uncertainty/domino-joint-inference.md | 35 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 ++- .../stochastic-future-integration.md | 15 ++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/domino-joint-inference.md b/docs/uncertainty/domino-joint-inference.md index 426bc6030..619f3d6e2 100644 --- a/docs/uncertainty/domino-joint-inference.md +++ b/docs/uncertainty/domino-joint-inference.md @@ -152,3 +152,38 @@ The runtime-source difference and its earlier exact likelihood parity checks rem Completion does not resolve the disagreement or establish trustworthy posterior coverage. Predictive stability and a defensible exploration of the joint parameter/initial-state distribution remain required before any posterior publication or legacy comparison claim. The source hashes and completion checks are retained in `logs/uncertainty_domino_conditioned_checkpoint_20260912/completion-comparison-20260913.json`. + +### Future predictions from the completed populations + +Array `22671823` replays every complete weighted row of each unassessed population through the 97-action suffix following the fitted 64-action prefix. +It uses the retained physical parameter values directly, avoiding a lossy inverse transform back into prior coordinates, while preserving each row's state coordinates and correlations. +Before evaluating the population, each worker reconstructs the archived best candidate exactly, repeats its full 161-action history, and reproduces its original fitting log likelihood exactly. +The first retained row is also repeated exactly, every fitting prefix has finite likelihood, and no row is dropped or reweighted using future observations. + +For Cartesian readings, conditional output means and variances follow the declared scalar error process filtered on the fitting prefix, with the original future sensor variance retained. +Native toppling indicators use the frozen domain threshold; they describe model predictions, not achieved task outcomes. +These diagnostics consume unassessed populations to investigate their disagreement and do not bypass the production posterior-assessment boundary. + +Both jobs completed and all 128 compressed history artifacts passed their hash checks. +Each job performed 10,787 native actions, including its reconstruction checks, and took approximately 3.5 minutes. + +| Quantity | Numerical seed 100 | Numerical seed 101 | +| --- | ---: | ---: | +| Future Cartesian conditional-mean RMSE | 0.011585 m | 0.012130 m | +| Future native-mean RMSE | 0.011688 m | 0.012354 m | +| Whole-future mixture log density | 14105.63278 | 14434.93358 | +| Final predicted toppling probability, domino_1 | 0.96875 | 0.32628 | + +Across all future Cartesian readings, the two conditional means differ by 0.006934 m RMS, with a maximum difference of 0.052394 m. +Their predicted standard deviations differ by 0.001694 m RMS. +These apparently similar averaged errors do not imply equivalent decisions. +At primitive step 158, the two populations predict `domino_3` toppling with probabilities 0.90625 and 0, respectively. +Their final `domino_1` predictions also differ by more than 0.64. + +The full future log-density estimates differ by 329.30080, and each is dominated by roughly one weighted contribution among the 64 rows. +An informative future can concentrate those contributions even under a valid prefix posterior, so this alone is not proof that the probability model is wrong. +Together with the replica differences, it leaves the present finite-population predictive calculation unsuitable for numerical acceptance. +A low average position error on one suffix cannot establish posterior coverage, toppling/timing stability, or unchanged agent performance. + +The source populations, reconstruction code, full histories, per-row scores, complete forecast curves and comparison checks are retained in `logs/uncertainty_domino_population_forecast_20260913`. +This is a population-stability diagnostic; a completed matched comparison against the incumbent predictions and the broader validation gates remain required. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 313465d84..0bba2ef11 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -13,8 +13,10 @@ The subsequent offline sequential integrator retains complete histories and bloc Forty-four functional tests and focused static checks pass. All four native sequential integrations completed with exact replay and factor accounting, but both independent-run comparisons fail the declared density-stability diagnostic and retain only one or two original ancestors. The next offline proposal mixes original and position-guided velocity directions while retaining the full mixture correction, so it targets the same probability model. -Fifty-one functional tests and focused static checks pass; its matched native Balloons pilot is submitted for numerical evaluation. +Fifty-one functional tests and focused static checks pass; all four guided native pilots completed, but both whole-history density comparisons still fail the declared consistency diagnostic. The two [Domino conditioned-base pilots](domino-joint-inference.md#completed-numerical-pilots-september-13) also completed, with strongly different parameter summaries and one original ancestor each; numerical adequacy remains unestablished. +The completed Domino population replay gives similar mean Cartesian errors (1.16 and 1.21 cm), but predicted toppling differs by as much as 0.90625 at the same frame. +The disagreement therefore affects goal-relevant predictions despite similar averaged feature errors. Fan recovery jobs are active after confirmed allocation timeouts, and the two Balloons fits have been submitted for continuation from their saved complete-stage checkpoints under the same numerical budgets. The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. diff --git a/docs/uncertainty/stochastic-future-integration.md b/docs/uncertainty/stochastic-future-integration.md index 8d84aaf79..0a689c5dc 100644 --- a/docs/uncertainty/stochastic-future-integration.md +++ b/docs/uncertainty/stochastic-future-integration.md @@ -186,3 +186,18 @@ Its frozen plan and worker are in `logs/uncertainty_balloons_guided_future_20260 Each moving extension retains its component selector, direction coordinates, proposal mean, correction and separately recomputed original speed factor. The worker verifies their density decomposition and preserves the full-reference, prefix and terminal replay checks. These checks establish accounting and replay consistency; numerical stability of the complete density estimate remains to be evaluated from the pilot results. + +All four guided tasks completed with verified density decomposition, exact prefix/terminal replay and valid artifact hashes. +The numerical consistency result remains negative: + +| Evaluated future | Seed 911 log density | Seed 912 log density | Between-seed difference | Final original ancestors, seeds 911 / 912 | +| --- | ---: | ---: | ---: | --- | +| Generated701 | 4963.60727 | 4959.95918 | 3.64809 | 1 / 1 | +| Recorded | -2548.06723 | -2527.55554 | 20.51169 | 2 / 1 | + +Both differences exceed the predeclared 0.2 diagnostic limit. +Some individual block effective counts increased, but that does not establish accurate complete-history integration. +With only two replicas, smaller gaps than the preceding pilot are not evidence of a reliable variance reduction or calibrated prediction. +The jobs took approximately 10 minutes each and again performed 85,676 native actions in total. +The full results and artifact checker are retained in the guided bundle's `verification.json` and `verify_report.py`. +Guidance preserves the intended probability model, but this budget and proposal do not pass the numerical gate. From 6981d8c7d8b993d2b48e01e5d91049566d18546d Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 05:19:51 -0400 Subject: [PATCH 05/94] Add ordered process-safe target batches for offline inference --- docs/uncertainty/batch-evaluation.md | 78 ++++++ docs/uncertainty/domino-joint-inference.md | 30 +- docs/uncertainty/implementation-progress.md | 8 + .../code_sim_learning/inference_evaluation.py | 67 +++++ .../code_sim_learning/inference_sampling.py | 143 +++++++--- .../test_inference_evaluation.py | 260 ++++++++++++++++++ 6 files changed, 543 insertions(+), 43 deletions(-) create mode 100644 docs/uncertainty/batch-evaluation.md create mode 100644 predicators/code_sim_learning/inference_evaluation.py create mode 100644 tests/code_sim_learning/test_inference_evaluation.py diff --git a/docs/uncertainty/batch-evaluation.md b/docs/uncertainty/batch-evaluation.md new file mode 100644 index 000000000..9279ce43f --- /dev/null +++ b/docs/uncertainty/batch-evaluation.md @@ -0,0 +1,78 @@ +# Ordered batch evaluation for offline inference + +September 13, 2026. +This implements an evaluation-throughput component of the [simplification proposal](simplification-proposal.md), without changing the production estimator. +The motivation is the multi-hour physical posterior pilots and their unresolved prediction disagreement, documented in [Domino joint inference](domino-joint-inference.md). +More cores can reduce evaluation latency; they do not resolve missing posterior modes or certify numerical adequacy. + +## Contract + +`BatchedTarget` accepts an immutable ordered tuple of proposal coordinates and returns one `TargetEvaluation` per proposal. +Each result includes the original proposal identity, the complete joint coordinates, the conditional-base/proposal log factor and the remaining log likelihood. +The sampler verifies cardinality, proposal order, output dimensions and box-prior invariants before consuming any batch. +Coordinates must be finite, and log factors may be finite or negative infinity; NaN and positive infinity are errors. +A zero conditional-base factor requires zero target support. + +Each worker must own the entire conditional map and likelihood evaluation. +The physical conditional map creates a candidate-specific native world that the likelihood consumes, so concurrently calling the existing two callbacks against shared state is unsafe. +The caller supplies a synchronous ordered map or an ordered process-executor map, owns the executor lifetime and identifies all dependencies in the immutable runtime identity. +Setup and worker failures propagate rather than becoming zero-likelihood candidates. +No worker may drop, replace or silently retry an evaluation. + +## Sampler integration + +`sample_batch` uses the same tempering, conditional density corrections, weights, resampling and symmetric Metropolis acceptance calculation in both modes. +The new mode evaluates all valid proposals in a mutation sweep together, then applies accepted updates in particle order. +Each particle's proposal is independent of the other particles' acceptance decisions within that sweep. +A proposal outside the declared support is rejected without clipping or target evaluation. + +The new mode draws an acceptance uniform for every proposal, including proposals outside support and proposals whose likelihood is zero. +This makes its random stream independent of worker completion order and evaluation outcomes. +It is a separately identified random schedule, `tempered_smc_ordered_batch_v1`, and is not bitwise equivalent to the original scalar schedule. +The existing scalar path retains its original schedule and checkpoint signature. +Checkpoints reject cross-mode continuation, while synchronous and parallel maps within batch mode must reproduce the same population, weights and diagnostics exactly. +Callers record the selected schedule with their frozen runtime and experiment provenance. + +The sampler reserves the remaining evaluation allowance before dispatching workers. +If a full batch would exceed it, only the permitted prefix is evaluated, and the run returns `budget_exhausted` with no posterior samples. +Only complete initialization and temperature boundaries produce checkpoints. +A failed or interrupted batch therefore cannot publish a partial stage as a completed posterior. +Resuming preserves the cumulative numerical budget; repeated work after interruption remains a separately reported compute cost. + +## Validation + +The numerical references include an analytically solvable truncated Beta distribution after a nonlinear coordinate transformation, an uninformed uniform parameter, exact process-map agreement, zero-support points, budget exhaustion and interrupted-stage recovery. +A separate comparison uses the pre-change scalar implementation to check complete result and checkpoint equality across conditional/unconditional targets, blocked/full-vector proposals and multiple seeds. +The checks run on compute nodes, with frozen sources retained in `logs/uncertainty_batch_evaluation_v2_20260913`. + +A native Domino validation uses the existing conditional initialization and full 64-action likelihood as one indivisible worker operation. +It compares fixed candidate evaluations and a short complete sampler run between synchronous evaluation and four isolated processes. +It measures initialization and worker startup overhead separately from the steady-state sampler time. +Its small population and short schedule test execution equivalence and cost only; they do not provide a usable posterior or agent performance result. +Frozen inputs and worker source are retained in `logs/uncertainty_domino_batch_evaluation_20260913`. + +## Completed native and reference checks + +Compute job `22672354` completed on node1412 with four allocated CPUs. +All 32 fixed native target records match exactly across execution modes, including proposal/joint coordinates, conditional-base factors and remaining likelihoods. +Twenty-two of those records have finite remaining likelihood. +The subsequent 32-particle, two-temperature, one-move sampler completes all 96 evaluations in both modes with exactly identical samples, weights and diagnostics. +This deliberately short run retains one original ancestor and does not establish numerical adequacy. + +| Measurement | Synchronous evaluation | Four isolated processes | +| --- | ---: | ---: | +| Initialization plus 32 fixed candidates | 52.08 s | 23.38 s | +| Subsequent 96-evaluation sampler | 98.50 s | 26.19 s | +| Combined measured work | 150.58 s | 49.57 s | + +The measured sampler portion is 3.76 times faster; the combined work including initialization and fixed-candidate validation is 3.04 times faster. +These are single matched latency measurements on the declared hardware, not a reduction in logical evaluations or a guarantee for larger fits and other domains. + +The corrected reference run `22672392` passes 32 functional tests and 16 exact comparisons against the original scalar implementation, including every emitted checkpoint. +The first run, `22672316`, exposed a test error: a support assertion inspected zero-weight rows retained when resampling is skipped. +The corrected assertion checks positive posterior mass, and the original failed report remains archived. + +Final check job `22672431` completed successfully after replacing a dynamically constructed invalid-field test with explicit typed calls. +It reran all nine batch-target tests and the sixteen scalar parity comparisons, and passed three-file mypy, pylint and pinned formatting checks. +The preceding 32-test suite remains valid for the unchanged implementation; the final fixture and static-check artifacts are in `logs/uncertainty_batch_evaluation_v3_20260913`. +The implementation files match the frozen native-validation sources byte for byte. diff --git a/docs/uncertainty/domino-joint-inference.md b/docs/uncertainty/domino-joint-inference.md index 619f3d6e2..4fcb957cd 100644 --- a/docs/uncertainty/domino-joint-inference.md +++ b/docs/uncertainty/domino-joint-inference.md @@ -186,4 +186,32 @@ Together with the replica differences, it leaves the present finite-population p A low average position error on one suffix cannot establish posterior coverage, toppling/timing stability, or unchanged agent performance. The source populations, reconstruction code, full histories, per-row scores, complete forecast curves and comparison checks are retained in `logs/uncertainty_domino_population_forecast_20260913`. -This is a population-stability diagnostic; a completed matched comparison against the incumbent predictions and the broader validation gates remain required. +This population-stability diagnostic is followed by the matched incumbent comparison below; broader validation gates remain required. + +## Matched legacy prediction diagnostic, September 13 + +Compute job `22672106` completed evaluation of the already-frozen predictions from the cold legacy fit and both new populations. +All three use the same fixed program and 64-action fitting prefix, followed by the same 97-action future suffix. +The worker verifies input identities and all 128 compressed population histories, and independently reconstructs their previously reported means and toppling curves. +Stored noiseless states are evaluator-only labels, read after the forecasts were fixed; they enter neither inference nor prediction. +This evaluation takes no native simulator steps. + +| Forecast | Cartesian RMSE to noisy readings | Cartesian RMSE to stored truth | Mean toppling Brier error | Final domino_1 toppling probability | +| --- | ---: | ---: | ---: | ---: | +| Cold legacy fit | 0.011062 m | 0.004933 m | 0.003436 | 0 | +| Population 100, unassessed | 0.011585 m | 0.006138 m | 0.002801 | 0.96875 | +| Population 101, unassessed | 0.012130 m | 0.006849 m | 0.002824 | 0.32628 | + +The legacy fit has smaller position error on this suffix. +Both new populations have slightly smaller Brier error averaged over all 97 frames and six objects, but their final predictions disagree substantially. +The recorded first toppling frames for objects 0 through 5 are `[157, 161, absent, 159, 160, absent]`. +Legacy predicts `[157, absent, absent, 159, 161, absent]`. +Population 100 mostly predicts object 3 toppling at frame 158, one frame early, while population 101 predicts frame 159. +A long interval before any toppling makes the frame-averaged metric particularly insufficient as a goal or event-timing assessment. +These are descriptive errors on one training suffix, not independent-trial uncertainty estimates or evidence of an agent advantage. + +The cold legacy fit took 214.39 seconds using its six-CPU allocation. +The new serial population fits took over five hours each, with an additional timed-out attempt for seed 100. +The methods change both parameter inference and initial-state handling, so the planned matched ablations are still required. +Numerical stability, predictive adequacy and inference cost have not passed the replacement gate. +The report, frozen inputs and evaluator are in `logs/uncertainty_domino_legacy_prediction_comparison_20260913`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 0bba2ef11..221b7f27c 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -19,6 +19,14 @@ The completed Domino population replay gives similar mean Cartesian errors (1.16 The disagreement therefore affects goal-relevant predictions despite similar averaged feature errors. Fan recovery jobs are active after confirmed allocation timeouts, and the two Balloons fits have been submitted for continuation from their saved complete-stage checkpoints under the same numerical budgets. +The completed matched Domino diagnostic now compares all three forecasts on the same 97-action suffix, with stored truth used only for evaluation after predictions were frozen. +Legacy has lower Cartesian error, while the new populations have slightly lower frame-averaged toppling Brier error and substantially different final toppling probabilities. +This does not establish a replacement advantage, and the new multi-hour fits retain an unresolved cost problem. +The next [ordered batch evaluation component](batch-evaluation.md) combines each conditional map and likelihood into an indivisible worker operation, allowing isolated processes to evaluate a mutation sweep concurrently. +Its scalar path retains the original random schedule, while batch mode uses a separate checkpoint identity and reserves its numerical budget before dispatch. +Thirty-two functional tests, sixteen exact comparisons against the original scalar implementation and final focused type/lint/format checks pass. +The native Domino check reproduces all target values and complete sampler output exactly across one and four processes, with a measured 3.76-fold sampler speedup; numerical adequacy remains unestablished. + The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. Twenty-three functional tests and focused type/lint/format checks pass; running fits retain their existing frozen source. diff --git a/predicators/code_sim_learning/inference_evaluation.py b/predicators/code_sim_learning/inference_evaluation.py new file mode 100644 index 000000000..80affcf56 --- /dev/null +++ b/predicators/code_sim_learning/inference_evaluation.py @@ -0,0 +1,67 @@ +"""Ordered, indivisible target evaluations for offline inference workers. + +Each worker must own the entire conditional map and likelihood +calculation. In particular, native worlds cannot be shared between +concurrent evaluations. The caller owns process isolation, executor +lifetime and runtime provenance. +""" +import math +from dataclasses import dataclass +from typing import Callable, Tuple + + +@dataclass(frozen=True) +class TargetEvaluation: + """Proposal identity, full joint point and its two distinct log factors.""" + proposal: Tuple[float, ...] + joint: Tuple[float, ...] + log_base: float + log_likelihood: float + + def __post_init__(self) -> None: + for name in ("proposal", "joint"): + values = tuple(float(v) for v in getattr(self, name)) + if not values or not all(math.isfinite(v) for v in values): + raise ValueError("Target coordinates must be finite") + object.__setattr__(self, name, values) + for name in ("log_base", "log_likelihood"): + value = float(getattr(self, name)) + if math.isnan(value) or value == math.inf: + raise ValueError("Target factor is NaN or positive infinity") + object.__setattr__(self, name, value) + if self.log_base == -math.inf and self.log_likelihood != -math.inf: + raise ValueError("Zero base support requires zero target support") + + +@dataclass(frozen=True) +class BatchedTarget: + """Evaluate immutable proposals in order, with no dropped or retried rows. + + A synchronous map and a process executor map use the same contract. + Completion order must not change returned order. Setup and worker + errors propagate; zero support is a returned density, not a caught + exception. All dependencies must be identified by the sampler's + runtime identity. + """ + evaluate_many: Callable[[Tuple[Tuple[float, ...], ...]], + Tuple[TargetEvaluation, ...]] + + def evaluate(self, proposals: Tuple[Tuple[float, ...], + ...], joint_dimension: int, + conditional: bool) -> Tuple[TargetEvaluation, ...]: + """Validate identities before the sampler can consume any result.""" + results = tuple(self.evaluate_many(proposals)) + if len(results) != len(proposals): + raise ValueError("Target batch returned the wrong number of rows") + for proposal, result in zip(proposals, results): + if result.proposal != proposal: + raise ValueError( + "Target batch changed proposal identity/order") + if len(result.joint) != joint_dimension: + raise ValueError( + "Target batch returned invalid joint dimension") + if not conditional and (result.joint != proposal + or result.log_base != 0.): + raise ValueError( + "Box target must preserve coordinates and base") + return results diff --git a/predicators/code_sim_learning/inference_sampling.py b/predicators/code_sim_learning/inference_sampling.py index e118888e0..4cdf35b5f 100644 --- a/predicators/code_sim_learning/inference_sampling.py +++ b/predicators/code_sim_learning/inference_sampling.py @@ -10,7 +10,7 @@ import json import math from dataclasses import asdict, dataclass -from typing import Callable, List, Literal, Optional, Tuple, Union +from typing import Callable, Iterator, List, Literal, Optional, Tuple, Union import numpy as np @@ -18,6 +18,7 @@ SamplerCheckpoint from predicators.code_sim_learning.inference_data import InferenceIdentity, \ content_digest +from predicators.code_sim_learning.inference_evaluation import BatchedTarget @dataclass(frozen=True) @@ -231,7 +232,8 @@ class _BudgetExceeded(Exception): def sample_batch(prior: Union[BoxPrior, ConditionedPrior], identity: InferenceIdentity, - log_likelihood: Callable[[np.ndarray], float], + log_likelihood: Union[Callable[[np.ndarray], float], + BatchedTarget], config: SamplerConfig, seed: int, *, @@ -273,11 +275,22 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], Interrupted stages are repeated from their last saved boundary; the checkpoint is not a posterior or a numerical adequacy certificate. Callers must identify all callback dependencies in identity.runtime. + + Alternatively, a BatchedTarget owns both conditioning and likelihood and + receives one ordered population per mutation sweep. Workers must evaluate + whole candidates in isolated processes. This mode pre-draws an acceptance + uniform for every proposal, including rejected ones, so worker scheduling + cannot change the random stream. Its separately identified checkpoint + kernel cannot resume scalar-mode checkpoints. Scalar calls preserve the + original draw schedule. Both modes target the same declared distribution; + their random streams and resulting finite populations can differ. """ if identity.prior != prior.digest: raise ValueError("Prior differs from immutable inference identity") conditional = isinstance(prior, ConditionedPrior) - if conditional != (condition is not None): + batched = isinstance(log_likelihood, BatchedTarget) + if (batched and condition is not None) or (not batched and conditional != + (condition is not None)): raise ValueError( "A conditional prior requires exactly one coordinate map") proposal_prior = prior.proposal if isinstance(prior, @@ -309,7 +322,8 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], signature = content_digest( json.dumps( { - "kernel": "tempered_smc_stage_checkpoint_v1", + "kernel": "tempered_smc_ordered_batch_v1" + if batched else "tempered_smc_stage_checkpoint_v1", "identity": identity.digest, "prior": prior.digest, "config": asdict(config), @@ -412,11 +426,75 @@ def evaluate(candidate: np.ndarray) -> Tuple[float, float, np.ndarray]: raise ValueError("Base weight returned NaN or positive infinity") if base == -math.inf: return -math.inf, base, joint.copy() + assert not isinstance(log_likelihood, BatchedTarget) value = float(log_likelihood(joint.copy())) if math.isnan(value) or value == math.inf: raise ValueError("Likelihood returned NaN or positive infinity") return value, base, joint.copy() + def evaluate_many( + candidates: List[np.ndarray] + ) -> Iterator[Tuple[float, float, np.ndarray]]: + nonlocal evaluations + if not isinstance(log_likelihood, BatchedTarget): + for candidate in candidates: + yield evaluate(candidate) + return + allowed = min(len(candidates), config.max_evaluations - evaluations) + proposals = tuple( + tuple(float(v) for v in candidate) + for candidate in candidates[:allowed]) + # Reserve the logical budget before any workers can start. A partial + # stage never emits a checkpoint or usable posterior population. + evaluations += allowed + if proposals: + rows = log_likelihood.evaluate(proposals, len(prior.names), + conditional) + for row in rows: + yield row.log_likelihood, row.log_base, np.asarray(row.joint) + if allowed < len(candidates): + raise _BudgetExceeded + + def propose(index: int) -> np.ndarray: + if config.proposal_blocks: + # A state-independent mixture of symmetric block kernels. + block = list(config.proposal_blocks[int( + rng.integers(len(config.proposal_blocks)))]) + proposal = particles[index].copy() + proposal[block] += rng.normal(size=len(block)) * \ + (upper[block] - lower[block]) * config.proposal_scale + return proposal + return particles[index] + rng.normal( + size=len(proposal_prior.names)) * \ + (upper - lower) * config.proposal_scale + + def supported(proposal: np.ndarray) -> bool: + return bool( + np.all(np.isfinite(proposal)) and np.all(proposal >= lower) + and np.all(proposal <= upper)) + + def accept(index: int, + proposal: np.ndarray, + trial: Tuple[float, float, np.ndarray], + beta: float, + uniform: Optional[float] = None) -> None: + nonlocal accepted + proposed_likelihood, proposed_base, proposed_joint = trial + if proposed_likelihood == -math.inf: + return + log_ratio = beta * (proposed_likelihood - likelihoods[index]) + if conditional: + log_ratio += proposed_base - base_weights[index] + # The original scalar path draws only after a finite evaluation. + # Batch mode supplies its independently pre-drawn acceptance value. + draw = rng.random() if uniform is None else uniform + if math.log(1.0 - draw) < log_ratio: + particles[index] = proposal + likelihoods[index] = proposed_likelihood + base_weights[index] = proposed_base + joints[index] = proposed_joint + accepted += 1 + def result( status: Literal["complete", "budget_exhausted", "no_particle_support"] ) -> BatchPosterior: @@ -443,9 +521,8 @@ def result( try: if resume is None: - for i in range(count): - likelihoods[i], base_weights[i], joints[i] = evaluate( - particles[i]) + for i, trial in enumerate(evaluate_many(list(particles))): + likelihoods[i], base_weights[i], joints[i] = trial initial_finite += int(math.isfinite(likelihoods[i])) if not initial_finite: # Finite initialization may simply have missed valid support. @@ -485,41 +562,23 @@ def result( weights.fill(1.0 / count) resampling_count += 1 for _ in range(config.moves): - for i in range(count): - attempted += 1 - if config.proposal_blocks: - # A state-independent uniform mixture of symmetric - # block kernels preserves the same tempered target. - block = config.proposal_blocks[int( - rng.integers(len(config.proposal_blocks)))] - block_indices = list(block) - proposal = particles[i].copy() - proposal[block_indices] += rng.normal( - size=len(block_indices)) * \ - (upper[block_indices] - lower[block_indices]) * \ - config.proposal_scale - else: - # Preserve the original full-vector random stream. - proposal = particles[i] + rng.normal( - size=len(proposal_prior.names)) * \ - (upper - lower) * config.proposal_scale - if not np.all(np.isfinite(proposal)) or np.any( - proposal < lower) or np.any(proposal > upper): - continue - trial = evaluate(proposal) - proposed_likelihood, proposed_base, proposed_joint = trial - if proposed_likelihood == -math.inf: - continue - log_ratio = beta * (proposed_likelihood - likelihoods[i]) - if conditional: - log_ratio += proposed_base - base_weights[i] - # 1-random lies in (0, 1], so log never sees zero. - if math.log(1.0 - rng.random()) < log_ratio: - particles[i] = proposal - likelihoods[i] = proposed_likelihood - base_weights[i] = proposed_base - joints[i] = proposed_joint - accepted += 1 + if batched: + pending = [] + for i in range(count): + attempted += 1 + proposal = propose(i) + uniform = float(rng.random()) + if supported(proposal): + pending.append((i, proposal, uniform)) + trials = evaluate_many([row[1] for row in pending]) + for row, trial in zip(pending, trials): + accept(row[0], row[1], trial, beta, row[2]) + else: + for i in range(count): + attempted += 1 + proposal = propose(i) + if supported(proposal): + accept(i, proposal, evaluate(proposal), beta) completed = beta completed_stage = stage emit_checkpoint() diff --git a/tests/code_sim_learning/test_inference_evaluation.py b/tests/code_sim_learning/test_inference_evaluation.py new file mode 100644 index 000000000..9891256c2 --- /dev/null +++ b/tests/code_sim_learning/test_inference_evaluation.py @@ -0,0 +1,260 @@ +"""Batched targets preserve conditional densities, budgets and +replayability.""" +import math +import multiprocessing +from concurrent.futures import ProcessPoolExecutor +from dataclasses import replace +from typing import List, Tuple + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_checkpoint import \ + SamplerCheckpoint +from predicators.code_sim_learning.inference_data import InferenceIdentity, \ + content_digest +from predicators.code_sim_learning.inference_evaluation import BatchedTarget, \ + TargetEvaluation +from predicators.code_sim_learning.inference_sampling import BoxPrior, \ + ConditionedPrior, PriorPoint, SamplerConfig, sample_batch + + +def _target(proposal: Tuple[float, ...]) -> TargetEvaluation: + """Uniform x via x=u², then a truncated likelihood proportional to x.""" + u = proposal[0] + x = u**2 + return TargetEvaluation(proposal, (x, 1 - x), + math.log(2 * u) if u >= .2 else -math.inf, + math.log(x) if u >= .4 else -math.inf) + + +def _mapped( + proposals: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple(_target(proposal) for proposal in proposals) + + +def _setup() -> Tuple[ConditionedPrior, InferenceIdentity, SamplerConfig]: + digest = content_digest(b"batched conditional density reference") + prior = ConditionedPrior(("x", "complement"), digest, digest, + BoxPrior(("u", ), ((0., 1.), ))) + identity = InferenceIdentity(digest, digest, digest, prior.digest, digest) + return prior, identity, SamplerConfig(particles=48, + temperatures=5, + moves=3, + proposal_scale=.3, + proposal_blocks=((0, ), )) + + +def test_process_scheduling_and_resume() -> None: + """Process isolation and checkpoint recovery give exactly the same run.""" + prior, identity, config = _setup() + saved: List[SamplerCheckpoint] = [] + expected = sample_batch(prior, + identity, + BatchedTarget(_mapped), + config, + 41, + checkpoint=saved.append) + assert expected.status == "complete" + assert expected.accepted_moves > 0 + assert expected.evaluations < config.particles * ( + 1 + config.moves * config.temperatures) + with ProcessPoolExecutor( + max_workers=2, + mp_context=multiprocessing.get_context("spawn")) as executor: + + def parallel( + proposals: Tuple[Tuple[float, ...], ...] + ) -> Tuple[TargetEvaluation, ...]: + return tuple(executor.map(_target, proposals)) + + actual = sample_batch(prior, identity, BatchedTarget(parallel), config, + 41) + assert actual == expected + for checkpoint in (saved[0], saved[2], saved[-1]): + assert sample_batch(prior, + identity, + BatchedTarget(parallel), + config, + 41, + resume=checkpoint) == expected + + with pytest.raises(ValueError, match="Checkpoint differs"): + sample_batch(prior, + identity, + lambda _: 0., + config, + 41, + condition=lambda x: PriorPoint((x[0], 1 - x[0]), 0.), + resume=saved[0]) + with pytest.raises(ValueError, match="coordinate map"): + sample_batch(prior, + identity, + BatchedTarget(_mapped), + config, + 41, + condition=lambda x: PriorPoint(tuple(x), 0.)) + + +@pytest.mark.parametrize("seed", [7, 19]) +def test_conditional_reference_moments(seed: int) -> None: + """Against analytic truncated Beta(2,1), including the coordinate + factor.""" + prior, identity, config = _setup() + config = replace(config, + particles=1200, + temperatures=12, + moves=4, + proposal_scale=.15) + result = sample_batch(prior, identity, BatchedTarget(_mapped), config, + seed) + assert result.status == "complete" + x = np.asarray(result.samples)[:, 0] + a = .4**2 + expected = 2 / 3 * (1 - a**3) / (1 - a**2) + second = .5 * (1 - a**4) / (1 - a**2) + # Zero-weight rows remain in the population when resampling is skipped. + assert np.min(x[np.asarray(result.weights) > 0]) >= a + assert np.max(x) < 1 + assert np.average(x, weights=result.weights) == pytest.approx(expected, + abs=.018) + assert np.average(x**2, weights=result.weights) == pytest.approx(second, + abs=.018) + np.testing.assert_allclose(np.sum(result.samples, axis=1), 1.) + + +@pytest.mark.parametrize("budget", [3, 51, 150]) +def test_budget_reservation_and_interrupted_batches(budget: int) -> None: + """Never dispatch extra work or publish an unfinished temperature.""" + prior, identity, config = _setup() + config = replace(config, max_evaluations=budget) + calls = 0 + saved: List[SamplerCheckpoint] = [] + + def counted( + proposals: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + nonlocal calls + calls += len(proposals) + assert calls <= budget + return _mapped(proposals) + + result = sample_batch(prior, + identity, + BatchedTarget(counted), + config, + 41, + checkpoint=saved.append) + assert result.status == "budget_exhausted" + assert result.evaluations == calls == budget + assert not result.samples and not result.weights + if budget < config.particles: + assert not saved + else: + checkpoint = saved[-1] + calls = checkpoint.unpack()["evaluations"] + assert sample_batch(prior, + identity, + BatchedTarget(counted), + config, + 41, + resume=checkpoint) == result + + +def test_worker_failure_keeps_last_complete_stage() -> None: + """A batch exception is an infrastructure error, not zero likelihood.""" + prior, identity, config = _setup() + saved: List[SamplerCheckpoint] = [] + calls = 0 + + def interrupted( + proposals: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + nonlocal calls + calls += 1 + if calls == 3: + raise RuntimeError("worker interrupted") + return _mapped(proposals) + + with pytest.raises(RuntimeError, match="worker interrupted"): + sample_batch(prior, + identity, + BatchedTarget(interrupted), + config, + 41, + checkpoint=saved.append) + assert len(saved) == 1 + resumed = sample_batch(prior, + identity, + BatchedTarget(_mapped), + config, + 41, + resume=saved[-1]) + assert resumed == sample_batch(prior, identity, BatchedTarget(_mapped), + config, 41) + + +def test_bad_batch_results_are_rejected() -> None: + """A reordered or incomplete map cannot silently corrupt particle + weights.""" + proposals = ((.3, ), (.7, )) + cases = ( + (lambda p: _mapped(p)[:-1], "number of rows"), + (lambda p: _mapped(p)[::-1], "identity/order"), + (lambda p: tuple(replace(row, joint=(.1, )) + for row in _mapped(p)), "joint dimension"), + ) + for callback, message in cases: + with pytest.raises(ValueError, match=message): + BatchedTarget(callback).evaluate(proposals, 2, True) + with pytest.raises(ValueError, match="Box target"): + BatchedTarget(_mapped).evaluate(proposals, 2, False) + for name in ("log_base", "log_likelihood"): + for invalid in (math.nan, math.inf): + with pytest.raises(ValueError, match="NaN or positive infinity"): + if name == "log_base": + replace(_target((.7, )), log_base=invalid) + else: + replace(_target((.7, )), log_likelihood=invalid) + with pytest.raises(ValueError, match="zero target support"): + replace(_target((.7, )), log_base=-math.inf) + with pytest.raises(ValueError, match="finite"): + replace(_target((.7, )), joint=(math.nan, 0.)) + + +def test_box_target_and_zero_support() -> None: + """Unconditioned batches retain the box prior and failed-result + semantics.""" + _, identity, config = _setup() + prior = BoxPrior(("x", ), ((0., 1.), )) + identity = replace(identity, prior=prior.digest) + config = replace(config, particles=1200) + + def constant( + proposals: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple(TargetEvaluation(p, p, 0., 0.) for p in proposals) + + result = sample_batch(prior, identity, BatchedTarget(constant), config, 7) + assert result.status == "complete" + x = np.asarray(result.samples)[:, 0] + assert np.average(x, weights=result.weights) == pytest.approx(.5, abs=.025) + assert np.average(x**2, weights=result.weights) == pytest.approx(1 / 3, + abs=.025) + + def unsupported( + proposals: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple(TargetEvaluation(p, p, 0., -math.inf) for p in proposals) + + saved: List[SamplerCheckpoint] = [] + failed = sample_batch(prior, + identity, + BatchedTarget(unsupported), + config, + 7, + checkpoint=saved.append) + assert failed.status == "no_particle_support" + assert failed.evaluations == config.particles + assert not failed.samples and not saved From a2e0c3b8a54729a0359cad90828bd3ae351e6201 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 05:37:06 -0400 Subject: [PATCH 06/94] Mix global block refreshes into offline posterior exploration --- docs/uncertainty/implementation-progress.md | 6 + docs/uncertainty/proposal-refresh.md | 84 +++++++++++ .../code_sim_learning/inference_sampling.py | 33 ++++- .../test_inference_refresh.py | 132 ++++++++++++++++++ 4 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 docs/uncertainty/proposal-refresh.md create mode 100644 tests/code_sim_learning/test_inference_refresh.py diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 221b7f27c..ccca0ec9a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -27,6 +27,12 @@ Its scalar path retains the original random schedule, while batch mode uses a se Thirty-two functional tests, sixteen exact comparisons against the original scalar implementation and final focused type/lint/format checks pass. The native Domino check reproduces all target values and complete sampler output exactly across one and four processes, with a measured 3.76-fold sampler speedup; numerical adequacy remains unestablished. +The subsequent [conditional-parameter slices and mixed proposals](proposal-refresh.md) identify a concrete exploration issue: restitution changes leave the complete fitting history unchanged at two checked scenes, yet the two populations retain narrow, different restitution ranges. +A mixed local/full-range block proposal now preserves the same fixed conditional target while allowing larger numerical moves. +Forty-one functional tests, thirty-two disabled-refresh compatibility comparisons and final focused type/lint/format checks pass. +The native mixed-proposal check also matches target values and sampler output exactly between synchronous and four-process execution. +The two refresh-arm Domino fits are running; two matched local-only controls await the array concurrency limit, and no new posterior is yet available. + The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. Twenty-three functional tests and focused type/lint/format checks pass; running fits retain their existing frozen source. diff --git a/docs/uncertainty/proposal-refresh.md b/docs/uncertainty/proposal-refresh.md new file mode 100644 index 000000000..62537b3c2 --- /dev/null +++ b/docs/uncertainty/proposal-refresh.md @@ -0,0 +1,84 @@ +# Exploring the fixed posterior with global block proposals + +September 13, 2026. +This is an offline numerical change within Stage B of the [simplification proposal](simplification-proposal.md). +It leaves the original prior, conditional coordinate map, observation model and agent unchanged. + +## Evidence motivating the change + +The completed Domino runs select uniformly among 25 proposal blocks and make eight moves at each of 32 temperatures. +A given physical parameter therefore receives only 10.24 proposed updates per particle in expectation before accounting for acceptance or resampling ancestry. +The aggregate accepted-move count includes state and auxiliary coordinates, so it does not establish parameter exploration. + +Compute job `22672492` evaluates conditional target slices at the two archived best scenes. +For each scene it checks seven prior quantiles of each physical parameter, holding all other coordinates fixed. +It also repeats both original reference points and reproduces their archived likelihoods exactly. +The worker performs 4,736 native actions and finishes in 40.19 seconds on four compute CPUs. + +| Parameter | Finite log-target range at scene 100 | Finite log-target range at scene 101 | Finite grid points per scene | +| --- | ---: | ---: | --- | +| Lateral friction | 1334.23 | 402.91 | 7/7, 7/7 | +| Restitution | 0 | 0 | 7/7, 7/7 | +| Rolling friction | 727.03 | 108.52 | 7/7, 7/7 | +| Spinning friction | 135.64 | 6.98 | 7/7, 7/7 | +| Mass | 86.07 | 11.73 | 6/7, 6/7 | + +Changing restitution leaves the complete predicted 64-action history byte-for-byte identical at both checked scenes. +Nevertheless, the completed populations place restitution in different narrow ranges: approximately [0.259, 0.635] and [0.689, 0.900]. +The slices are conditional checks at two scenes, not a proof that restitution is globally unidentifiable or that its full posterior equals its prior. +They do show that narrow, different finite-population summaries cannot be taken as evidence of identification without an exploration check. +The other parameters have substantial conditional likelihood variation, so replacing all local moves with global proposals could sharply reduce their acceptance. +Inputs, physical predictions' hashes and complete slice values are retained in `logs/uncertainty_domino_parameter_slices_20260913`. + +## Mixed proposal kernel + +`SamplerConfig.refresh_probability` mixes local Gaussian random walks with independent uniform proposals within the selected block of the declared proposal box. +The block-selection distribution and mixture probability are fixed independently of the current state. +Both proposal components are symmetric on that box, so the existing Metropolis ratio remains valid, including the conditional-base density correction and tempered remaining likelihood. +A refresh proposes a new point; it does not automatically accept it, replace the fixed generative prior or discard accumulated evidence. +The mixture is a numerical exploration strategy, not another statistical uncertainty estimate. + +Probability zero preserves the existing scalar and batch random streams exactly. +The checkpoint signature omits the newly added zero-valued field for compatibility, while a nonzero value participates in the signature and rejects mismatched continuation. +Serialized configuration reports include the explicit new field. +There is no agent-facing flag or deployment change. + +## Numerical validation + +A public-sampler reference reproduces concentration of an unobserved independent coordinate when an informative nuisance coordinate causes resampling to one ancestor and local moves are too small to explore its prior range. +Full-range block proposals recover the unobserved coordinate's uniform mean and variance in both scalar and batch execution across two seeds. +That recovery does not certify the separate approximation of the sharply observed coordinate. +A truncated Beta reference checks the nonlinear coordinate factor, mixed and fully global proposals, blocked and full-vector modes, and checkpoint replay. +Budget exhaustion still returns no usable posterior samples. + +Compute job `22672869` passes all 41 functional tests and 32 comparisons against the previous sampler, including exact old-checkpoint continuation with refresh disabled. +Its static check exposed an inferred test-variable type of `object`; the fixture now explicitly annotates the callable-or-batched-target union. +Final check job `22673128` reran all nine new reference tests and 32 compatibility comparisons and passed two-file mypy, pylint and pinned formatting checks. +The follow-up checks and native process-equivalence validation retain frozen sources in `logs/uncertainty_refresh_checks_v2_20260913` and `logs/uncertainty_domino_refresh_validation_20260913`. + +## Matched Domino experiment + +The prepared comparison uses two numerical seeds, 100 and 101, for each of a local-only arm and a 50/50 local/full-range arm. +Both use 64 particles, 32 cubic-spaced temperatures, eight moves, the same 25 blocks, local scale 0.05 and at most 16,448 target evaluations. +Both evaluate complete targets through four isolated processes with the same frozen program, training prefix, conditional map, prior and output model. +The refresh probability is the manipulated setting; the sampled random trajectories can differ between arms. +Each run verifies an archived target point before fitting and records weighted parameter summaries and ancestry at every complete stage. +Checkpoints preserve cumulative numerical budgets across interruption. + +The frozen plan also declares exploratory comparisons of their subsequent 97-action forecasts: at most 0.0025 m RMS difference between conditional position means, 0.20 maximum toppling-curve probability difference and 0.15 final toppling-probability difference between numerical replicas. +These are triage thresholds, not a posterior-adequacy certificate or confidence interval. +Parameter exploration, model mismatch, predictive errors and cost remain separate diagnostics. +Passing this pilot still requires budget sensitivity and broader comparisons before posterior use in planning. +The old production estimator remains the default. + +Array `22673150` is submitted on `mit_preemptable`, dependent on successful final static and native validation (`22673128` and `22672945`). +Tasks 0 and 1 are the refresh arm; tasks 2 and 3 are local-only controls, with numerical seeds 100 and 101 in each arm. +Each task requests four CPUs on the same declared AMD worker node, with a four-hour limit and a maximum of two simultaneous array tasks. +The complete source, submission manifest, prospective prediction thresholds and checkpoints are in `logs/uncertainty_domino_refresh_comparison_20260913`. +Queued, interrupted and incomplete fits are not completed posterior results. + +Native validation `22672945` completed successfully with a 50/50 proposal mixture. +All 32 fixed target evaluations and the subsequent 95-evaluation sampler result match exactly between synchronous and four-process execution. +The sampler portion took 96.91 seconds synchronously and 25.40 seconds in parallel. +This short run retains one ancestor and remains a kernel-equivalence check, not a numerically adequate physical posterior. +The validation dependency is satisfied, and comparison tasks 0 and 1 have started; the local-only controls await the two-task concurrency limit. diff --git a/predicators/code_sim_learning/inference_sampling.py b/predicators/code_sim_learning/inference_sampling.py index 4cdf35b5f..78f333e3e 100644 --- a/predicators/code_sim_learning/inference_sampling.py +++ b/predicators/code_sim_learning/inference_sampling.py @@ -69,7 +69,10 @@ class SamplerConfig: Nonempty blocks partition the proposal coordinates and are selected uniformly for symmetric moves. Empty blocks preserve full-vector moves. An explicit increasing schedule may replace equally spaced - temperatures, with the same stage count and final target. + temperatures, with the same stage count and final target. A fixed + refresh probability mixes in independent uniform proposals within + the selected block. Zero preserves the original random-walk + schedule. """ particles: int = 512 temperatures: int = 32 @@ -79,6 +82,7 @@ class SamplerConfig: resample_ess_fraction: float = 0.5 proposal_blocks: Tuple[Tuple[int, ...], ...] = () temperature_schedule: Tuple[float, ...] = () + refresh_probability: float = 0. def __post_init__(self) -> None: for value in (self.particles, self.temperatures, self.moves, @@ -89,6 +93,9 @@ def __post_init__(self) -> None: raise ValueError("Proposal scale must be finite and positive") if not 0 < self.resample_ess_fraction <= 1: raise ValueError("Resampling ESS fraction must lie in (0, 1]") + if not math.isfinite(self.refresh_probability) or not \ + 0 <= self.refresh_probability <= 1: + raise ValueError("Refresh probability must lie in [0, 1]") blocks = tuple(tuple(block) for block in self.proposal_blocks) members = [index for block in blocks for index in block] if any(not block for block in blocks) or any( @@ -282,7 +289,12 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], uniform for every proposal, including rejected ones, so worker scheduling cannot change the random stream. Its separately identified checkpoint kernel cannot resume scalar-mode checkpoints. Scalar calls preserve the - original draw schedule. Both modes target the same declared distribution; + original draw schedule. A nonzero refresh probability mixes independent + uniform proposals within the selected block into either execution mode. + Both proposal components are symmetric on the declared box, so the same + conditional-base and likelihood Metropolis ratio applies. Refreshing is + a numerical move, not resampling parameters from a new inference prior. + Both modes target the same declared distribution; their random streams and resulting finite populations can differ. """ if identity.prior != prior.digest: @@ -319,6 +331,10 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], accepted = 0 attempted = 0 completed_stage = 0 + signature_config = asdict(config) + if config.refresh_probability == 0.: + # Keep default checkpoints compatible with the original schedule. + del signature_config["refresh_probability"] signature = content_digest( json.dumps( { @@ -326,7 +342,7 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], if batched else "tempered_smc_stage_checkpoint_v1", "identity": identity.digest, "prior": prior.digest, - "config": asdict(config), + "config": signature_config, "seed": seed, "numpy": np.__version__ }, @@ -461,9 +477,16 @@ def propose(index: int) -> np.ndarray: block = list(config.proposal_blocks[int( rng.integers(len(config.proposal_blocks)))]) proposal = particles[index].copy() - proposal[block] += rng.normal(size=len(block)) * \ - (upper[block] - lower[block]) * config.proposal_scale + if config.refresh_probability > 0 and \ + rng.random() < config.refresh_probability: + proposal[block] = rng.uniform(lower[block], upper[block]) + else: + proposal[block] += rng.normal(size=len(block)) * \ + (upper[block] - lower[block]) * config.proposal_scale return proposal + if config.refresh_probability > 0 and \ + rng.random() < config.refresh_probability: + return rng.uniform(lower, upper) return particles[index] + rng.normal( size=len(proposal_prior.names)) * \ (upper - lower) * config.proposal_scale diff --git a/tests/code_sim_learning/test_inference_refresh.py b/tests/code_sim_learning/test_inference_refresh.py new file mode 100644 index 000000000..e477e45b1 --- /dev/null +++ b/tests/code_sim_learning/test_inference_refresh.py @@ -0,0 +1,132 @@ +"""Global block moves recover uncertainty lost through finite-population +drift.""" +import math +from dataclasses import replace +from typing import Callable, List, Tuple, Union + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_checkpoint import \ + SamplerCheckpoint +from predicators.code_sim_learning.inference_data import InferenceIdentity, \ + content_digest +from predicators.code_sim_learning.inference_evaluation import BatchedTarget, \ + TargetEvaluation +from predicators.code_sim_learning.inference_sampling import BoxPrior, \ + ConditionedPrior, PriorPoint, SamplerConfig, sample_batch + + +def _identity(prior: BoxPrior) -> InferenceIdentity: + digest = content_digest(b"refresh numerical reference") + return InferenceIdentity(digest, digest, digest, prior.digest, digest) + + +@pytest.mark.parametrize("seed", [7, 19]) +@pytest.mark.parametrize("batch", [False, True]) +def test_uninformed_coordinate_after_weight_collapse(seed: int, + batch: bool) -> None: + """A sharply observed nuisance coordinate cannot identify its independent + companion.""" + prior = BoxPrior(("observed", "unused"), ((0., 1.), (0., 1.))) + config = SamplerConfig(particles=512, + temperatures=1, + moves=12, + proposal_scale=1e-6, + max_evaluations=6656, + proposal_blocks=((0, ), (1, ))) + + def likelihood(value: np.ndarray) -> float: + return float(-.5 * ((value[0] - .613) / 1e-6)**2) + + def mapped( + points: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple( + TargetEvaluation(p, p, 0., likelihood(np.asarray(p))) + for p in points) + + target: Union[Callable[[np.ndarray], float], BatchedTarget] = \ + BatchedTarget(mapped) if batch else likelihood + local = sample_batch(prior, _identity(prior), target, config, seed) + refreshed = sample_batch(prior, _identity(prior), target, + replace(config, refresh_probability=1.), seed) + assert local.status == refreshed.status == "complete" + assert local.surviving_ancestors == 1 + local_unused = np.asarray(local.samples)[:, 1] + assert np.var(local_unused) < 1e-8 + unused = np.asarray(refreshed.samples)[:, 1] + assert np.average(unused, + weights=refreshed.weights) == pytest.approx(.5, abs=.04) + assert np.average((unused - .5)**2, + weights=refreshed.weights) == pytest.approx(1 / 12, + abs=.015) + # Recovery of this independent coordinate does not certify the sharply + # concentrated observed-coordinate approximation. + + +@pytest.mark.parametrize("refresh", [.4, 1.]) +@pytest.mark.parametrize("blocked", [False, True]) +def test_conditional_measure_and_checkpoint(refresh: float, + blocked: bool) -> None: + """Refreshes retain nonlinear coordinate factors and exact replay.""" + box = BoxPrior(("u", ), ((0., 1.), )) + digest = content_digest(b"conditional refresh") + prior = ConditionedPrior(("x", ), digest, digest, box) + identity = replace(_identity(box), prior=prior.digest) + config = SamplerConfig(particles=1200, + temperatures=4, + moves=4, + refresh_probability=refresh, + proposal_blocks=((0, ), ) if blocked else ()) + + def condition(value: np.ndarray) -> PriorPoint: + return PriorPoint((value[0]**2, ), math.log(2 * value[0])) + + def likelihood(value: np.ndarray) -> float: + return math.log(value[0]) if value[0] >= .2 else -math.inf + + saved: List[SamplerCheckpoint] = [] + result = sample_batch(prior, + identity, + likelihood, + config, + 9, + condition=condition, + checkpoint=saved.append) + assert result.status == "complete" + x = np.asarray(result.samples)[:, 0] + expected = 2 / 3 * (1 - .2**3) / (1 - .2**2) + assert np.average(x, weights=result.weights) == pytest.approx(expected, + abs=.018) + assert np.min(x[np.asarray(result.weights) > 0]) >= .2 + assert sample_batch(prior, + identity, + likelihood, + config, + 9, + condition=condition, + resume=saved[2]) == result + with pytest.raises(ValueError, match="Checkpoint differs"): + sample_batch(prior, + identity, + likelihood, + replace(config, refresh_probability=0.), + 9, + condition=condition, + resume=saved[2]) + + +def test_refresh_invalid_probability_and_budget() -> None: + """A global move is charged normally and cannot publish a partial fit.""" + for invalid in (-.01, 1.01, math.nan, math.inf): + with pytest.raises(ValueError, match="Refresh probability"): + SamplerConfig(refresh_probability=invalid) + prior = BoxPrior(("x", ), ((0., 1.), )) + config = SamplerConfig(particles=16, + max_evaluations=17, + refresh_probability=1.) + result = sample_batch(prior, _identity(prior), lambda _: 0., config, 0) + assert result.status == "budget_exhausted" + assert result.evaluations == 17 + assert not result.samples and not result.weights From f2fac5c5a8bb5c02bc22a24c082d31723fd4bf4c Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 06:03:50 -0400 Subject: [PATCH 07/94] Validate point-start ablation and complete-checkpoint forecasts --- docs/uncertainty/checkpoint-forecasts.md | 58 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 +++ docs/uncertainty/initial-state-ablation.md | 54 +++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 docs/uncertainty/checkpoint-forecasts.md create mode 100644 docs/uncertainty/initial-state-ablation.md diff --git a/docs/uncertainty/checkpoint-forecasts.md b/docs/uncertainty/checkpoint-forecasts.md new file mode 100644 index 000000000..212d98d39 --- /dev/null +++ b/docs/uncertainty/checkpoint-forecasts.md @@ -0,0 +1,58 @@ +# Forecasting complete offline checkpoints + +September 13, 2026. +This follow-up evaluates the numerical exploration comparison and the [initial-state ablation](initial-state-ablation.md) on their common 97-action Domino suffix. +It is an offline diagnostic of unassessed populations, not posterior publication or an agent solve-rate experiment. + +The worker snapshots the selected complete fit report and checksum-protected checkpoint into its own artifact directory. +It reconstructs the typed inference request and resumes the completed checkpoint with a target callback that must never be called. +The resulting complete population, weights, counters and diagnostics must exactly match the fit report. +A missing or incomplete source cannot produce forecasts, and a report/checkpoint mismatch fails validation. +The explicit zero refresh configuration field is the only compatibility adjustment for earlier reports that predate that field. + +Each positive-weight row reconstructs its physical state from the saved proposal coordinates and verifies exact equality with the retained physical parameters and state coordinates. +This avoids inverting physical parameters back into unit coordinates, which could introduce a rounding error before replay. +Point-start rows combine their five saved parameter coordinates with the frozen constant state coordinates. +Zero-weight rows are recorded as such and contribute no forecast mass; positive-weight histories are never dropped because their future predictions are poor. + +Every physical continuation replays all 161 recorded actions uninterrupted in its own fresh world. +The first positive-weight history is repeated exactly, and each row must reproduce its saved fitting-prefix factor and the declared output-model identity. +Joint and point-start targets check their different base-factor conventions explicitly. +Forecast means and variances use only the fitting-prefix observations to condition the scalar output-error process. +Future observations enter only after the physical continuations and prefix-conditioned moments have been constructed, for density and error evaluation. +Posterior weights remain the fitting weights; future likelihoods do not reweight the plotted forecasts. + +The output preserves complete compressed histories, hashes, weighted Cartesian moments, native toppling curves and whole-future mixture scores. +Work is counted in native actions, including the repeated reference history. +The source program was historically synthesized from training experience, so this suffix is not established as unseen during program synthesis. +It remains a fixed-program, fixed-recorded-action prediction diagnostic. + +Before applying the worker to pending fits, compute job `22673647_0` checks the entire saved population from the completed original numerical seed 100. +It must exactly reproduce the earlier scalar forecast's means, variances, toppling curves, errors and mixture score using the new checkpoint-driven, four-process path. +The validation and prospective source mapping are frozen in `logs/uncertainty_domino_checkpoint_forecasts_20260913`. +New-source forecast jobs will require both that validation and their own source fit to complete successfully. + +## Completed validation and scheduled follow-ups + +The first attempt, `22673647_0`, failed because its frozen observation module predated `log_future_likelihood`. +That setup failure and its allocation are retained; the failed worker did not report a complete native-action count. +The corrected bundle uses the same observation module as the previously validated scalar forecast and checks for the scoring method before dispatching histories. +The running fits were not changed. + +Corrected job `22673693_0` completed successfully in an 83-second allocation, with 77.38 worker seconds and 10,465 native actions including the repeated first row. +The reconstructed checkpoint, all aggregate metrics and all 64 complete decoded history artifacts match the previous forecast exactly. +The conditional Cartesian RMSE is 0.01158500918 m and the whole-future mixture log score is 14105.63277779, reproducing the earlier diagnostic rather than providing a new estimator result. +Artifact hashes and decoded-history equality are independently verified in `validation-verification.json`. + +| Forecast job | Source fit | Arm | Numerical seed | +| --- | --- | --- | --- | +| `22673718_1` | `22673150_0` | Joint state, mixed proposals | 100 | +| `22673719_2` | `22673150_1` | Joint state, mixed proposals | 101 | +| `22673720_3` | `22673150_2` | Joint state, local proposals | 100 | +| `22673721_4` | `22673150_3` | Joint state, local proposals | 101 | +| `22673722_5` | `22673316_0` | Point start, mixed proposals | 100 | +| `22673723_6` | `22673316_1` | Point start, mixed proposals | 101 | + +Each submitted follow-up requires both successful validation and successful completion of its source fit. +The corrected source, source mapping and submission manifest are in `logs/uncertainty_domino_checkpoint_forecasts_v2_20260913`. +These compute dependencies generate diagnostics only; they do not restore the disabled MB/MF task monitor or send result notifications to other tasks. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index ccca0ec9a..d73fddfc1 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -33,6 +33,12 @@ Forty-one functional tests, thirty-two disabled-refresh compatibility comparison The native mixed-proposal check also matches target values and sampler output exactly between synchronous and four-process execution. The two refresh-arm Domino fits are running; two matched local-only controls await the array concurrency limit, and no new posterior is yet available. +The [initial-state ablation](initial-state-ablation.md) now supplies a first-observation-only point-start comparison while retaining the original parameter prior and output model. +Its selected scene is feasible, repeats exactly and yields finite full-prefix likelihoods at 39 of 64 random parameter settings. +Two parameter-only fits are running alongside the joint fits; this is an explicitly labeled approximation, not an exact state observation. +The [checkpoint-driven forecast follow-up](checkpoint-forecasts.md) now reproduces every complete history and aggregate metric from the earlier 64-row forecast exactly. +Six follow-up jobs are submitted with dependencies on successful completion of their individual source fits; no new prediction comparisons are complete yet. + The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. Twenty-three functional tests and focused type/lint/format checks pass; running fits retain their existing frozen source. diff --git a/docs/uncertainty/initial-state-ablation.md b/docs/uncertainty/initial-state-ablation.md new file mode 100644 index 000000000..b08b7628f --- /dev/null +++ b/docs/uncertainty/initial-state-ablation.md @@ -0,0 +1,54 @@ +# Initial-state ablation with a fixed parameter prior + +September 13, 2026. +This addresses the Stage B requirement to separate uncertain recording starts from parameter-inference changes in the [simplification proposal](simplification-proposal.md). +The new arm is an explicitly labeled point-state approximation for offline comparison, not a claim that the initial noisy observation reveals the true physical state. + +## Controlled comparison + +The joint arm uses the complete declared initial-scene law and the five original parameter priors from [Domino joint inference](domino-joint-inference.md). +The point-start arm retains those parameter priors, the same fixed program, the same 64-action observation ledger, and the same scalar/coupled-orientation output model. +It replaces uncertain initial-state inference with one deterministic scene chosen using only the first observation and the previously declared initialization policy. +No later fitting frame, held-out suffix, archived fitted scene or private evaluator state selects that point. + +The selection uses the higher-mass rest case, exact observed controlled joints, and coordinate-wise medians of the original first-observation proposal for remaining quantities. +Horizontal body positions and yaw therefore come from the truncated first-reading proposals, support heights follow the declared upright rest geometry, and unobserved Gaussian joint positions take their prior medians. +Rest velocities remain zero as a deliberate point approximation, not a conclusion drawn from missing recording metadata. +The selector coordinate is 0.4 within the original rest interval [0, 0.8), and other initial-state proposal coordinates are 0.5. +If that scene fails geometry or initial-observation support, the protocol reports it unsupported rather than searching for a replacement using later observations. + +Writing this point as `x_hat(o_0)`, the ablation targets the original parameter prior times the conditional remaining-recording likelihood at that fixed state. +The output-error process is still conditioned on the first observation, preserving its declared temporal dependence. +The initial-observation density and fixed-state/proposal factor are constant in the parameters at this scene and cancel from their normalized posterior approximation. +The live target checks that the removed factor remains equal to the frozen preflight value for every evaluation. +This is not the full joint posterior with state uncertainty integrated out, and its apparent parameter confidence must be interpreted accordingly. + +## Native preflight + +Compute job `22673275` completed on `mit_preemptable` with four CPUs. +The median scene passes the declared geometric checks and has finite initial-observation likelihood, with log density 88.31077418. +The median-parameter reference repeats exactly. +All 64 separately sampled parameter settings retain the same initial state and initial-base factor; 39 have finite likelihood over the complete 64-action prefix. +The remaining zero-likelihood settings stay in the report and are not discarded as infrastructure failures. +The preflight performs 4,224 native actions in 42.11 worker seconds. +A finite search failing to find support would not, by itself, prove that the fixed-state model is inconsistent. + +The report, selected scene, first-observation identity, parameter proposals and complete outcomes are in `logs/uncertainty_domino_point_start_preflight_20260913`. +The state-selection rule was fixed before those subsequent-action likelihoods were inspected. + +## Submitted fits + +Array `22673316` runs numerical seeds 100 and 101 under the point-state approximation. +Both use the tested 50/50 local/full-range proposal mixture, 64 particles, 32 cubic-spaced temperatures, eight moves, local scale 0.05 and a maximum of 16,448 evaluations. +Each requests four CPUs, 20 GB and four hours on the same declared AMD worker node, with complete-stage checkpoints and at most two simultaneous tasks. +The worker first reproduces the frozen preflight target and verifies the output-model identity. +Both tasks have started and passed their preflights. + +There are five active proposal coordinates, one for each physical parameter, rather than the joint arm's 95-coordinate representation. +The complete output rows retain the constant initial-state coordinates for unambiguous replay. +Blocks partition the active coordinates, so the point-start arm gives each parameter more proposed updates within the same total evaluation budget. +That reduced numerical difficulty is recorded explicitly; prediction differences cannot be interpreted before assessing each approximation's stability. +The ablation does not isolate the legacy carried-prior policy, whose sequential comparison remains required separately. + +Frozen source, runtime identities, submission hashes, stage summaries and checkpoints are in `logs/uncertainty_domino_point_start_fit_20260913`. +The production agent remains unchanged, and incomplete fits are not usable posterior results. From 3d2fd1e7517012e6650f3eb238aa3912333cfc9f Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 06:21:32 -0400 Subject: [PATCH 08/94] Document causal Fan prefix support validation --- docs/uncertainty/fan-prefix-comparison.md | 49 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 16 +++++++ 2 files changed, 65 insertions(+) create mode 100644 docs/uncertainty/fan-prefix-comparison.md diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md new file mode 100644 index 000000000..bfef483de --- /dev/null +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -0,0 +1,49 @@ +# Fan prefix comparison + +This is a Stage B development comparison under the [simplification proposal](simplification-proposal.md). +The existing full-recording Fan inference pilots use all 132 actions, so they cannot provide an unused suffix from that recording. +The separate legacy fit `22638308_1` uses 64 actions and already supplies predictions for the remaining 68 actions. + +## Prefix-only support check + +Job `22674026` completed successfully on `mit_preemptable` on September 13, 2026, with one CPU on node1412. +It used 106 allocation seconds, 90.47 worker seconds and 4,160 native actions under its 20-minute allocation limit. +Its frozen bundle is `logs/uncertainty_fan_prefix_support_20260913`. +The submission manifest records worker and configuration hashes. + +The worker retains only the initial observation and first 64 action/observation transitions before constructing any candidate scene or scoring predictions. +It checks that the sensor schema and every retained observation use exactly the first observation's feature keys. +The full recording hash remains provenance, while the inference data identity describes only the retained prefix. +It uses the same historical program, physical runtime, original scene prior and output discrepancy model as the full-recording Fan pilots. +The historical program may have seen later training experience during synthesis, so this is an estimator comparison on a reserved suffix, not an unseen-program evaluation. + +The search does not import the full-recording supported candidate or its guided proposal. +It checks 16 stratified original-prior fan speeds at the first-observation proposal median scene, followed by 48 independently sampled scenes from the original first-observation proposal restricted to its robot/ball rest components. +Rest-component restriction is a declared support-search choice; these candidates are neither joint-prior samples nor posterior samples. +Unknown rotor state and fixture locations retain the declared chart semantics. +No omitted velocity is claimed to be an exact observed zero. + +Every candidate is simulated for 64 native actions in a fresh world, with geometry witnesses, full-prefix likelihood, initial likelihood and complete prediction digest retained. +The first feasible finite candidate is repeated exactly; if none is found, the first median candidate is repeated instead. +A finite search with no supported candidate cannot prove that the target has no support. + +## Acceptance and follow-up + +The completed native audit found the following support: + +| Candidate construction | Probes | Valid initial geometry | Finite likelihood and valid geometry | +| --- | ---: | ---: | ---: | +| First-observation proposal median, stratified speed | 16 | 0 | 0 | +| Sampled first-observation proposal, robot/ball rest cases | 48 | 12 | 8 | + +The first supported candidate, index 26, repeated with exactly matching likelihoods, geometry witnesses and complete predicted-observation digest. +The median scene has a 5.11 mm initial penetration and is unsuitable as a fixed-state baseline without a separately declared feasible selection policy. +The independent verification checks all 64 result records, native action accounting and all submitted script hashes. +Reports and verification are retained in the frozen bundle. + +This establishes prefix-only support for constructing a prefix-specific inference proposal; it does not establish numerical adequacy. +Any selected guide must depend only on the prefix, and its proposal density must be included in the inference correction. +Keep a component with full original support if adding local guidance. +Independent numerical replicas and budget sensitivity remain required before treating fitted samples as usable posterior estimates. +Only then compare the frozen-weight predictions on the remaining 68 actions against the existing legacy predictions, including ball trajectory, event timing, goal outcomes and computation cost. +This support audit produces no agent solve-rate result and does not change the production fitter. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index d73fddfc1..99a4d60d0 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -4,6 +4,22 @@ Updated September 13, 2026. This tracks implementation of the [simplification proposal](simplification-proposal.md). The incumbent estimator remains the production default. +## Current stage + +Stage 0 interface preservation is complete, with scripted behavior parity checked. +Stage A has implemented probability and replay components, but physical support and numerical validation remain incomplete. +The active work is Stage B offline comparison; Stage C live posterior use and Stage E retirement are not yet approved by the evidence. +Stage D execution smoothing remains optional and deferred. +The full plan remains incomplete. + +The [Fan prefix comparison](fan-prefix-comparison.md) now has a completed compute support audit, `22674026`. +It removes the 68-action suffix before scene construction and scoring and does not reuse the full-recording proposal guide. +Eight of 48 sampled rest scenes have valid geometry and finite prefix likelihood, and the first supported trajectory repeats exactly. +All 16 median-scene probes instead have an initial penetration, ruling out that unmodified point-state baseline. +This prepares a comparison against the completed 64-action legacy fit while the existing full-recording fits continue unchanged. + +## Recent evidence + The new offline [stochastic future integration component](stochastic-future-integration.md) retains exact joint/speed density factors and reports Monte Carlo concentration explicitly. Forty functional tests and focused static checks pass. Its first native Balloons diagnostic reproduces the reference trajectory but fails all four numerical comparisons: eight and 64 complete paths remain dominated by a single contribution. From f609cce7c628d3591e6b37ae76e0063da05e9dee Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 06:33:11 -0400 Subject: [PATCH 09/94] Record validated parallel Fan prefix inference launch --- docs/uncertainty/fan-prefix-comparison.md | 33 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 +++ 2 files changed, 37 insertions(+) diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index bfef483de..239da5798 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -47,3 +47,36 @@ Keep a component with full original support if adding local guidance. Independent numerical replicas and budget sensitivity remain required before treating fitted samples as usable posterior estimates. Only then compare the frozen-weight predictions on the remaining 68 actions against the existing legacy predictions, including ball trajectory, event timing, goal outcomes and computation cost. This support audit produces no agent solve-rate result and does not change the production fitter. + +## Prefix inference protocol + +The follow-up bundle is `logs/uncertainty_fan_prefix_inference_20260913`. +It uses the original 120-coordinate proposal conditioned on the first reading, with the prior/proposal correction retained. +It adds no local mixture around a selected supported candidate. +Both resting and moving robot/ball cases remain available with their original probabilities, unlike the restricted support search above. +The parameter prior remains `Uniform[0, 1]` for fan speed. + +The base factor contains the initial observation likelihood and the original-prior/proposal correction, subject to initial geometry and complete-prefix exact constraints. +Tempering applies to the remaining prefix log likelihood, so each observation enters exactly once. +The inference data identity includes only the first 64 actions and 65 observations. +The fixed source, offline modules, worker scripts, actual numerical runtime and isolated working directory enter the runtime identity. + +Two numerical replicas are specified with seeds 302 and 303, 64 particles, 32 cubic temperature stages, eight moves per stage and a maximum of 16,448 target evaluations each. +Their proposal kernel mixes local moves with full-range block refreshes at probability 0.5. +The parameter has its own block; other blocks group coupled robot, fixture, ball, switch and rotor coordinates. +Four isolated processes evaluate complete candidate maps and likelihoods in order. +Complete-stage checkpoints preserve the fixed numerical budget across preemption; interrupted work remains additional actual cost. + +The compute preflight checks retained support-audit factors and identical initial sampler states across serial and four-process execution before the replicas may launch. +A completed fit remains numerically unassessed until independent prediction comparisons and budget sensitivity have been evaluated. +The two replicas are inference diagnostics on the same recording, not two agent seeds or evidence of solve-rate improvement. + +The corrected preflight `22674192` completed successfully in 159 allocation seconds and 152.93 worker seconds, with 9,728 native actions. +All 12 checked native target rows and the entire 64-particle initial sampler state match exactly between serial and four-process evaluation. +Initialization found 11 finite particles with weight effective sample size 9.10; this describes initialization only. +Serial initialization took 84.68 seconds and parallel initialization took 21.64 seconds, excluding process startup and the separate fixed-row checks. +The preceding attempt `22674140` failed in report construction because checkpoint log weights are serialized strings; its native comparisons had reached equality assertions, and its allocation and report remain retained. +The corrected attempt changes only summary decoding and adds the initial weight effective sample size. +The launch manifest pins the successful preflight, target scripts and configuration before any fit starts. +Array `22674242` submits task 0 for numerical seed 302 and task 1 for numerical seed 303, each with four CPUs, 20 GB and a four-hour allocation limit on `mit_preemptable`. +Neither has a completed posterior or prediction comparison yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 99a4d60d0..ba3250de1 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -17,6 +17,10 @@ It removes the 68-action suffix before scene construction and scoring and does n Eight of 48 sampled rest scenes have valid geometry and finite prefix likelihood, and the first supported trajectory repeats exactly. All 16 median-scene probes instead have an initial penetration, ruling out that unmodified point-state baseline. This prepares a comparison against the completed 64-action legacy fit while the existing full-recording fits continue unchanged. +The next prefix inference preflight, `22674192`, exactly matches 12 retained native targets and the complete initial population across serial and four-process execution. +It finds 11 supported initial particles with weight effective sample size 9.10 and verifies a roughly fourfold initialization speedup. +Two full-prior prefix fits are submitted as `22674242_0` and `_1`, with numerical seeds 302 and 303 and unchanged original support for moving and resting states. +These are offline inference replicas; completed prediction comparisons and numerical assessment remain pending. ## Recent evidence From c7fa8cb975305802be59efb27cd9b650166f8fbe Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 06:44:20 -0400 Subject: [PATCH 10/94] Validate Fan checkpoint forecasts and queue matched comparisons --- docs/uncertainty/fan-prefix-comparison.md | 36 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 39 insertions(+) diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index 239da5798..df0ea21c2 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -80,3 +80,39 @@ The corrected attempt changes only summary decoding and adds the initial weight The launch manifest pins the successful preflight, target scripts and configuration before any fit starts. Array `22674242` submits task 0 for numerical seed 302 and task 1 for numerical seed 303, each with four CPUs, 20 GB and a four-hour allocation limit on `mit_preemptable`. Neither has a completed posterior or prediction comparison yet. + +## Reserved-suffix forecast validation + +The separate bundle `logs/uncertainty_fan_prefix_forecast_20260913` prepares comparisons on the 68 actions reserved from fitting. +Its source guard requires a complete fit report and a matching, complete checksum-verified checkpoint. +Resuming that checkpoint must reproduce the entire saved sampler result without requesting any new target evaluation. +The source prior, inference identity and configuration must also match the validated fitting protocol. + +Prediction uses a separately identified observation module with conditional future scoring; the running fit snapshots remain unchanged. +Every positive-weight particle reconstructs its saved native scene directly from its original proposal coordinates. +Its physical fan speed, model digest and both prefix target factors must match the fitting checkpoint exactly. +Zero-weight rows are identified explicitly, and poor future likelihood never removes a positive-weight row or changes its weight. +Each trajectory runs all 132 actions from its initial scene, preserving the physical state through the fit/prediction boundary. +The first positive-weight trajectory is repeated, and compressed complete histories are retained with checksums. + +Ball-position predictions include both the native simulator mean and the output-error mean conditioned only on the prefix. +Reported events include switch/fan activation and the learned target-hit readout. +The native geometric goal predicate is evaluated separately using the frozen environment's axis tolerances, so a learned target-hit field cannot silently substitute for goal geometry. +The comparison reports goal probability, first goal occurrence within the suffix, event Brier errors and native computation cost alongside position error. +The legacy report must use the identical prefix data and candidate program, with predictions aligned by their primitive step indices. + +A short native fixture, `22674287`, completed one temperature stage and one move using the original fitting runtime, with 7,872 native actions and 66.08 worker seconds. +It has one surviving initial ancestor and is explicitly unassessed. +It exists only to exercise complete-checkpoint recovery and forecast comparison before the two full fits finish. + +Forecast validation `22674541_0` completed successfully in 71 allocation seconds and 64.73 worker seconds, with 8,580 native actions. +It recovered the complete checkpoint exactly and verified the saved prefix factors for every positive-weight particle under the forecast runtime. +Independent artifact checks verify all 64 complete 133-frame histories, their hashes, original physical samples and unchanged fitting weights. +All 64 fixture particles assign zero density to the observed future; they remain included in the trajectory, event and goal summaries, and the empirical mixture density is reported as zero. +This is useful coverage of the evaluator's failure handling, not evidence of a reliable posterior or an estimator advantage. +The two full-fit forecast jobs are submitted with dependencies on this successful validation and their individual source fits. + +| Numerical seed | Prefix fit | Dependent forecast | +| --- | --- | --- | +| 302 | `22674242_0` | `22674579_1` | +| 303 | `22674242_1` | `22674580_2` | diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index ba3250de1..9d16521ea 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -21,6 +21,9 @@ The next prefix inference preflight, `22674192`, exactly matches 12 retained nat It finds 11 supported initial particles with weight effective sample size 9.10 and verifies a roughly fourfold initialization speedup. Two full-prior prefix fits are submitted as `22674242_0` and `_1`, with numerical seeds 302 and 303 and unchanged original support for moving and resting states. These are offline inference replicas; completed prediction comparisons and numerical assessment remain pending. +The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. +It retains every positive-weight particle even though all fixture future densities are zero. +Two forecast follow-ups are submitted with dependencies on successful completion of their individual full prefix fits; no full-fit Fan prediction result is available yet. ## Recent evidence From 16b5720329a4f9a735e5341e77063d2a97d9fa63 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 06:53:23 -0400 Subject: [PATCH 11/94] Add paired sequential carried-center comparison protocol --- docs/uncertainty/carried-center-comparison.md | 54 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 57 insertions(+) create mode 100644 docs/uncertainty/carried-center-comparison.md diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md new file mode 100644 index 000000000..bea99262f --- /dev/null +++ b/docs/uncertainty/carried-center-comparison.md @@ -0,0 +1,54 @@ +# Isolating carried prior centers + +The [Stage B plan](simplification-proposal.md) requires separating the effects of fixed-prior fitting and uncertain initial-state inference. +The existing cold legacy comparisons contain no previous fit, so they cannot exercise carrying. +This experiment isolates carrying within the legacy fitter, separately from the new joint sampler and the initial-state ablation. + +The implemented legacy policy carries accepted fitted parameter values into subsequent prior centers. +It does not carry a full posterior density or its covariance. +An anchored fallback does not update the stored center, and an earlier accepted center can remain stored across later fits that do not replace it. +The experiment calls the actual `fit_prior_anchors` and `note_carried_posterior` methods to preserve these semantics. + +## Paired protocol + +Use the same frozen Fan and Domino programs, latest parameter declarations, physical runtime, noise settings, trajectory preparation and legacy width/selection rules as the completed cold comparisons. +One arm retains registry prior centers; the other enables `code_sim_learning_carry_posterior`. +Both begin without carried history. +The fitting prefix grows from 64 to 96 actions, repeats the identical 96-action fit, then uses the complete recording. +Full recordings contain 132 Fan actions and 161 Domino actions. +Predictions after each fit use its selected values, and actions beyond that fit's prefix remain unused by fitting. +The final full-recording fit has no reserved suffix. + +Both arms use fixed declared optimizer initial values and pass their own previously selected values to the same hold policy, with no parameter declaration edits. +Neither uses a fit cache or explainability cache, so the repeated-data stage actually reruns fitting. +Neither adds a cross-cycle consistency adjustment. +These choices isolate the carried-center policy; they are not a reconstruction of a complete agent conversation or every production publication path. +The parameter prior family and initial-state treatment remain those of the legacy fitter. +In particular, the fixed-center arm is not the new fixed-prior Bayesian estimator. + +The first 64-action fit must exactly reproduce the archived cold fit's fitted values, selected values and diagnostic report before later stages proceed. +The original observed frames must remain unchanged across preparation, fitting and prediction. +Each stage records its data identity, entering anchors, carried values before and after fitting, selected parameters, legacy widths, full predictions and native computation cost. +The repeated 96-action stages share an identical data identity. + +## Interpretation and execution + +The comparison checks whether carrying changes applied parameters, diagnostic widths or predictions, including after a repeated fit with no new evidence. +Legacy diagnostic widths are not newly asserted to be credible intervals. +If neither domain carries an accepted center into a later fit, retain that result as an inactive-policy control; it cannot establish that removing active carrying is harmless. +Active-policy coverage and the remaining five-domain inference comparisons would still be needed. + +The frozen bundle is `logs/uncertainty_carried_prior_comparison_20260913`. +Array `22674821` is submitted to `mit_preemptable`, with six CPUs, 16 GB and a two-hour allocation limit per task on node1412, allowing two concurrent tasks. + +| Task | Domain | Carry accepted centers | +| --- | --- | --- | +| `22674821_0` | Fan | Off | +| `22674821_1` | Fan | On | +| `22674821_2` | Domino | Off | +| `22674821_3` | Domino | On | + +The preceding array `22674643` failed its configuration guard before fitting because the guard compared runtime tuples with JSON lists. +The serialized configurations were verified identical; the corrected guard normalizes representation before comparison. +The failed reports and original driver are retained, and those setup outcomes are not agent failures. +The replacement array is currently queued for resources; no paired outcome is available yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 9d16521ea..7871d9933 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -24,6 +24,9 @@ These are offline inference replicas; completed prediction comparisons and numer The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. It retains every positive-weight particle even though all fixture future densities are zero. Two forecast follow-ups are submitted with dependencies on successful completion of their individual full prefix fits; no full-fit Fan prediction result is available yet. +The separate [carried-center comparison](carried-center-comparison.md) now has four paired legacy tasks queued as `22674821`, covering Fan and Domino with carrying off/on. +Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. +It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. ## Recent evidence From a5f41e21f23e7f6f0806433d336b7c28dcb1584b Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:05:54 -0400 Subject: [PATCH 12/94] Validate combined prediction assessment and record Fan carry control --- docs/uncertainty/carried-center-comparison.md | 19 ++++++- docs/uncertainty/domino-comparison-summary.md | 50 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 ++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/domino-comparison-summary.md diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index bea99262f..e6e7ebf9b 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -51,4 +51,21 @@ Array `22674821` is submitted to `mit_preemptable`, with six CPUs, 16 GB and a t The preceding array `22674643` failed its configuration guard before fitting because the guard compared runtime tuples with JSON lists. The serialized configurations were verified identical; the corrected guard normalizes representation before comparison. The failed reports and original driver are retained, and those setup outcomes are not agent failures. -The replacement array is currently queued for resources; no paired outcome is available yet. +The Fan pair has completed successfully; the Domino pair is running. + +## Completed Fan control + +Both Fan arms reproduce the archived cold fit exactly and produce identical selected parameters, diagnostic widths and complete prediction histories at every stage. + +| Fitted actions | Selected fan speed in both arms | Legacy diagnostic width | Carry active before fit | +| --- | ---: | ---: | --- | +| 64 | 0.0846 | 0.1 | No | +| 96 | 0.0846 | 0.1 | No | +| 96, repeated data | 0.0846 | 0.1 | No | +| 132 | 0.0846 | 0.1 | No | + +Every verdict is anchored, so no accepted fitted center enters a subsequent prior. +The repeated 96-action fits change neither the selected value nor the prediction history in either arm. +Each arm uses 11,332 recorded native simulator actions across the four fits and their predictions. +The paired comparison and source hashes are retained in `fan-comparison.json` in the bundle. +This is an inactive-policy control and does not establish that removing active carrying preserves performance. diff --git a/docs/uncertainty/domino-comparison-summary.md b/docs/uncertainty/domino-comparison-summary.md new file mode 100644 index 000000000..9d5b720b1 --- /dev/null +++ b/docs/uncertainty/domino-comparison-summary.md @@ -0,0 +1,50 @@ +# Combined Domino prediction assessment + +The [Stage B comparison](simplification-proposal.md) now has a validated report builder for the three matched Domino protocols. +It compares local proposals, mixed local/full-range proposals, and the explicit fixed-initial-state approximation on the same 64 fitted actions and 97 reserved actions. +Each protocol has two independent numerical replicas, seeds 100 and 101. +These are inference replicas on one recording, not agent solve-rate seeds. + +The report requires a completed forecast and its pinned completed source fit before accepting a result row. +It verifies the program, data, output model, numerical budget and proposal setting against the declared protocol. +It checks every positive-weight complete history, its physical sample and weight against the fit snapshot, then reconstructs the reported toppling curve from the saved histories. +Dropping a positive-weight particle, changing its physical sample or altering the aggregate event curve fails validation. +Missing or failed inputs remain explicit, and a pair is not compared until both members are available. + +The report includes position error, frame-averaged and per-object toppling Brier errors, final toppling probabilities, event timing mass and inference cost. +Stored truth enters this evaluator only; it does not select parameters, particles, proposals or predictions. +Frame averages are descriptive and are not treated as independent experimental samples. +The legacy point forecast remains a separate baseline, with its own initialization and fitting policy. + +## Predeclared screening checks + +| Difference between replicas | Maximum for the initial screen | +| --- | ---: | +| RMS difference across conditional mean xyz coordinates | 2.5 mm | +| Largest toppling probability difference across objects and reserved frames | 0.20 | +| Largest final-frame toppling probability difference | 0.15 | + +These thresholds were recorded in the mixed-proposal plan before its fits completed. +Passing this screen does not establish numerical adequacy, acceptable cost or safe use in planning. +Budget sensitivity, broader recordings and the remaining stages are still required. +Allocation seconds, allocated CPUs and allocated CPU-seconds are reported separately from target-evaluation counts and recorded forecast native actions. +Interrupted or missing accounting is not silently reported as zero total cost. + +## Validation and scheduled reports + +Compute validation `22674965` reproduces both earlier population diagnostics exactly, verifies 64 complete saved histories and rejects three deliberately corrupted inputs. +The earlier populations fail all three screening checks: their coordinate-mean RMS difference is 6.93 mm, largest toppling probability gap is 0.90625, and final probability gap is 0.64247. +This reproduces the previously known disagreement; it is not a new result for the current fits. +The first generated current summary is explicitly incomplete because all six forecast inputs were still missing. + +The frozen bundle is `logs/uncertainty_domino_comparison_summary_20260913`. +Three lightweight summary jobs are queued after their respective forecast pairs reach terminal states, with successful report-builder validation also required. +Each snapshot includes whichever other protocols have completed by then and marks the remainder incomplete. + +| Protocol triggering the snapshot | Summary job | +| --- | --- | +| Mixed proposals | `22675142` | +| Local proposals | `22675143` | +| Fixed initial state | `22675144` | + +This is a finite dependency chain for offline analysis, not a restored MB/MF notification monitor. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 7871d9933..0ffca6a2d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -27,6 +27,10 @@ Two forecast follow-ups are submitted with dependencies on successful completion The separate [carried-center comparison](carried-center-comparison.md) now has four paired legacy tasks queued as `22674821`, covering Fan and Domino with carrying off/on. Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. +The Fan pair has now completed: all four stages retain speed 0.0846, report an anchored verdict and produce identical predictions across arms, including the repeated-data stage. +No accepted center was carried, so this is an inactive-policy control; the Domino pair is running. +The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. +Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. ## Recent evidence From 8f0c003b915e55edeab9bdbbea6652ae63ab1ff8 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:09:06 -0400 Subject: [PATCH 13/94] Record completed mixed-proposal fits and unresolved concentration --- docs/uncertainty/implementation-progress.md | 3 +++ docs/uncertainty/proposal-refresh.md | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 0ffca6a2d..c9da2afad 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -31,6 +31,9 @@ The Fan pair has now completed: all four stages retain speed 0.0846, report an a No accepted center was carried, so this is an inactive-policy control; the Domino pair is running. The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. +Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. +Seed 101 places more than 97% empirical mass at a single value for each of four physical parameters, so its coincident central quantiles cannot establish precise identification. +Their reserved-action forecasts are queued, and matched local controls are starting; numerical adequacy remains unresolved. ## Recent evidence diff --git a/docs/uncertainty/proposal-refresh.md b/docs/uncertainty/proposal-refresh.md index 62537b3c2..33005f5be 100644 --- a/docs/uncertainty/proposal-refresh.md +++ b/docs/uncertainty/proposal-refresh.md @@ -81,4 +81,21 @@ Native validation `22672945` completed successfully with a 50/50 proposal mixtur All 32 fixed target evaluations and the subsequent 95-evaluation sampler result match exactly between synchronous and four-process execution. The sampler portion took 96.91 seconds synchronously and 25.40 seconds in parallel. This short run retains one ancestor and remains a kernel-equivalence check, not a numerically adequate physical posterior. -The validation dependency is satisfied, and comparison tasks 0 and 1 have started; the local-only controls await the two-task concurrency limit. +The validation dependency is satisfied, and comparison tasks 0 and 1 have now completed; the local-only controls are starting as resources become available. + +## Completed mixed-proposal fits + +Both numerical replicas finish all 32 temperature stages, but they remain unassessed. + +| Numerical seed | Target evaluations | Allocation seconds, four CPUs | Surviving initial ancestors | +| --- | ---: | ---: | ---: | +| 100 | 15,420 | 5,295 | 2 | +| 101 | 15,384 | 5,212 | 1 | + +Seed 101 assigns 97.65%-99.71% of its empirical mass to one retained value for each of lateral friction, rolling friction, spinning friction and mass. +For each of those four parameters, its 5th, 50th and 95th empirical percentiles therefore coincide. +This concentration must not be presented as evidence of precise physical identification. +Seed 100 has substantially less concentration and different uncertainty summaries for those parameters. +The mass diagnostics, source hashes and complete fit reports are retained in `completed-refresh-mass-diagnostic.json` and the source reports in the comparison bundle. +The broader moves have not yet established reliable joint inference; the reserved-action predictions and matched local controls remain necessary, followed by budget sensitivity. +The two forecast jobs are queued for the specified compute node. From 021c542fc6310f54a3145cbf60ffc3757139d0a5 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:21:41 -0400 Subject: [PATCH 14/94] Record mixed-proposal predictions and investigate local target scales --- docs/uncertainty/domino-comparison-summary.md | 25 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 11 +++++--- docs/uncertainty/proposal-refresh.md | 17 ++++++++++++- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/docs/uncertainty/domino-comparison-summary.md b/docs/uncertainty/domino-comparison-summary.md index 9d5b720b1..c8a0d6cba 100644 --- a/docs/uncertainty/domino-comparison-summary.md +++ b/docs/uncertainty/domino-comparison-summary.md @@ -48,3 +48,28 @@ Each snapshot includes whichever other protocols have completed by then and mark | Fixed initial state | `22675144` | This is a finite dependency chain for offline analysis, not a restored MB/MF notification monitor. + +## Completed mixed-proposal prediction pair + +Both mixed-proposal forecasts completed, and summary `22675142` verified all 128 positive-weight histories across the two replicas. +The local-proposal and fixed-initial-state comparisons remain incomplete in this snapshot. + +| Forecast | Position RMSE to noisy readings | Toppling Brier error | Final probability for domino 1 to topple | +| --- | ---: | ---: | ---: | +| Legacy point forecast | 11.062 mm | 0.0034364 | 0 | +| Mixed proposals, numerical seed 100 | 10.906 mm | 0.0001380 | 0.9375 | +| Mixed proposals, numerical seed 101 | 10.854 mm | 0.0000596 | 0.9539 | + +The recorded final outcome for domino 1 is toppled. +Position errors for the particle forecasts use their prefix-conditioned output means; the reports also retain native simulator mean errors separately. +These are descriptive results for one reserved action sequence, not agent performance or independent-dataset calibration. + +| Replica agreement check | Observed difference | Threshold | Result | +| --- | ---: | ---: | --- | +| RMS across mean xyz coordinates | 3.038 mm | 2.5 mm | Fails | +| Largest toppling probability gap | 0.18385 | 0.20 | Passes | +| Largest final toppling probability gap | 0.01645 | 0.15 | Passes | + +The pair still fails the complete predeclared screen. +The improved agreement relative to the earlier populations does not isolate the proposal change because the matched local controls have not completed. +The concentrated parameter values, budget sensitivity and substantial inference cost also remain unresolved. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index c9da2afad..1e9e3801c 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -24,7 +24,7 @@ These are offline inference replicas; completed prediction comparisons and numer The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. It retains every positive-weight particle even though all fixture future densities are zero. Two forecast follow-ups are submitted with dependencies on successful completion of their individual full prefix fits; no full-fit Fan prediction result is available yet. -The separate [carried-center comparison](carried-center-comparison.md) now has four paired legacy tasks queued as `22674821`, covering Fan and Domino with carrying off/on. +The separate [carried-center comparison](carried-center-comparison.md) submits four paired legacy tasks as `22674821`, covering Fan and Domino with carrying off/on. Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. The Fan pair has now completed: all four stages retain speed 0.0846, report an anchored verdict and produce identical predictions across arms, including the repeated-data stage. @@ -33,7 +33,10 @@ The [combined Domino assessment](domino-comparison-summary.md) reproduces prior Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. Seed 101 places more than 97% empirical mass at a single value for each of four physical parameters, so its coincident central quantiles cannot establish precise identification. -Their reserved-action forecasts are queued, and matched local controls are starting; numerical adequacy remains unresolved. +Matched local controls are starting; numerical adequacy remains unresolved. +Those two forecasts have now completed: both toppling-agreement checks pass, but the 3.038 mm difference between position means fails the 2.5 mm screen. +Their descriptive reserved-action errors are lower than the legacy point forecast on this recording; matched local controls, budget sensitivity and numerical adequacy remain unresolved. +A local conditional-scale audit is queued to investigate the concentrated parameter values before making another sampler change. ## Recent evidence @@ -64,13 +67,13 @@ The subsequent [conditional-parameter slices and mixed proposals](proposal-refre A mixed local/full-range block proposal now preserves the same fixed conditional target while allowing larger numerical moves. Forty-one functional tests, thirty-two disabled-refresh compatibility comparisons and final focused type/lint/format checks pass. The native mixed-proposal check also matches target values and sampler output exactly between synchronous and four-process execution. -The two refresh-arm Domino fits are running; two matched local-only controls await the array concurrency limit, and no new posterior is yet available. +Both mixed-proposal Domino fits and forecasts have completed; matched local-only controls are running, and numerical adequacy remains unresolved. The [initial-state ablation](initial-state-ablation.md) now supplies a first-observation-only point-start comparison while retaining the original parameter prior and output model. Its selected scene is feasible, repeats exactly and yields finite full-prefix likelihoods at 39 of 64 random parameter settings. Two parameter-only fits are running alongside the joint fits; this is an explicitly labeled approximation, not an exact state observation. The [checkpoint-driven forecast follow-up](checkpoint-forecasts.md) now reproduces every complete history and aggregate metric from the earlier 64-row forecast exactly. -Six follow-up jobs are submitted with dependencies on successful completion of their individual source fits; no new prediction comparisons are complete yet. +Six follow-up jobs are submitted with dependencies on successful completion of their individual source fits; the mixed-proposal forecast pair is complete, while the local and fixed-initial-state comparisons remain pending. The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. diff --git a/docs/uncertainty/proposal-refresh.md b/docs/uncertainty/proposal-refresh.md index 33005f5be..dc5e524fc 100644 --- a/docs/uncertainty/proposal-refresh.md +++ b/docs/uncertainty/proposal-refresh.md @@ -98,4 +98,19 @@ This concentration must not be presented as evidence of precise physical identif Seed 100 has substantially less concentration and different uncertainty summaries for those parameters. The mass diagnostics, source hashes and complete fit reports are retained in `completed-refresh-mass-diagnostic.json` and the source reports in the comparison bundle. The broader moves have not yet established reliable joint inference; the reserved-action predictions and matched local controls remain necessary, followed by budget sensitivity. -The two forecast jobs are queued for the specified compute node. +The two forecast jobs have completed and pass both toppling-agreement checks, while their 3.038 mm coordinate-mean difference misses the 2.5 mm screen. +See the [combined prediction assessment](domino-comparison-summary.md) for the matched reserved-action metrics and remaining limitations. + +## Local-scale diagnostic + +A separate audit now probes the final target near the first maximum-weight particle from each completed replica. +It varies one parameter coordinate at a time while keeping every initial-state coordinate fixed. +The signed displacements range from 0.0001 to 0.1 in the original unit proposal coordinates, including the current local scale 0.05. +The audit repeats each source point, checks its exact saved target factors and computes symmetric Metropolis acceptance probabilities from the target differences. +These conditional slices can diagnose inappropriate local step scales, but they are not marginal posterior widths or a test of global mode coverage. +No candidate from the audit replaces a fitted particle or changes a forecast. + +The frozen bundle is `logs/uncertainty_domino_local_scale_audit_20260913`. +The first attempt, `22675376`, failed before evaluating targets because its instrumentation expected a factory entry absent from the Domino worker. +The corrected audit instruments the actual candidate initializer and is queued as `22675469` on `mit_preemptable`, with four CPUs and a 20-minute limit on node1412. +Its results remain pending. From edd4328e929b8414834183a4a3ac6c085e148410 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:31:55 -0400 Subject: [PATCH 15/94] Record initial-state comparisons and launch inference budget checks --- docs/uncertainty/carried-center-comparison.md | 15 ++++- docs/uncertainty/domino-budget-sensitivity.md | 59 +++++++++++++++++++ docs/uncertainty/domino-comparison-summary.md | 9 +++ docs/uncertainty/implementation-progress.md | 12 ++-- docs/uncertainty/initial-state-ablation.md | 20 ++++++- docs/uncertainty/proposal-refresh.md | 13 +++- 6 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 docs/uncertainty/domino-budget-sensitivity.md diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index e6e7ebf9b..a8121966a 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -51,7 +51,7 @@ Array `22674821` is submitted to `mit_preemptable`, with six CPUs, 16 GB and a t The preceding array `22674643` failed its configuration guard before fitting because the guard compared runtime tuples with JSON lists. The serialized configurations were verified identical; the corrected guard normalizes representation before comparison. The failed reports and original driver are retained, and those setup outcomes are not agent failures. -The Fan pair has completed successfully; the Domino pair is running. +Both Fan and Domino pairs have completed successfully. ## Completed Fan control @@ -69,3 +69,16 @@ The repeated 96-action fits change neither the selected value nor the prediction Each arm uses 11,332 recorded native simulator actions across the four fits and their predictions. The paired comparison and source hashes are retained in `fan-comparison.json` in the bundle. This is an inactive-policy control and does not establish that removing active carrying preserves performance. + +## Completed Domino control + +Both Domino arms reproduce their archived cold fit exactly and produce identical selected values, diagnostic widths and complete prediction histories at all four stages. +No accepted center is carried into a later fit. +Selected lateral friction remains 0.674, restitution 0.02, rolling friction 0.006, spinning friction 0.5 and mass 0.1. +The repeated 96-action stage changes neither selected values, widths nor predictions in either arm. +Each arm records 59,252 native actions across the four fits and predictions. +The paired report and source hashes are retained in `domino-comparison.json` in the frozen bundle. + +Together, these two recordings cover only inactive carrying. +They do not establish whether removing accepted-center feedback changes fitting or repeated-data confidence when the policy is active. +That comparison requires additional development experience with an accepted fitted center, retaining this inactive result instead of replacing it. diff --git a/docs/uncertainty/domino-budget-sensitivity.md b/docs/uncertainty/domino-budget-sensitivity.md new file mode 100644 index 000000000..7020905c3 --- /dev/null +++ b/docs/uncertainty/domino-budget-sensitivity.md @@ -0,0 +1,59 @@ +# Domino population-size sensitivity + +September 13, 2026. +This is a Stage B numerical comparison under the [simplification proposal](simplification-proposal.md), using the same development recording and frozen simulator as the completed [64-particle forecasts](domino-comparison-summary.md). +These are offline inference replicas, not agent experiments or additional independent task datasets. + +## Why this comparison + +The 64-particle joint fits retain one or two initial ancestors and disagree on some decision-relevant predictions. +The point-start approximation retains five or six ancestors and passes the initial prediction-agreement screens, but that does not establish convergence or trustworthy parameter uncertainty. +The completed local-scale audit shows irregular target changes near retained scenes, with no uniformly better smaller proposal scale. +The next controlled numerical change increases population size instead of changing the probability model or selecting a new proposal heuristic. + +## Controlled protocol + +Run both the complete joint target and the separately labeled first-observation point-start approximation with 128 particles, using numerical seeds 100 and 101 for each. +Each run starts from its original declared prior/proposal, with no reuse of a completed 64-particle population as an initialization. +The only numerical configuration changes are 64 to 128 particles and a maximum evaluation budget of 16,448 to 32,896. +Retain 32 cubic-spaced temperatures, eight moves per stage, local scale 0.05, the 50/50 local/full-range mixture and the original proposal blocks. +The joint representation has 25 blocks; the point-start representation has five. +These are independent numerical replicas within each setting; using the same seed numbers across population sizes does not make their later random schedules identical. + +The data, physical program, parameter prior, initial-state policy, output model and native runtime remain fixed within each state treatment. +Worker, setup, preparation and driver files are copied byte-for-byte from their corresponding 64-particle bundles. +The new run directories contain no previous inference checkpoints. +Each new fit repeats its archived native preflight before sampling and then saves complete-stage checkpoints under the changed numerical configuration. +All four tasks have passed those preflights and started sampling on compute nodes. + +| State treatment | Fit tasks | Dependent forecast tasks | +| --- | --- | --- | +| Joint uncertain initial state | `22675729_0`, `22675729_1` | `22675741_0`, `22675742_1` | +| Fixed first-observation state | `22675730_0`, `22675730_1` | `22675743_2`, `22675744_3` | + +Each fit requests four CPUs, 20 GB and six hours on node1412 in `mit_preemptable`. +Each two-task array permits at most two concurrent replicas. +Forecasts retain the existing four-CPU, 20-GB, 30-minute allocation and begin only after their corresponding fit succeeds. +Runtime accounting must include actual allocation cost and repeated work after interruption, separately from the sampler's cumulative evaluation count. + +## Evaluation fixed before outcomes + +Use the original 64-action fitting prefix and 97-action causal suffix. +The forecast driver remains unchanged and restores the complete checkpoint, verifies the retained physical samples and prefix factors, then propagates every positive-weight particle through the full recorded history. +It repeats the first complete history and records all weights and explicit zero-weight rows. +No forecast or future observation alters the fitted population. + +For each state treatment, compare both 128-particle replicas with each other and all four 64-versus-128 replica pairs. +Retain the existing thresholds of 2.5 mm for RMS differences in conditional position means, 0.20 for the largest toppling-curve gap and 0.15 for the largest final toppling gap. +Report every pair rather than selecting the best agreement. +Also inspect parameter quantiles, mass at repeated values, ancestry, position errors, toppling errors, event timing and compute cost. +Do not pool the joint and fixed-state populations or treat them as estimates of the same target. + +These thresholds remain exploratory screens, not proof of calibrated uncertainty or completed Stage B acceptance. +A stable but poorly predictive result still needs its prediction failures reported. +A failure at the larger population size is evidence against relying on apparent stability at the smaller size. +Broader recordings, active carried-center coverage, the other domains and later planning gates remain necessary. +The production agent is unchanged. + +Frozen bundles and submission hashes are in `logs/uncertainty_domino_budget_joint_20260913`, `logs/uncertainty_domino_budget_point_20260913` and `logs/uncertainty_domino_budget_forecast_20260913`. +These finite inference and prediction jobs do not re-enable the MB/MF notification monitor. diff --git a/docs/uncertainty/domino-comparison-summary.md b/docs/uncertainty/domino-comparison-summary.md index c8a0d6cba..621528bdc 100644 --- a/docs/uncertainty/domino-comparison-summary.md +++ b/docs/uncertainty/domino-comparison-summary.md @@ -73,3 +73,12 @@ These are descriptive results for one reserved action sequence, not agent perfor The pair still fails the complete predeclared screen. The improved agreement relative to the earlier populations does not isolate the proposal change because the matched local controls have not completed. The concentrated parameter values, budget sensitivity and substantial inference cost also remain unresolved. + +## Completed fixed-initial-state pair + +Summary `22675144` additionally verifies the two complete point-start forecasts, including every positive-weight history. +Their 0.880 mm position-mean difference, 0.01215 maximum toppling-curve gap and 0.01215 final gap pass all three exploratory screens. +Their position RMSEs are 11.441 and 11.399 mm, and their toppling Brier errors are 0.0000028704 and 0.0000038637. +See the [initial-state ablation](initial-state-ablation.md) for its approximation, cost and source reports. +The local-only pair remains incomplete in this snapshot. +The next [budget comparison](domino-budget-sensitivity.md) tests each completed target at twice the particle count; no production use is approved by this initial screen. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 1e9e3801c..642792fca 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -28,7 +28,8 @@ The separate [carried-center comparison](carried-center-comparison.md) submits f Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. The Fan pair has now completed: all four stages retain speed 0.0846, report an anchored verdict and produce identical predictions across arms, including the repeated-data stage. -No accepted center was carried, so this is an inactive-policy control; the Domino pair is running. +No accepted center was carried, so this is an inactive-policy control. +The Domino pair has also completed with identical predictions and no active carrying; active-policy coverage remains open. The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. @@ -36,7 +37,10 @@ Seed 101 places more than 97% empirical mass at a single value for each of four Matched local controls are starting; numerical adequacy remains unresolved. Those two forecasts have now completed: both toppling-agreement checks pass, but the 3.038 mm difference between position means fails the 2.5 mm screen. Their descriptive reserved-action errors are lower than the legacy point forecast on this recording; matched local controls, budget sensitivity and numerical adequacy remain unresolved. -A local conditional-scale audit is queued to investigate the concentrated parameter values before making another sampler change. +The local conditional-scale audit has completed with exact saved-reference replay. +It finds rough target changes with no uniformly successful smaller step scale; no new scale heuristic is selected. +The point-start fits and forecasts have also completed, passing all three exploratory agreement screens while showing worse position error and better toppling error than legacy on this recording. +Four [population-size comparison fits](domino-budget-sensitivity.md) are running at 128 particles, two numerical seeds for each state treatment, with unchanged priors, likelihoods and proposal rules. ## Recent evidence @@ -71,9 +75,9 @@ Both mixed-proposal Domino fits and forecasts have completed; matched local-only The [initial-state ablation](initial-state-ablation.md) now supplies a first-observation-only point-start comparison while retaining the original parameter prior and output model. Its selected scene is feasible, repeats exactly and yields finite full-prefix likelihoods at 39 of 64 random parameter settings. -Two parameter-only fits are running alongside the joint fits; this is an explicitly labeled approximation, not an exact state observation. +Two parameter-only fits have completed; this is an explicitly labeled approximation, not an exact state observation. The [checkpoint-driven forecast follow-up](checkpoint-forecasts.md) now reproduces every complete history and aggregate metric from the earlier 64-row forecast exactly. -Six follow-up jobs are submitted with dependencies on successful completion of their individual source fits; the mixed-proposal forecast pair is complete, while the local and fixed-initial-state comparisons remain pending. +Six follow-up jobs are submitted with dependencies on successful completion of their individual source fits; the mixed-proposal and fixed-initial-state pairs are complete, while the local comparison remains pending. The latest [likelihood cost reduction](likelihood-cost.md) preserves all 2,560 archived orientation densities and five complete Fan likelihoods exactly on the checked runtimes. It removes array reductions from two-term quadrature sums, making the measured density evaluations about four times faster while retaining the statistical model and numerical acceptance checks. diff --git a/docs/uncertainty/initial-state-ablation.md b/docs/uncertainty/initial-state-ablation.md index b08b7628f..1e084bd47 100644 --- a/docs/uncertainty/initial-state-ablation.md +++ b/docs/uncertainty/initial-state-ablation.md @@ -42,7 +42,7 @@ Array `22673316` runs numerical seeds 100 and 101 under the point-state approxim Both use the tested 50/50 local/full-range proposal mixture, 64 particles, 32 cubic-spaced temperatures, eight moves, local scale 0.05 and a maximum of 16,448 evaluations. Each requests four CPUs, 20 GB and four hours on the same declared AMD worker node, with complete-stage checkpoints and at most two simultaneous tasks. The worker first reproduces the frozen preflight target and verifies the output-model identity. -Both tasks have started and passed their preflights. +Both tasks completed all 32 stages and passed their preflights. There are five active proposal coordinates, one for each physical parameter, rather than the joint arm's 95-coordinate representation. The complete output rows retain the constant initial-state coordinates for unambiguous replay. @@ -52,3 +52,21 @@ The ablation does not isolate the legacy carried-prior policy, whose sequential Frozen source, runtime identities, submission hashes, stage summaries and checkpoints are in `logs/uncertainty_domino_point_start_fit_20260913`. The production agent remains unchanged, and incomplete fits are not usable posterior results. + +## Completed point-start comparison + +Fits `22673316_0` and `_1` completed in 5,751 and 5,580 allocation seconds on four CPUs, using 15,927 and 15,952 target evaluations. +They retained five and six initial ancestors, respectively. +Forecasts `22673722_5` and `22673723_6` completed, and summary `22675144` verified all 128 positive-weight complete histories and unchanged source weights. + +| Numerical seed | Conditional position RMSE to noisy readings | Toppling Brier error | Final toppling probability for domino 1 | +| --- | ---: | ---: | ---: | +| 100 | 11.441 mm | 0.0000028704 | 0.97016 | +| 101 | 11.399 mm | 0.0000038637 | 0.95801 | + +The two replicas differ by 0.880 mm in their conditional position means and by at most 0.01215 in their toppling probabilities, including at the final frame. +All three predeclared exploratory agreement screens pass. +The point-state forecasts have greater position error than the legacy forecast (11.062 mm) and the two mixed joint forecasts (10.906 and 10.854 mm), while their toppling Brier errors are smaller on this reserved sequence. +Neither result establishes an agent advantage or adequate parameter uncertainty. +The comparison is one recording with different numerical difficulty between the state treatments. +A [population-size comparison](domino-budget-sensitivity.md) now tests both targets at 128 particles before interpreting agreement at 64 particles as stability. diff --git a/docs/uncertainty/proposal-refresh.md b/docs/uncertainty/proposal-refresh.md index dc5e524fc..098519b8d 100644 --- a/docs/uncertainty/proposal-refresh.md +++ b/docs/uncertainty/proposal-refresh.md @@ -112,5 +112,14 @@ No candidate from the audit replaces a fitted particle or changes a forecast. The frozen bundle is `logs/uncertainty_domino_local_scale_audit_20260913`. The first attempt, `22675376`, failed before evaluating targets because its instrumentation expected a factory entry absent from the Domino worker. -The corrected audit instruments the actual candidate initializer and is queued as `22675469` on `mit_preemptable`, with four CPUs and a 20-minute limit on node1412. -Its results remain pending. +The corrected audit instruments the actual candidate initializer and completed as `22675469` on `mit_preemptable`. +It used 10,048 native actions, 76.98 worker seconds and 82 allocation seconds on four CPUs. +Both complete checkpoints recover exactly, and the two repeats of each selected reference reproduce their saved joint values and both target factors exactly. +Seven out-of-box proposals remain explicit; the other 157 evaluations include the four reference repeats. + +Restitution has zero target difference at all 16 tested offsets for both selected scenes. +For seed 101, even the best tested lateral-friction and mass moves have acceptance probabilities only 0.00498 and 0.01855; rolling friction reaches only 0.000743. +Smaller changes do not uniformly improve acceptance: at seed 101, spinning friction shifted by -0.03 has acceptance 0.297, compared with 0.0311 at -0.0001. +These fixed-state slices show a rough conditional target and do not justify declaring one smaller proposal scale sufficient. +They neither establish global identifiability nor measure marginal widths. +The next experiment therefore checks [population-size sensitivity](domino-budget-sensitivity.md) under the unchanged proposal kernel, rather than choosing another scale from these two selected states. From 0bf3cc58041b477e7e77a4f0644cb3abbf95b268 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:37:14 -0400 Subject: [PATCH 16/94] Add earlier-program test of repeated-data prior carrying --- docs/uncertainty/carried-center-comparison.md | 30 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 33 insertions(+) diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index a8121966a..8b1b48090 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -82,3 +82,33 @@ The paired report and source hashes are retained in `domino-comparison.json` in Together, these two recordings cover only inactive carrying. They do not establish whether removing accepted-center feedback changes fitting or repeated-data confidence when the policy is active. That comparison requires additional development experience with an accepted fitted center, retaining this inactive result instead of replacing it. + +## Earlier-program repeated-data comparison + +A targeted inspection of the existing Domino seed-0 training log identifies an accepted fit followed by carrying: the first full 161-action fit moves lateral friction from its declared 0.3 starting value, and the subsequent fit carries 0.674. +The saved `cycle_000_vers_001_simulator.py` has declared friction 0.3, whereas the latest `cycle_000_vers_002_simulator.py` used above already declares 0.674. +An AST comparison confirms that the only executable difference between those two saved artifacts is this declared initial value; documentation strings also differ. +The saved files are used unchanged, including their original generated declarations. +This selects a candidate active-policy case from training experience, not from test-level outcomes. +It does not guarantee that every historical fit detail will reproduce on the controlled runtime. + +The earlier-program comparison uses the same complete 161-action training recording three times, starting both arms with empty carried history. +Both arms retain the fixed earlier program, same data, parameter bounds, optimizer configuration, preparation, ordinary held-value policy and prediction path. +Only accepted-center carrying is toggled. +The first full-data fits must match exactly across arms in entering anchors, fitted and selected values, diagnostic reports and complete predictions before a difference in later fits can be attributed to carrying. +All three stages must have identical data identities. +There are no held-out actions in this repeated-data diagnostic, so its predictions are in-sample replay and cannot establish predictive improvement. +Legacy widths remain diagnostic quantities, not newly validated credible intervals. + +| Arm | Compute task | Initial declared friction | Fitting actions by stage | +| --- | --- | ---: | --- | +| Fixed registry centers | `22675807_0` | 0.3 | 161, 161, 161 | +| Carry accepted centers | `22675807_1` | 0.3 | 161, 161, 161 | + +The array requests six CPUs, 16 GB and two hours per task on node1412 in `mit_preemptable`. +A finite report job, `22675822`, depends on both fits completing successfully. +The paired validator passes a repetition fixture derived from the previous inactive reports and rejects altered first-fit values, an altered first-fit verdict and changed repeated data. +That fixture validates report guards only; it is not another physical fit or evidence of active carrying. +The run, source-selection rationale, AST comparison, validation report and submission hashes are frozen in `logs/uncertainty_carried_prior_active_20260913`. +Actual accepted-center coverage and any effects remain pending these runs. +The earlier inactive controls are retained separately, and the production estimator is unchanged. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 642792fca..986a6784a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -30,6 +30,9 @@ It explicitly records whether carrying is ever active, rather than treating unch The Fan pair has now completed: all four stages retain speed 0.0846, report an anchored verdict and produce identical predictions across arms, including the repeated-data stage. No accepted center was carried, so this is an inactive-policy control. The Domino pair has also completed with identical predictions and no active carrying; active-policy coverage remains open. +A separate repeated-data pair now uses the earlier saved Domino program, whose declared friction is 0.3 instead of the already learned 0.674. +Historical training logs identify an accepted fit and later carrying at that earlier starting point; the new controlled reproduction has three identical 161-action fits and requires exact first-fit agreement between arms. +Tasks `22675807_0` and `_1` and their dependent comparison `22675822` are submitted; active-policy results remain pending. The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. From c8d663a36daf3200ff7678270361adb4ff18c2bc Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:43:03 -0400 Subject: [PATCH 17/94] Validate complete replica comparisons across inference budgets --- docs/uncertainty/domino-budget-sensitivity.md | 30 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 33 insertions(+) diff --git a/docs/uncertainty/domino-budget-sensitivity.md b/docs/uncertainty/domino-budget-sensitivity.md index 7020905c3..1bcf5b102 100644 --- a/docs/uncertainty/domino-budget-sensitivity.md +++ b/docs/uncertainty/domino-budget-sensitivity.md @@ -57,3 +57,33 @@ The production agent is unchanged. Frozen bundles and submission hashes are in `logs/uncertainty_domino_budget_joint_20260913`, `logs/uncertainty_domino_budget_point_20260913` and `logs/uncertainty_domino_budget_forecast_20260913`. These finite inference and prediction jobs do not re-enable the MB/MF notification monitor. + +## Report validation and follow-up + +The comparison report is prepared in `logs/uncertainty_domino_budget_summary_20260913`. +It enumerates all six pairs among the two 64-particle and two 128-particle replicas for each state treatment, producing twelve pairs total, including eight cross-budget pairs. +Missing or unsuccessful forecasts remain explicit and cannot produce a completed pair or a passed screen. +The paired checks require the same complete inference identity and prior, allowing only the declared particle-count and evaluation-budget differences in sampler configuration. +Joint-state and point-state targets are never compared as estimates of the same distribution. + +Each completed row verifies its source fit, numerical configuration, numerical seed, complete checkpoint provenance, unchanged weights and all saved positive-weight histories using the previously validated history checker. +Its parameter report independently reconstructs the saved empirical 5th, 50th and 95th percentiles and records the largest mass at any exact retained value. +This prevents coincident quantiles from silently being described as precise parameter identification. +Cost fields retain scheduler accounting, latest-attempt fitting seconds and forecast native actions separately, with interrupted-work limitations explicit. + +Compute job `22675912` tests reproduction of the four completed 64-particle forecasts and both existing replica screens before the new results arrive. +It also exercises rejection of mismatched priors, runtimes, proposal settings, state treatments, actual fit data identities and a 64-particle population merely relabeled as 128 particles. +These are analysis checks, not new physical fits or agent results. + +The validation completed successfully in job `22675912`, using one CPU and nine allocation seconds with no native simulation. +All four reference forecasts, both existing replica comparisons and 256 complete saved histories reproduce exactly; all six deliberately incomparable inputs are rejected. +The initial summary is explicitly incomplete, with four completed rows and two completed pairs; ten pairs still require the larger fits and forecasts. +Finite snapshot jobs `22675954` and `22675955` depend on validated reporting and termination of the joint and point-state forecast pairs, respectively. +Each snapshot reports every available result while retaining missing or failed inputs explicitly. +These analysis dependencies do not create result notifications in this task. + +The baseline parameter diagnostics also show why prediction agreement alone is insufficient. +For the point-start replicas, median lateral friction is 0.2734 versus 0.4649 and median rolling friction is 0.003994 versus 0.001571, despite passing the reserved-sequence prediction screens. +Their largest exact-value masses reach 33.84% and 56.02%, respectively, across the five reported parameters. +This does not prove that the parameter distributions are wrong, but it prevents interpreting agreement on one action sequence as established uncertainty over other plans. +The comparison retains all empirical quantiles and concentration measures for the larger-population check. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 986a6784a..e4838d656 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -44,6 +44,9 @@ The local conditional-scale audit has completed with exact saved-reference repla It finds rough target changes with no uniformly successful smaller step scale; no new scale heuristic is selected. The point-start fits and forecasts have also completed, passing all three exploratory agreement screens while showing worse position error and better toppling error than legacy on this recording. Four [population-size comparison fits](domino-budget-sensitivity.md) are running at 128 particles, two numerical seeds for each state treatment, with unchanged priors, likelihoods and proposal rules. +The population-size report passes compute validation `22675912`, reproducing all four completed forecasts and 256 histories while rejecting six incomparable-input cases. +It records all twelve within-target replica pairs, including eight cross-budget pairs, and independently checks empirical parameter quantiles and repeated-value mass. +Its first snapshot retains ten incomplete pairs; finite follow-up summaries `22675954` and `22675955` depend on the larger forecasts. ## Recent evidence From e088302bb54c8191afd5b5cad8de189d95a5f6e7 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 07:51:27 -0400 Subject: [PATCH 18/94] Isolate prior carrying from mutable simulator reference defaults --- docs/uncertainty/carried-center-comparison.md | 44 ++++++++++++++++++- docs/uncertainty/implementation-progress.md | 5 ++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index 8b1b48090..5852d4405 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -110,5 +110,47 @@ A finite report job, `22675822`, depends on both fits completing successfully. The paired validator passes a repetition fixture derived from the previous inactive reports and rejects altered first-fit values, an altered first-fit verdict and changed repeated data. That fixture validates report guards only; it is not another physical fit or evidence of active carrying. The run, source-selection rationale, AST comparison, validation report and submission hashes are frozen in `logs/uncertainty_carried_prior_active_20260913`. -Actual accepted-center coverage and any effects remain pending these runs. +The initial fit now reproduces an accepted friction estimate, but the first comparison exposed an additional reference-lifetime effect described below. The earlier inactive controls are retained separately, and the production estimator is unchanged. + +## Mutable reference defaults and the corrected isolated comparison + +Task `22675807_0` completes its first fit and selects friction 0.6739569455671225 with a weakly identified verdict. +It then fails the entering-anchor guard before the second fit. +This is a diagnostic harness failure, not an agent failure or a completed repeated-data comparison. +The completed first-stage report is retained; the paired summary `22675822` was cancelled because its required off-arm success can no longer occur. +Task `22675807_1` retains the original reference-lifetime protocol and remains a separate diagnostic, not the on arm of the corrected pair. + +The failure is caused by a concrete behavior of the frozen subclass interface. +`PyBulletEnv._agent_param_info()` returns current `_agent_param_values` as the registry defaults. +Applying selected parameters to the reused reference changes those defaults, and `physical_param_anchors()` reads them on the next fit. +Explicit carrying can therefore be disabled while this reference-lifetime path still feeds fitted values into later prior centers. +The prior inactive controls did not expose the distinction because their selected values stayed at their initial defaults. + +Direct compute audit `22676108` reproduces this through the actual frozen subclass, `physical_param_anchors()` and `fit_prior_anchors()` methods. +With the carry flag off, the anchor changes from 0.3 to 0.6739569455671225 after applying the first fit to that reference. +A separate unfitted reference retains exactly the original anchors. +The audit uses zero native actions and completes in 26 allocation seconds on one CPU. +Its preceding attempt `22676089` failed during setup because a local list reused the factory's counter name; the original script and error remain preserved separately. + +The corrected comparison keeps the reference used to obtain registry centers at its original declared values. +Predictions still use fresh rollout worlds with explicitly selected parameters, and both arms retain their normal previously applied held values. +Only the carrying-enabled arm can replace a prior center through the actual accepted-center policy. +This isolates explicit carrying; it does not assert that simply turning off the historical flag reproduces fixed-prior fitting in every production path. +It also leaves the historical production registry behavior unchanged while the replacement remains under evaluation. + +| Corrected arm | Task | Reference policy | +| --- | --- | --- | +| Fixed centers | `22676127_0` | Unfitted reference, no carried centers | +| Explicit carrying | `22676127_1` | Unfitted reference, accepted centers overlaid by the existing carry method | + +Both tasks repeat the same 161-action recording three times with the earlier frozen program. +Each must reproduce the saved completed first fit exactly before advancing, including its data, anchors, fitted values, applied values and diagnostic report. +The paired report additionally requires exact first-fit agreement between arms and unchanged data across every repetition. +Its validator accepts the declared repetition fixture and rejects changed reference policies, failed first-fit reference checks and changed first-fit values. +These report checks are not physical inference results. + +Array `22676127` depends on successful native metadata validation and requests six CPUs, 16 GB and two hours per task on node1412 in `mit_preemptable`. +Finite report `22676128` depends on successful completion of both tasks. +The corrected frozen bundle is `logs/uncertainty_carried_prior_isolated_20260913`; source hashes, setup failures and the original-reference diagnostic remain separate. +The probability model for the new inference method already uses an explicit immutable prior identity; this experiment checks the legacy feedback mechanisms it is meant to replace. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index e4838d656..047710ce8 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -32,7 +32,10 @@ No accepted center was carried, so this is an inactive-policy control. The Domino pair has also completed with identical predictions and no active carrying; active-policy coverage remains open. A separate repeated-data pair now uses the earlier saved Domino program, whose declared friction is 0.3 instead of the already learned 0.674. Historical training logs identify an accepted fit and later carrying at that earlier starting point; the new controlled reproduction has three identical 161-action fits and requires exact first-fit agreement between arms. -Tasks `22675807_0` and `_1` and their dependent comparison `22675822` are submitted; active-policy results remain pending. +The first off-arm fit selects friction 0.6739569, then its second-fit guard detects that applying fitted values to the reused subclass reference also changes registry defaults. +Native audit `22676108` confirms that this can shift prior centers with explicit carrying off. +The corrected isolated pair `22676127` keeps an unfitted registry reference and uses fresh selected-parameter worlds for predictions; both arms must reproduce that first fit exactly before repeated-data comparisons. +Its paired summary `22676128` is queued; the interrupted original comparison remains separate and is not an agent outcome. The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. From b66ba12791508b2a2dc80740651afd9221cf0863 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:01:24 -0400 Subject: [PATCH 19/94] Validate paired Fan forecasts from complete weighted histories --- docs/uncertainty/fan-prefix-comparison.md | 21 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 2 ++ 2 files changed, 23 insertions(+) diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index df0ea21c2..6c253da50 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -116,3 +116,24 @@ The two full-fit forecast jobs are submitted with dependencies on this successfu | --- | --- | --- | | 302 | `22674242_0` | `22674579_1` | | 303 | `22674242_1` | `22674580_2` | + +## Paired forecast report + +The report in `logs/uncertainty_fan_prefix_summary_20260913` verifies the two full prefix-fit forecasts before comparing them. +It checks their frozen script and plan hashes, complete source fits, identical prior and inference identities, numerical configurations and expected numerical seeds. +It reconstructs native ball-position means, fan/switch/target event probabilities, geometric goal probabilities and cumulative goal probabilities from every positive-weight complete history. +Original samples, weights, first-goal timing and zero-weight exclusions must match the stored artifacts exactly. +Future mixture density is reconstructed from the original weighted per-history scores, retaining zero-density predictions explicitly. + +The two rows report position errors, event and goal Brier errors, parameter quantiles, ancestry, target evaluations, native actions and allocation/attempt costs alongside the matched legacy prediction. +The pair reports differences between conditional and native position means, event curves, geometric goal curves and probability of reaching the goal within the suffix. +These are descriptive comparisons on one recording, not a new acceptance threshold, calibrated posterior claim or agent solve-rate comparison. +The learned target-hit readout and native geometric goal remain separate predictions. +The real environment's recorded target-hit feature uses that geometric goal condition; the source implementation is an axis-wise distance check with tolerance `pos_gap / 2`. + +Compute validation `22676293` completed with one CPU and 22 allocation seconds, performing no native simulation. +It verifies all 64 complete fixture histories and exactly reconstructs positions, events, geometric goals and timing while retaining all 64 zero-future-density particles. +It rejects dropped particles, altered weights, changed native means, changed event probabilities, changed goal probabilities and an altered zero-density mixture. +The initial summary is explicitly incomplete because the full inference forecasts are still pending. +Finite summary `22676382` depends on successful validation and termination of both full forecast jobs. +This does not re-enable the MB/MF notification monitor. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 047710ce8..cf380cb37 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -24,6 +24,8 @@ These are offline inference replicas; completed prediction comparisons and numer The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. It retains every positive-weight particle even though all fixture future densities are zero. Two forecast follow-ups are submitted with dependencies on successful completion of their individual full prefix fits; no full-fit Fan prediction result is available yet. +The paired Fan report now passes compute validation `22676293`, reconstructing all 64 fixture histories and rejecting six corrupted-output cases, including removal of zero future density. +Finite summary `22676382` will compare full-fit position, event and geometric-goal predictions after both forecasts terminate; it does not automatically approve a posterior. The separate [carried-center comparison](carried-center-comparison.md) submits four paired legacy tasks as `22674821`, covering Fan and Domino with carrying off/on. Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. From 65de6827c7dc400dd5c4785fa6b0d52559e110b0 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:16:58 -0400 Subject: [PATCH 20/94] Record Fan prefix outcomes and queue population-size comparison --- docs/uncertainty/carried-center-comparison.md | 21 ++++++++ docs/uncertainty/fan-prefix-comparison.md | 49 ++++++++++++++++++- docs/uncertainty/implementation-progress.md | 7 ++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index 5852d4405..17bbef6f2 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -154,3 +154,24 @@ Array `22676127` depends on successful native metadata validation and requests s Finite report `22676128` depends on successful completion of both tasks. The corrected frozen bundle is `logs/uncertainty_carried_prior_isolated_20260913`; source hashes, setup failures and the original-reference diagnostic remain separate. The probability model for the new inference method already uses an explicit immutable prior identity; this experiment checks the legacy feedback mechanisms it is meant to replace. + +## Completed original reused-reference carrying arm + +The original carrying task `22675807_1` completed all three identical-data fits in 1,211 allocation seconds and 1,187.65 worker seconds, recording 100,803 native simulator actions. +These are simulation actions used by fitting and replay, not new environment interactions or agent solve-rate results. +All selected parameter values and complete prediction histories are exactly unchanged across the three stages. +Its initial fit matches the completed first stage of the interrupted off arm exactly. + +| Repeated-data fit | Entering friction anchor | Selected friction | Legacy width | Verdict | +| --- | ---: | ---: | ---: | --- | +| First | 0.3 | 0.6739569 | 0.3449425 | Weakly identified | +| Second | 0.6739569 | 0.6739569 | 0.4926973 | Anchored | +| Third | 0.6739569 | 0.6739569 | 0.4926973 | Anchored | + +The width changes after refitting identical evidence and then remains stable; this case does not show progressive narrowing. +It does not isolate explicit carrying from mutable reference defaults and cannot establish a change in decisions or performance. +The verification and source hashes are retained in `reused-reference-diagnostic.json` in the original active-comparison bundle. +The corrected isolated pair remains necessary. + +Both corrected tasks have now completed their first fit and passed the exact saved-reference check, with entering friction anchor 0.3 and width 0.3449425. +Their repeated-data stages are running; the isolated paired outcome is not yet complete. diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index 6c253da50..babc93a50 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -79,7 +79,7 @@ The preceding attempt `22674140` failed in report construction because checkpoin The corrected attempt changes only summary decoding and adds the initial weight effective sample size. The launch manifest pins the successful preflight, target scripts and configuration before any fit starts. Array `22674242` submits task 0 for numerical seed 302 and task 1 for numerical seed 303, each with four CPUs, 20 GB and a four-hour allocation limit on `mit_preemptable`. -Neither has a completed posterior or prediction comparison yet. +Both fits and their reserved-suffix forecasts have now completed; numerical adequacy remains unestablished, as detailed below. ## Reserved-suffix forecast validation @@ -137,3 +137,50 @@ It rejects dropped particles, altered weights, changed native means, changed eve The initial summary is explicitly incomplete because the full inference forecasts are still pending. Finite summary `22676382` depends on successful validation and termination of both full forecast jobs. This does not re-enable the MB/MF notification monitor. + +## Completed 64-particle prefix comparison + +Both fits complete all 32 stages: seed 302 uses 15,466 target evaluations and retains one initial ancestor; seed 303 uses 15,609 evaluations and retains two. +They take 5,521 and 5,568 allocation seconds on four CPUs, with 989,824 and 998,976 completed native fitting actions. +These are numerical replicas on the same development recording, not independent agent seeds. +The two forecasts complete in 73 and 79 allocation seconds and retain every positive-weight history. + +| Forecast | Conditional ball-position RMSE to noisy readings | Native ball-position RMSE | Native goal Brier error | Probability of reaching geometric goal in suffix | +| --- | ---: | ---: | ---: | ---: | +| Legacy point forecast | 7.072 mm | 7.072 mm | 0.044118 | 1.0 | +| Numerical seed 302 | 6.311 mm | 6.442 mm | 0.042419 | 0.78125 | +| Numerical seed 303 | 6.130 mm | 6.172 mm | 0.020787 | 0.90209 | + +The recorded goal first occurs at action 116, within the reserved suffix. +The new replicas have lower position error on this recording, but their conditional mean positions differ by 2.866 mm and their native geometric goal curves differ by as much as 0.33212. +Their probability of ever reaching the goal within the suffix differs by 0.12084. +Only one positive-weight particle in each population assigns nonzero density to the complete observed future; all other particles still contribute to the position, event and goal forecasts at their unchanged weights. +Their similar mixture log densities, 18,398.66 and 18,398.31, therefore do not establish a stable density estimate. +No domain-specific numerical acceptance threshold is introduced after these outcomes. + +The fan-speed empirical 5th, 50th and 95th percentiles are `[0.06622, 0.08353, 0.91372]` and `[0.07732, 0.10989, 0.93241]`. +Their largest masses at an exact retained speed value are 0.328125 and 0.283687. +These empirical distributions, ancestry and decision-relevant prediction differences require further numerical assessment. +The cost is also substantial: the matched legacy fit and replay used 30.88 worker seconds and 1,777 native actions. +This comparison does not demonstrate an agent efficiency or solve-rate improvement. + +The first completed summary `22676382` preserved the prediction metrics but omitted parameter quantiles because it read a generic field instead of `fan_speed_quantiles`. +The corrected `compare-v2.py` independently reconstructs those weighted quantiles and checks the saved values. +Validation `22676681` reproduces the history checks, validates both completed parameter summaries and rejects altered quantiles; it completes in ten allocation seconds with zero native simulation. +The corrected complete report is `summary-22676681.json` in the paired-report bundle. +The first report and reader remain preserved as the reporting defect's reproduction. + +## Larger-population comparison + +The next numerical comparison doubles particles from 64 to 128 and the evaluation limit from 16,448 to 32,896 while preserving the program, observations, original prior, output model, proposal kernel, temperature schedule and moves. +It uses the same two numerical seeds, 302 and 303, without reusing a completed population as initialization. +The frozen bundle is `logs/uncertainty_fan_prefix_budget_20260913`. +Native preflight `22676726` must reproduce fixed target factors and match the entire 128-particle initial population between serial and four-process execution. +Its target identity and prior must exactly equal the validated 64-particle protocol, allowing only the two declared numerical configuration changes. +The preflight action count scales with the actual population size. + +Finite gate `22676770` validates that evidence and pins the run inputs before array `22676775` can start. +The two full fits request four CPUs, 20 GB and six hours each on node1412 in `mit_preemptable`, with complete-stage checkpoints and at most two simultaneous tasks. +Their same-suffix forecasts will be prepared from the validated larger configuration. +Compare both larger replicas and all four cross-budget replica pairs, preserving zero-density outcomes and actual computation costs. +These jobs do not change the production estimator or restore result notifications. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index cf380cb37..2e2d2699d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -23,9 +23,11 @@ Two full-prior prefix fits are submitted as `22674242_0` and `_1`, with numerica These are offline inference replicas; completed prediction comparisons and numerical assessment remain pending. The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. It retains every positive-weight particle even though all fixture future densities are zero. -Two forecast follow-ups are submitted with dependencies on successful completion of their individual full prefix fits; no full-fit Fan prediction result is available yet. +Both full Fan prefix fits and their forecast follow-ups have completed; the two populations retain one and two initial ancestors. The paired Fan report now passes compute validation `22676293`, reconstructing all 64 fixture histories and rejecting six corrupted-output cases, including removal of zero future density. -Finite summary `22676382` will compare full-fit position, event and geometric-goal predictions after both forecasts terminate; it does not automatically approve a posterior. +The completed paired report gives lower position error than legacy on this recording, but a maximum 0.33212 geometric-goal probability disagreement and only one nonzero full-future-density particle per replica. +Corrected summary `22676681` also reconstructs and verifies the two parameter-quantile reports; the first summary omitted their Fan-specific field. +A 128-particle comparison is queued as array `22676775`, gated by native preflight `22676726` and validation finalizer `22676770`; no posterior is approved by the current evidence. The separate [carried-center comparison](carried-center-comparison.md) submits four paired legacy tasks as `22674821`, covering Fan and Domino with carrying off/on. Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. @@ -38,6 +40,7 @@ The first off-arm fit selects friction 0.6739569, then its second-fit guard dete Native audit `22676108` confirms that this can shift prior centers with explicit carrying off. The corrected isolated pair `22676127` keeps an unfitted registry reference and uses fresh selected-parameter worlds for predictions; both arms must reproduce that first fit exactly before repeated-data comparisons. Its paired summary `22676128` is queued; the interrupted original comparison remains separate and is not an agent outcome. +The original reused-reference carrying arm has completed: selected values and predictions stay identical, while friction width grows from 0.34494 to 0.49270 on the first repeated fit, then stays stable. The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. From aa86877a0538f9eae8c763ae4b19344a40fab908 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:28:35 -0400 Subject: [PATCH 21/94] Validate Fan cross-budget reports and queue reserved forecasts --- docs/uncertainty/carried-center-comparison.md | 5 ++- docs/uncertainty/fan-prefix-comparison.md | 36 ++++++++++++++++++- docs/uncertainty/implementation-progress.md | 3 ++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index 17bbef6f2..3ee47e69c 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -174,4 +174,7 @@ The verification and source hashes are retained in `reused-reference-diagnostic. The corrected isolated pair remains necessary. Both corrected tasks have now completed their first fit and passed the exact saved-reference check, with entering friction anchor 0.3 and width 0.3449425. -Their repeated-data stages are running; the isolated paired outcome is not yet complete. +Both corrected tasks have also completed their second fit: the off arm retains the declared friction anchor 0.3, while the on arm enters with the accepted 0.6739569. +The isolated carrying arm `22676127_1` has completed all three repetitions in 1,141 allocation seconds and 1,125.42 worker seconds with 100,803 native actions. +Its selected parameters and complete predictions remain exactly unchanged; friction width changes from 0.3449425 to 0.4926973 and then remains stable. +The fixed-center arm is still running, so the isolated paired outcome is not yet complete. diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index babc93a50..9f9b9382e 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -181,6 +181,40 @@ The preflight action count scales with the actual population size. Finite gate `22676770` validates that evidence and pins the run inputs before array `22676775` can start. The two full fits request four CPUs, 20 GB and six hours each on node1412 in `mit_preemptable`, with complete-stage checkpoints and at most two simultaneous tasks. -Their same-suffix forecasts will be prepared from the validated larger configuration. +Their same-suffix forecasts are prepared as dependent jobs under the validated larger configuration, as detailed below. Compare both larger replicas and all four cross-budget replica pairs, preserving zero-density outcomes and actual computation costs. These jobs do not change the production estimator or restore result notifications. + +## Cross-budget forecast pipeline + +The frozen forecast bundle is `logs/uncertainty_fan_budget_forecast_20260913`. +Its simulator mapping, model, prediction worker, checkpoint recovery driver and runtime preparation are byte-for-byte copies of the validated 64-particle forecast implementation. +The forecast plan is finalized only after the successful larger-fit preflight is available, with exact checks on the original target identity, prior and permitted numerical differences. +Contract validation rejects changed identities, priors, proposal configurations, incomplete native equality checks and a mismatched initial population size. +This is a manifest validation, not a new native inference result. + +Finite preparation job `22677008` depends on successful fit gate `22676770`. +Forecast `22677013_0` depends on that preparation and fit `22676775_0`; forecast `22677014_1` depends on that preparation and fit `22676775_1`. +Each forecast requests four CPUs, 20 GB and 30 minutes on node1412 in `mit_preemptable`. +Every positive-weight particle retains its original weight, complete 133-frame history and future-density outcome. +The 68 forecast actions remain excluded from scene construction and parameter fitting. + +The budget report in `logs/uncertainty_fan_budget_summary_20260913` compares all four populations and all six pairs: one within each budget and four across budgets. +It imports the checksum-pinned complete-history verifier used by the corrected 64-particle report. +Each row must match its own validated configuration, expected numerical seed and source artifact hashes; pairs must have identical target identities and priors, allowing only particle count and the proportional evaluation limit to differ. +It reports the original weighted position, event and goal predictions, zero future densities, parameter quantiles, retained-value concentration, ancestry and computation costs. +No acceptance threshold or posterior publication is introduced by this report. + +Compute validation `22677052` completed on one CPU in eight allocation seconds with zero native simulation. +It exactly reproduced both completed baseline rows and their pair comparison from 128 saved histories. +It rejected seven mismatches: target identity, prior, proposal, mislabeled particle count, dropped history, altered weight and discarded zero-density outcome. +The first summary correctly contains two completed rows, one completed pair and five incomplete pairs. +Finite summary `22677060` depends on successful report validation and termination of both larger forecasts. +These finite analysis jobs do not send notifications or re-enable the disabled MB/MF monitor. + +The larger native preflight `22676726` has now completed in 268 allocation seconds, using four CPUs and 17,920 native actions. +All retained target factors and the complete 128-particle initial state match exactly across serial and four-process execution. +Independent comparison confirms the original prior and target identity are unchanged and only the two declared numerical configuration fields differ. +The initial population contains 24 finite particles with weight effective sample size 20.19; these are initialization diagnostics, not a completed posterior. +Serial initialization takes 170.43 seconds and parallel initialization 44.72 seconds, excluding the separate fixed-row comparisons and startup. +The validated preflight checksum is `1d90e36965fff29d925e87fe93717f50586720cc598175836c80593c31090b11`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2e2d2699d..4a80017a5 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -28,6 +28,9 @@ The paired Fan report now passes compute validation `22676293`, reconstructing a The completed paired report gives lower position error than legacy on this recording, but a maximum 0.33212 geometric-goal probability disagreement and only one nonzero full-future-density particle per replica. Corrected summary `22676681` also reconstructs and verifies the two parameter-quantile reports; the first summary omitted their Fan-specific field. A 128-particle comparison is queued as array `22676775`, gated by native preflight `22676726` and validation finalizer `22676770`; no posterior is approved by the current evidence. +Its same-suffix forecasts `22677013_0` and `22677014_1` are now queued behind their individual fits and a checked plan-preparation gate. +The cross-budget reader passes compute validation `22677052`, reproducing 128 complete baseline histories and rejecting seven incomparable or corrupted inputs; summary `22677060` will retain all six replica pairs and their incomplete states. +The larger native preflight has now completed with exact target and 128-particle serial/parallel equality, 24 finite initial particles and weight effective sample size 20.19; its prior and target identity match the 64-particle reference. The separate [carried-center comparison](carried-center-comparison.md) submits four paired legacy tasks as `22674821`, covering Fan and Domino with carrying off/on. Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. From b05f3f563d4bc67d69604d008a1f951ff7f768b8 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:34:52 -0400 Subject: [PATCH 22/94] Record matched proposal and active prior-carry outcomes --- docs/uncertainty/carried-center-comparison.md | 34 +++++++++ docs/uncertainty/domino-comparison-summary.md | 34 +++++++++ docs/uncertainty/implementation-progress.md | 73 ++++++------------- 3 files changed, 92 insertions(+), 49 deletions(-) diff --git a/docs/uncertainty/carried-center-comparison.md b/docs/uncertainty/carried-center-comparison.md index 3ee47e69c..4d44830b3 100644 --- a/docs/uncertainty/carried-center-comparison.md +++ b/docs/uncertainty/carried-center-comparison.md @@ -178,3 +178,37 @@ Both corrected tasks have also completed their second fit: the off arm retains t The isolated carrying arm `22676127_1` has completed all three repetitions in 1,141 allocation seconds and 1,125.42 worker seconds with 100,803 native actions. Its selected parameters and complete predictions remain exactly unchanged; friction width changes from 0.3449425 to 0.4926973 and then remains stable. The fixed-center arm is still running, so the isolated paired outcome is not yet complete. + +## Completed isolated repeated-data comparison + +Both corrected arms and paired report `22676128` have completed. +The fixed-center arm uses 1,598 allocation seconds on six CPUs, 1,579.50 worker seconds and 150,435 native actions. +The carrying arm uses 1,141 allocation seconds on six CPUs, 1,125.42 worker seconds and 100,803 native actions. +Each arm repeats exactly the same 161-action recording three times; these costs belong to fitting and replay, not additional agent interactions. + +The source reports have identical runtime controls, program, data, parameter declarations, configuration, reference policy and worker script. +Both first fits reproduce the saved reference exactly. +All six selected parameter sets and complete 161-step prediction histories are exactly equal. +With fixed centers, all three full diagnostic reports are also exactly equal. +With carrying, the reports change after the first fit and then remain exactly equal for the final two repetitions. + +| Diagnostic | Fixed centers, every repetition | Carrying, first repetition | Carrying, later repetitions | +| --- | ---: | ---: | ---: | +| Entering friction anchor | 0.3 | 0.3 | 0.6739569 | +| Selected friction | 0.6739569 | 0.6739569 | 0.6739569 | +| Friction legacy width | 0.3449425 | 0.3449425 | 0.4926973 | +| Restitution legacy width | 0.4125 | 0.4125 | 0.3375 | +| Mass legacy width | 2.4283955 | 2.4283955 | 1.5453426 | +| Friction verdict | Weakly identified | Weakly identified | Anchored | + +Rolling-friction and spinning-friction widths remain unchanged across the arms. +The reported widths change in both directions after carrying, without new observations or a change in the selected simulator. +This case does not demonstrate progressive confidence accumulation: both policies become stable on the repeated evidence. +It does establish that accepted-center feedback changes the legacy uncertainty report independently of new data, while fixed centers preserve the fitted dynamics in this example. +That supports the replacement's explicit original-prior contract and supplies the previously missing active-carry repeated-data coverage. +It does not establish preservation across accumulating datasets, unseen interactions, different domains or closed-loop decisions; interval changes can affect planning even when the selected point simulator is unchanged. +There are no reserved future actions in this diagnostic, and neither arm's widths are newly validated credible intervals. + +`completed-verification.json` independently pins both final source reports and the paired report, verifies the runtime match and all six prediction histories, and checks the full diagnostic-report equalities. +The original mutable-reference attempt and inactive-policy controls remain separate evidence. +The production fitter remains unchanged. diff --git a/docs/uncertainty/domino-comparison-summary.md b/docs/uncertainty/domino-comparison-summary.md index 621528bdc..986789da6 100644 --- a/docs/uncertainty/domino-comparison-summary.md +++ b/docs/uncertainty/domino-comparison-summary.md @@ -82,3 +82,37 @@ Their position RMSEs are 11.441 and 11.399 mm, and their toppling Brier errors a See the [initial-state ablation](initial-state-ablation.md) for its approximation, cost and source reports. The local-only pair remains incomplete in this snapshot. The next [budget comparison](domino-budget-sensitivity.md) tests each completed target at twice the particle count; no production use is approved by this initial screen. + +## Completed matched local-proposal controls + +Both local fits and their reserved-action forecasts have completed, so summary `22675143` now contains all six populations and all three within-protocol pairs. +The two local fits use 14,365 and 14,429 target evaluations and retain one initial ancestor each. +Their four-CPU allocation times are 5,075 and 4,279 seconds; forecasts require 86 and 70 allocation seconds and 10,465 native actions each. +The completed report verifies all 384 positive-weight complete histories across the six populations. + +For each numerical seed, an additional source check confirms that the local and mixed fits have identical complete inference identities, priors and seeds. +Their numerical configurations differ only in full-range block-refresh probability: zero for the local arm and 0.5 for the mixed arm. +This closes the earlier matched-control gap; the evidence comes from two numerical replicas of one target, not repeated datasets or agent seeds. + +| Forecast | Conditional position RMSE | Toppling Brier error | Final probability for domino 1 to topple | +| --- | ---: | ---: | ---: | +| Legacy point forecast | 11.062 mm | 0.0034364 | 0 | +| Local proposals, numerical seed 100 | 11.848 mm | 0.0032677 | 0.95746 | +| Local proposals, numerical seed 101 | 12.320 mm | 0.0189003 | 0 | +| Mixed proposals, numerical seed 100 | 10.906 mm | 0.0001380 | 0.93750 | +| Mixed proposals, numerical seed 101 | 10.854 mm | 0.0000596 | 0.95395 | + +The local seed-101 population predicts no final toppling for any of the six objects, while the recorded final state has four toppled objects. +It assigns much larger rolling friction than the other local replica; its empirical median is 0.06193 versus 0.00326. +This is an observed association, not an isolated causal diagnosis of the prediction error. + +| Replica agreement check | Local proposals | Mixed proposals | Screen limit | +| --- | ---: | ---: | ---: | +| RMS difference across conditional position means | 7.954 mm | 3.038 mm | 2.5 mm | +| Largest toppling probability gap | 1.0 | 0.18385 | 0.20 | +| Largest final-frame toppling probability gap | 1.0 | 0.01645 | 0.15 | + +The local pair fails all three predeclared exploratory checks; the mixed pair passes the two toppling checks but still fails the position-mean check. +The proposal change reduces disagreement and prediction errors on this matched recording, yet does not establish numerical adequacy or justify deployment. +The ongoing 128-particle mixed and fixed-initial-state comparisons remain the next budget-sensitivity evidence. +The matching checks and source summary hash are retained in `completed-local-matching-verification.json` in the summary bundle. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 4a80017a5..542a09a9e 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -8,56 +8,31 @@ The incumbent estimator remains the production default. Stage 0 interface preservation is complete, with scripted behavior parity checked. Stage A has implemented probability and replay components, but physical support and numerical validation remain incomplete. -The active work is Stage B offline comparison; Stage C live posterior use and Stage E retirement are not yet approved by the evidence. +The active work is Stage B offline comparison; Stage C live posterior use and Stage E retirement have not passed their evidence requirements. Stage D execution smoothing remains optional and deferred. -The full plan remains incomplete. - -The [Fan prefix comparison](fan-prefix-comparison.md) now has a completed compute support audit, `22674026`. -It removes the 68-action suffix before scene construction and scoring and does not reuse the full-recording proposal guide. -Eight of 48 sampled rest scenes have valid geometry and finite prefix likelihood, and the first supported trajectory repeats exactly. -All 16 median-scene probes instead have an initial penetration, ruling out that unmodified point-state baseline. -This prepares a comparison against the completed 64-action legacy fit while the existing full-recording fits continue unchanged. -The next prefix inference preflight, `22674192`, exactly matches 12 retained native targets and the complete initial population across serial and four-process execution. -It finds 11 supported initial particles with weight effective sample size 9.10 and verifies a roughly fourfold initialization speedup. -Two full-prior prefix fits are submitted as `22674242_0` and `_1`, with numerical seeds 302 and 303 and unchanged original support for moving and resting states. -These are offline inference replicas; completed prediction comparisons and numerical assessment remain pending. -The Fan reserved-suffix forecast path now passes complete-checkpoint recovery, per-particle prefix-factor equality and checks on all 64 saved complete histories using an explicitly unassessed short fixture. -It retains every positive-weight particle even though all fixture future densities are zero. -Both full Fan prefix fits and their forecast follow-ups have completed; the two populations retain one and two initial ancestors. -The paired Fan report now passes compute validation `22676293`, reconstructing all 64 fixture histories and rejecting six corrupted-output cases, including removal of zero future density. -The completed paired report gives lower position error than legacy on this recording, but a maximum 0.33212 geometric-goal probability disagreement and only one nonzero full-future-density particle per replica. -Corrected summary `22676681` also reconstructs and verifies the two parameter-quantile reports; the first summary omitted their Fan-specific field. -A 128-particle comparison is queued as array `22676775`, gated by native preflight `22676726` and validation finalizer `22676770`; no posterior is approved by the current evidence. -Its same-suffix forecasts `22677013_0` and `22677014_1` are now queued behind their individual fits and a checked plan-preparation gate. -The cross-budget reader passes compute validation `22677052`, reproducing 128 complete baseline histories and rejecting seven incomparable or corrupted inputs; summary `22677060` will retain all six replica pairs and their incomplete states. -The larger native preflight has now completed with exact target and 128-particle serial/parallel equality, 24 finite initial particles and weight effective sample size 20.19; its prior and target identity match the 64-particle reference. -The separate [carried-center comparison](carried-center-comparison.md) submits four paired legacy tasks as `22674821`, covering Fan and Domino with carrying off/on. -Its 64/96/96/full prefix schedule tests both accumulating experience and refitting identical evidence while holding the other legacy mechanisms fixed. -It explicitly records whether carrying is ever active, rather than treating unchanged results with empty carried history as evidence about removing active carrying. -The Fan pair has now completed: all four stages retain speed 0.0846, report an anchored verdict and produce identical predictions across arms, including the repeated-data stage. -No accepted center was carried, so this is an inactive-policy control. -The Domino pair has also completed with identical predictions and no active carrying; active-policy coverage remains open. -A separate repeated-data pair now uses the earlier saved Domino program, whose declared friction is 0.3 instead of the already learned 0.674. -Historical training logs identify an accepted fit and later carrying at that earlier starting point; the new controlled reproduction has three identical 161-action fits and requires exact first-fit agreement between arms. -The first off-arm fit selects friction 0.6739569, then its second-fit guard detects that applying fitted values to the reused subclass reference also changes registry defaults. -Native audit `22676108` confirms that this can shift prior centers with explicit carrying off. -The corrected isolated pair `22676127` keeps an unfitted registry reference and uses fresh selected-parameter worlds for predictions; both arms must reproduce that first fit exactly before repeated-data comparisons. -Its paired summary `22676128` is queued; the interrupted original comparison remains separate and is not an agent outcome. -The original reused-reference carrying arm has completed: selected values and predictions stay identical, while friction width grows from 0.34494 to 0.49270 on the first repeated fit, then stays stable. -The [combined Domino assessment](domino-comparison-summary.md) reproduces prior diagnostics, verifies complete weighted histories and rejects dropped particles, altered samples and altered event aggregates. -Three finite follow-up jobs will produce comparison snapshots as the individual forecast pairs terminate; the new posterior comparisons remain incomplete. -Both mixed-proposal Domino fits have now completed all 32 stages, retaining two and one initial ancestors respectively. -Seed 101 places more than 97% empirical mass at a single value for each of four physical parameters, so its coincident central quantiles cannot establish precise identification. -Matched local controls are starting; numerical adequacy remains unresolved. -Those two forecasts have now completed: both toppling-agreement checks pass, but the 3.038 mm difference between position means fails the 2.5 mm screen. -Their descriptive reserved-action errors are lower than the legacy point forecast on this recording; matched local controls, budget sensitivity and numerical adequacy remain unresolved. -The local conditional-scale audit has completed with exact saved-reference replay. -It finds rough target changes with no uniformly successful smaller step scale; no new scale heuristic is selected. -The point-start fits and forecasts have also completed, passing all three exploratory agreement screens while showing worse position error and better toppling error than legacy on this recording. -Four [population-size comparison fits](domino-budget-sensitivity.md) are running at 128 particles, two numerical seeds for each state treatment, with unchanged priors, likelihoods and proposal rules. -The population-size report passes compute validation `22675912`, reproducing all four completed forecasts and 256 histories while rejecting six incomparable-input cases. -It records all twelve within-target replica pairs, including eight cross-budget pairs, and independently checks empirical parameter quantiles and repeated-value mass. -Its first snapshot retains ten incomplete pairs; finite follow-up summaries `22675954` and `22675955` depend on the larger forecasts. +The full plan remains incomplete, and the incumbent estimator remains the production default. + +The [combined Domino assessment](domino-comparison-summary.md) now has all six completed 64-particle populations, their reserved-action forecasts and verified complete weighted histories. +Matched local-only proposals fail all three initial stability screens; mixed local/full-range proposals reduce disagreement and pass both toppling screens, but still fail the position-mean screen. +The fixed-initial-state approximation passes all three screens while showing worse position error and better toppling error than legacy on this recording. +These are numerical replicas on one development recording, not agent seeds or evidence of calibrated uncertainty across tasks. +Four [larger Domino fits](domino-budget-sensitivity.md) are running at 128 particles with unchanged target priors and proposal rules, with forecast and comparison follow-ups already queued. + +The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. +Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. +The larger native preflight `22676726` matches the original prior and target identity and verifies exact serial/parallel initialization at 128 particles. +Both larger fits `22676775_0` and `_1` are running, with same-suffix forecasts `22677013_0` and `22677014_1` queued behind their individual fits. +The cross-budget reader passes compute validation `22677052`, reproducing 128 complete baseline histories and rejecting seven incomparable or corrupted inputs; finite summary `22677060` will retain all six replica pairs. + +The [isolated carried-center comparison](carried-center-comparison.md) has completed both arms and report `22676128`. +All six fits on identical data select the same parameters and produce exactly identical complete predictions. +Fixed centers preserve every diagnostic report across all three repetitions; carrying changes friction, restitution and mass widths after the first fit, then remains stable. +This supplies active-carry repeated-data coverage and supports an immutable original prior, but does not establish unchanged future predictions or planning decisions. +The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. + +The existing long-prefix Balloons and full-recording Fan fits remain separate ongoing diagnostics. +The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. +No current physical posterior has been approved for the acting agent. ## Recent evidence From 64f9ad9a7fbbc6dcf4ee25c234d2856e883a11cd Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:44:46 -0400 Subject: [PATCH 23/94] Launch repeated-dataset posterior coverage reference --- docs/uncertainty/implementation-progress.md | 2 + .../uncertainty/repeated-dataset-reference.md | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 docs/uncertainty/repeated-dataset-reference.md diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 542a09a9e..81103c123 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -31,6 +31,8 @@ This supplies active-carry repeated-data coverage and supports an immutable orig The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. The existing long-prefix Balloons and full-recording Fan fits remain separate ongoing diagnostics. +The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) now has a validated exact-posterior preflight and a queued 512-fit coverage study across two cases and particle budgets. +It addresses statistical behavior across independent synthetic datasets while keeping physical-domain calibration and live-agent acceptance separate. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. diff --git a/docs/uncertainty/repeated-dataset-reference.md b/docs/uncertainty/repeated-dataset-reference.md new file mode 100644 index 000000000..959ac92ad --- /dev/null +++ b/docs/uncertainty/repeated-dataset-reference.md @@ -0,0 +1,66 @@ +# Repeated-dataset posterior reference + +This study addresses the repeated-dataset parameter-coverage requirement in the [simplification proposal](simplification-proposal.md). +Earlier reference studies checked repeated numerical runs on fixed datasets. +This study instead generates independent datasets from a declared prior and observation law, then compares the candidate sampler with the exact posterior for each dataset. +It is a statistical implementation reference, not evidence of physical-domain calibration or an agent performance result. + +## Fixed protocol + +The frozen bundle is `logs/uncertainty_repeated_dataset_reference_20260913`, using source commit `748ca519b` without an offline overlay. +There are 128 independently generated datasets for each of two cases. +The generating parameter, observation noise and numerical sampler use separate random streams. +The truth enters only evaluation; the likelihood receives the design matrix, observations and declared noise scale. +Both numerical budgets use the same datasets, original priors and likelihoods. + +| Case | Unknown coordinates | Observations | Exact reference | +| --- | --- | --- | --- | +| Stationary object | Constant position | Eight independent Gaussian readings, noise standard deviation 0.2 | Univariate Gaussian posterior | +| Linear motion with uncertain start | Initial position and velocity | Eight positions at times 0.9 through 1.1, noise standard deviation 0.2 | Correlated bivariate Gaussian posterior | + +Every unknown has an independent standard normal original prior. +The numerical proposal uses uniform coordinates transformed through the inverse Gaussian CDF, preserving the normalized prior with a unit base-density ratio. +No fitted posterior becomes a later prior. +The second case deliberately makes initial position and velocity hard to distinguish over the short observed time span. + +Each dataset is fitted at 256 and 2,048 particles, giving 512 planned fits. +The sampler uses 32 temperature stages, eight moves per stage, local proposal scale 0.05 and block-refresh probability 0.5. +The evaluation limit is the particle count multiplied by 257. +The study retains complete weighted populations, data identities, numerical outcomes, ancestry, target evaluations and elapsed times. +These particles are numerical integration states, not environment or agent seeds. + +## Reference and reporting checks + +For each coordinate, the report checks the posterior CDF at the generating truth, central 90% interval coverage, and errors against the exact posterior CDF. +The predeclared numerical screen requires mean absolute CDF error at most 0.025 and its 95th percentile at most 0.075. +Coverage is prior-predictive: its reference expectation averages over parameters drawn from the declared prior as well as over observation noise. +It is not a guarantee of 90% frequentist coverage for every fixed parameter value. + +Exact binomial intervals and diagnostics retain dataset counts. +A familywise diagnostic level of 0.01 is divided across the six candidate-coordinate comparisons and three distinct exact-reference comparisons. +Posterior-CDF uniformity p-values are descriptive; no separate automatic rejection rule is attached to them. +Passing these screens does not approve a posterior for physical planning. + +A coordinate's full coverage comparison is produced only when all 128 planned fits for that case and budget are available and numerically complete. +Missing shards, infrastructure errors and noncomplete numerical results remain explicit. +Partial successful fits cannot silently replace the planned denominator. +The reporter checks every population's checksum, sampler configuration, data identity, original weights, empirical quantiles and CDF values. +It regenerates the independent dataset and solves the Gaussian reference again before accepting the reported truth or reference values. +It also checks identical datasets and exact reference distributions across the two budgets. + +## Validation and compute schedule + +Preflight `22677340` completed in ten allocation seconds on one compute CPU, with 3.28 worker seconds and zero native simulator actions. +Independent dense-grid integration agrees with the closed-form posterior means to at most 1.08e-14 and covariance entries to at most 2.34e-15 on the checked datasets. +Repeated atomic target evaluations match exactly and agree with direct Gaussian likelihood calculations. +A complete 256-particle correlated-case trial also finishes; this single dataset is harness validation, not a coverage result. + +Report validation `22677390` completed in ten allocation seconds on one CPU, checking a full weighted population and rejecting altered truth, readings, exact CDF, empirical CDF, quantiles, coverage indicators and particle count. +The full array `22677408` has started after successful report validation; its first four shards are running and the remaining four wait for the concurrency limit. +It has eight shards with at most four simultaneous jobs, each requesting one CPU, 4 GB and two hours on `mit_preemptable`. +The scheduler rejected a dependency on the earlier completed preflight; its successful accounting and report were verified directly before submission, and the current validation dependency remains in place. +No array was created by the rejected submission. + +Finite summary `22677409` waits for report validation and termination of the array, retaining any incomplete outcomes. +These jobs perform no native physics simulation and send no MB/MF notifications. +The production estimator and the ongoing physical-domain comparisons remain unchanged. From 9d9935cb89ac30f5498e664dda24e99c1c251934 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:52:10 -0400 Subject: [PATCH 24/94] Verify coverage reporting preserves planned dataset counts --- docs/uncertainty/repeated-dataset-reference.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/uncertainty/repeated-dataset-reference.md b/docs/uncertainty/repeated-dataset-reference.md index 959ac92ad..59d4276c1 100644 --- a/docs/uncertainty/repeated-dataset-reference.md +++ b/docs/uncertainty/repeated-dataset-reference.md @@ -64,3 +64,13 @@ No array was created by the rejected submission. Finite summary `22677409` waits for report validation and termination of the array, retaining any incomplete outcomes. These jobs perform no native physics simulation and send no MB/MF notifications. The production estimator and the ongoing physical-domain comparisons remain unchanged. + +## Partial-report execution check + +Compute job `22677945` completed in six allocation seconds on one CPU, performing no new inference trials or native simulation. +It ran the complete report path on 221 actual fitted populations from the running study, verifying their saved weights, samples, quantiles, data identities and exact references. +The four case/budget groups contained 57, 55, 55 and 54 available datasets respectively. +Every group correctly retained its planned denominator of 128 and withheld coverage estimates while incomplete. +The report remained incomplete, and the guard verified that partial successful fits cannot produce a full-group coverage result. +The frozen report and validation are `partial-summary-22677945.json` and `partial-validation-22677945.json` in the study bundle. +This checks the reporting path on real sampler outputs; it does not establish the study's eventual coverage or numerical accuracy. From 1a8881470954fcad0f1316c6ffb7d9d79a4ddbbc Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 08:59:51 -0400 Subject: [PATCH 25/94] Prepare verified full-recording Fan population comparison --- docs/uncertainty/fan-joint-inference.md | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/uncertainty/fan-joint-inference.md b/docs/uncertainty/fan-joint-inference.md index 95c4a618b..469e7cc96 100644 --- a/docs/uncertainty/fan-joint-inference.md +++ b/docs/uncertainty/fan-joint-inference.md @@ -151,3 +151,31 @@ The recovery allocations are twelve hours each, and original-attempt costs remai They are not additional independent fits or agent outcomes. The earlier descriptions of live original jobs and pending recoveries above record the state when the safeguards were implemented. No completed or numerically assessed Fan posterior is claimed by this update. + +## Complete-population report preparation + +The two full-recording recovery runs have reached saved stages 30 and 31 of 32, with their target-evaluation counters advancing. +Their configuration has 64 particles and uses all 132 recorded actions. +They are distinct from the newer prefix-only fits and the 128-particle prefix budget comparison. + +The report bundle `logs/uncertainty_fan_full_fit_summary_20260913` pins the recovery plan, original runtime overlay and worker scripts. +A completed source report must have a complete sampler at temperature one, the expected seed and numerical configuration, and matching prior and inference identities. +The reporter restores the complete sampler checkpoint under the frozen implementation and requires the entire recovered result to match the source report without requesting another target evaluation. +Weighted fan-speed quantiles are independently reconstructed and must match the saved values. +The report also retains mean, standard deviation, distinct positive-weight values, largest exact-value mass, ancestry and evaluation counts. + +The paired comparison requires identical complete target identities, priors, numerical configurations and recorded runtimes. +It reports the maximum difference between weighted empirical CDFs and their one-dimensional Wasserstein distance. +These are descriptive distances between dependent numerical populations, not an independent-sample hypothesis test or an automatic adequacy decision. +There are no reserved observations in this full-recording study, so marginal agreement cannot substitute for the separate prefix forecast comparison. + +Original timeout and recovery allocation costs are reported separately and combined only when both accounting records are available. +Latest-attempt worker time and native actions remain separate from sampler evaluations carried in the checkpoint. +Missing or noncomplete fits remain explicit; neither is interpreted as an agent outcome. + +Compute validation `22678043` completed in eleven allocation seconds on one CPU and performed no native simulation. +Known discrete distributions verify the weighted quantile convention, zero-weight exclusions, CDF distance and Wasserstein distance; invalid weights, lost mass and nonfinite values are rejected. +The initial report correctly remains incomplete while both source fits are running. +This validates the marginal calculations and incomplete-report path; completed-checkpoint recovery will be exercised when completed source fits exist. +Finite report `22678058` depends on successful validation and termination of both recovery runs. +It requests one CPU, 4 GB and ten minutes on `mit_preemptable`, without sending notifications or modifying the fits. From f4169d531eec51e1f026c9e9c687902a99bac812 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:20:30 -0400 Subject: [PATCH 26/94] Record repeated-dataset coverage and first full Fan fit --- docs/uncertainty/fan-joint-inference.md | 7 +++++ .../uncertainty/repeated-dataset-reference.md | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/docs/uncertainty/fan-joint-inference.md b/docs/uncertainty/fan-joint-inference.md index 469e7cc96..448ffeebe 100644 --- a/docs/uncertainty/fan-joint-inference.md +++ b/docs/uncertainty/fan-joint-inference.md @@ -179,3 +179,10 @@ The initial report correctly remains incomplete while both source fits are runni This validates the marginal calculations and incomplete-report path; completed-checkpoint recovery will be exercised when completed source fits exist. Finite report `22678058` depends on successful validation and termination of both recovery runs. It requests one CPU, 4 GB and ten minutes on `mit_preemptable`, without sending notifications or modifying the fits. + +The full-recording seed-201 recovery `22650787_1` has completed at 15,092 target evaluations, using 30,790 allocation seconds and 30,769.80 worker seconds in this recovery attempt. +It records 1,983,960 native actions in the recovery and retains one initial ancestor. +Report `22678127` now exercises complete-checkpoint recovery successfully and independently verifies its fan-speed quantiles: [0.09034013, 0.09036351, 0.09053017]. +There are eight distinct positive-weight speeds, with 0.75 of empirical mass at one exact speed value. +This concentration does not establish precise identification; the paired result remains incomplete until seed 200 finishes. +Including the original 28,812-second timeout, this seed has consumed 59,602 allocation CPU-seconds across the two attempts. diff --git a/docs/uncertainty/repeated-dataset-reference.md b/docs/uncertainty/repeated-dataset-reference.md index 59d4276c1..cd48e8916 100644 --- a/docs/uncertainty/repeated-dataset-reference.md +++ b/docs/uncertainty/repeated-dataset-reference.md @@ -74,3 +74,32 @@ Every group correctly retained its planned denominator of 128 and withheld cover The report remained incomplete, and the guard verified that partial successful fits cannot produce a full-group coverage result. The frozen report and validation are `partial-summary-22677945.json` and `partial-validation-22677945.json` in the study bundle. This checks the reporting path on real sampler outputs; it does not establish the study's eventual coverage or numerical accuracy. + +## Completed repeated-dataset study + +All eight array tasks and final report `22677409` have completed, with all 512 planned fits numerically complete. +Every case/budget group contains all 128 independently generated datasets; no failed or missing fits are excluded. +The report verifies the full saved weighted populations and paired data/reference identities. + +| Case and coordinate | Particles | Central 90% interval coverage | Mean absolute CDF error | 95th percentile CDF error | Numerical screen | +| --- | ---: | ---: | ---: | ---: | --- | +| Stationary position | 256 | 115/128 (89.84%) | 0.02288 | 0.05267 | Pass | +| Stationary position | 2,048 | 116/128 (90.63%) | 0.00807 | 0.02146 | Pass | +| Linear initial position | 256 | 114/128 (89.06%) | 0.02977 | 0.07698 | Fail | +| Linear velocity | 256 | 115/128 (89.84%) | 0.02750 | 0.07630 | Fail | +| Linear initial position | 2,048 | 118/128 (92.19%) | 0.00933 | 0.02194 | Pass | +| Linear velocity | 2,048 | 117/128 (91.41%) | 0.00915 | 0.02156 | Pass | + +The exact posterior covers the generating truth in 114/128 stationary-position cases, 118/128 linear-position cases and 116/128 velocity cases. +Neither candidate nor exact-reference coverage triggers the predeclared adjusted binomial diagnostic. +The intervals are too broad to claim tight calibration: the adjusted coverage interval for the first row, for example, is approximately [0.7843, 0.9652]. + +All three coordinate groups pass the numerical CDF-error screens at 2,048 particles. +At 256 particles, both correlated-coordinate groups fail those screens despite coverage close to 90%. +This directly demonstrates that apparently plausible interval coverage can conceal numerical posterior errors and supports retaining separate numerical checks. +These are successful synthetic prior-predictive reference checks at the larger budget, not calibration of the chosen physical scene priors or permission to use a physical posterior in planning. + +The eight shard allocations use 902, 682, 342, 351, 807, 384, 594 and 854 seconds respectively on one CPU each, with a concurrency limit of four. +Different compute-node hardware is retained in the job accounting; no hardware speedup is inferred. +The final reporting job uses six allocation seconds and performs no native simulation. +The full report checksum is `93f9c28813c2892a3404cd03907947cb8d97c972acd27ed6341edd6198d9f895`. From 0c52835485e7b585f3bb332921de343e6c9aeec8 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:21:44 -0400 Subject: [PATCH 27/94] Derive constant-output likelihoods with correlated error --- docs/uncertainty/constant-output-guides.md | 61 ++++++++++ docs/uncertainty/implementation-progress.md | 7 +- .../inference_output_error.py | 104 ++++++++++++++++++ .../test_inference_output_error.py | 79 ++++++++++++- 4 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 docs/uncertainty/constant-output-guides.md diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md new file mode 100644 index 000000000..6c16d5289 --- /dev/null +++ b/docs/uncertainty/constant-output-guides.md @@ -0,0 +1,61 @@ +# Constant-output likelihood and proposal guides + +The Fan proposal currently uses only the first noisy reading to place each fixture coordinate. +The frozen output model then scores every prefix reading, with correlated Gaussian output discrepancy retained. +A more informed proposal can use those prefix readings while preserving the original prior and complete likelihood. +This addresses proposal efficiency; it does not declare an object stationary from noisy observations or change the acting agent's execution estimator. + +## Implemented calculation + +`constant_output_likelihood()` in `inference_output_error.py` integrates the existing scalar AR(1) discrepancy model for a constant simulator output. +Its affine Kalman innovations collect a quadratic log likelihood in linear time, including missing readings at their original primitive-step positions. +The returned `ConstantOutputLikelihood` stores the center, scale and log density at the peak: + +```text +log p(readings | location) = log_peak - 0.5 * ((location - center) / sigma)^2 +``` + +This is a reading likelihood, not a normalized parameter posterior. +A caller must still supply its original prior and justify the output model. +The helper retains the model's initial discrepancy variance, temporal correlation, innovations and declared sensor noise. +Only when the discrepancy vanishes does it reduce to ordinary independent Gaussian averaging. +Exact observations require the existing explicit-conditioning machinery; zero sensor noise, missing evidence and unrepresentable numerical scales are rejected rather than assigned a floor. + +A static-fixture proposal can use the Gaussian center and scale, truncated to its original support, while retaining its explicit prior/proposal density correction. +The original simulator likelihood must still score each reading exactly once. +Using a data-informed proposal does not authorize adding the guide likelihood as additional evidence. +Quantized or otherwise transformed physical readouts can make the constant-location Gaussian an approximate guide; they remain governed by the original target evaluator. +The supplied history must exclude any reserved future when constructing a fitting proposal. + +## Verification + +Compute job `22678241` passed 33 functional tests, two-file mypy and pylint, and pinned formatting checks. +Independent dense Gaussian calculations verify centers, scales and full log densities for positive, negative, zero and unit persistence, missing readings and nonzero initial error. +Existing filtering and forecast tests also pass. +Separate tests check independent averaging, translation stability, exact-observation rejection and explicit numerical failures. +The final formatted source hashes are retained in `logs/uncertainty_constant_output_checks_20260913/checked-sources.json` and match the committed files. + +The subsequent Fan audit `22678319` completed in 34 allocation seconds on one compute CPU, with zero native simulator actions. +It reconstructs the exact 64-action fitting-data identity before extracting the 65 prefix readings. +Across 128 saved positive-weight prefix histories, all 30 fixture x/y/z coordinates remain exactly constant. +This verifies those histories, not arbitrary new candidate programs or every possible initial scene. +For each coordinate, the new quadratic likelihood matches the original scalar filtering likelihood at four locations to at most 1.14e-13 log-density error. + +| Fixture-location guide | Scale | +| --- | ---: | +| First reading alone | 5.000 mm | +| All 65 prefix readings with the existing correlated output-error law | 3.312 mm | +| Treating those readings as independent sensor noise only | 0.620 mm | + +The correlated guide's center differs from the first reading by as much as 6.067 mm across the fixture coordinates. +Using the independent-reading formula would substantially overstate the information under the current output model. +The new calculation retains the correlation instead of introducing a new noise assumption. + +## Next comparison + +The audit bundle is `logs/uncertainty_fan_constant_guide_audit_20260913`, with report checksum `5e0298336461e9ea09639a7828abbb20dd91ae4d7c17b06db3151051388b10a0`. +Before launching a guided fit, verify its complete proposal density, retain original support and demonstrate target equality at identical physical candidates. +A mixture with the original proposal can preserve support while the guide focuses fixture locations. +Keep the original canonical scene coordinates in saved joint samples, so proposal changes do not silently change their meanings. +Any new fit must retain the same fitting prefix, original scene prior, physical program and output model, and remain separate from the ongoing particle-count comparison. +No guided physical inference run has been launched by this change, and no production fitter or planning rule uses the helper yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 81103c123..fe188ec3c 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -31,8 +31,11 @@ This supplies active-carry repeated-data coverage and supports an immutable orig The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. The existing long-prefix Balloons and full-recording Fan fits remain separate ongoing diagnostics. -The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) now has a validated exact-posterior preflight and a queued 512-fit coverage study across two cases and particle budgets. -It addresses statistical behavior across independent synthetic datasets while keeping physical-domain calibration and live-agent acceptance separate. +The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has completed all 512 fits. +All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. +This supplies larger-budget synthetic prior-predictive reference evidence, while physical-domain calibration and live-agent acceptance remain separate. +The new [constant-output guide calculation](constant-output-guides.md) passes 33 tests and focused static checks, and matches all checked Fan fixture likelihoods while retaining correlated discrepancy. +A prefix-only saved-history audit verifies the 30 fixture coordinates and a 3.312 mm Gaussian guide scale; proposal-density and identical-target checks are required before any guided physical fit. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. diff --git a/predicators/code_sim_learning/inference_output_error.py b/predicators/code_sim_learning/inference_output_error.py index bf89cf06f..0a09d391f 100644 --- a/predicators/code_sim_learning/inference_output_error.py +++ b/predicators/code_sim_learning/inference_output_error.py @@ -60,6 +60,110 @@ def digest(self) -> str: sort_keys=True).encode("utf-8")) +@dataclass(frozen=True) +class ConstantOutputLikelihood: + """Gaussian-shaped likelihood of one constant simulator output. + + This is not a normalized posterior or a detected rest window. A + caller must justify the constant-output model and supply its prior. + It can also guide a density-corrected proposal while the original + simulator likelihood remains authoritative. + """ + center: float + sigma: float + log_peak: float + + def __post_init__(self) -> None: + if not all(math.isfinite(v) + for v in (self.center, self.sigma, self.log_peak)) or \ + self.sigma <= 0: + raise ValueError("A finite, positive-width location likelihood " + "is required") + + def log_likelihood(self, location: float) -> float: + """Evaluate the original reading density, including its constant.""" + if not math.isfinite(location): + raise ValueError("Constant output must be finite") + residual = (location - self.center) / self.sigma + value = self.log_peak - .5 * residual * residual + if not math.isfinite(value): + raise ConditioningNumericalError("Location likelihood overflow") + return value + + +def constant_output_likelihood( + process: GaussianOutputError, observations: Tuple[Optional[float], + ...], + sensor_sigma: float) -> ConstantOutputLikelihood: + """Integrate AR errors and collect the likelihood of a constant output. + + The affine Kalman innovations give its location and precision in + linear time, including missing primitive-step readings. Correlated + discrepancy is retained; this is generally not an arithmetic mean + with sensor sigma divided by the square root of the sample count. + Positive sensor noise is required. Exact coordinates need explicit + conditioning rather than a finite-width proposal from this helper. + Only the caller's supplied history is used; online callers must not + supply future observations. + """ + readings = tuple(v for v in observations if v is not None) + if not readings or any(not math.isfinite(v) for v in readings): + raise ValueError("At least one finite reading is required") + if not math.isfinite(sensor_sigma) or sensor_sigma <= 0: + raise ValueError("Constant-output proposal requires positive noise") + baseline = readings[0] + mean, response = 0., 0. + variance = process.initial_sigma * process.initial_sigma + noise_variance = sensor_sigma * sensor_sigma + precision_terms: List[float] = [] + information_terms: List[float] = [] + constant_terms: List[float] = [] + for index, observed in enumerate(observations): + if index: + mean *= process.persistence + response *= process.persistence + variance = (process.persistence * process.persistence * variance + + process.innovation_sigma * process.innovation_sigma) + if observed is None: + continue + innovation_variance = variance + noise_variance + if not math.isfinite(innovation_variance) or \ + innovation_variance <= 0 or noise_variance == 0: + raise ConditioningNumericalError("Location variance overflow " + "or underflow") + innovation = (observed - baseline) - mean + sensitivity = 1. - response + scale = math.sqrt(innovation_variance) + whitened, slope = innovation / scale, sensitivity / scale + precision_terms.append(slope * slope) + information_terms.append(slope * whitened) + constant_terms.append(-.5 * whitened * whitened - math.log(scale) - + .5 * math.log(2 * math.pi)) + gain = variance / innovation_variance + mean += gain * innovation + response += gain * sensitivity + variance = gain * noise_variance + if not all( + math.isfinite(v) + for v in precision_terms + information_terms + constant_terms): + raise ConditioningNumericalError("Location information overflow") + try: + precision = math.fsum(precision_terms) + information = math.fsum(information_terms) + constant = math.fsum(constant_terms) + except OverflowError as err: + raise ConditioningNumericalError("Location information overflow") \ + from err + if not math.isfinite(precision) or precision <= 0: + raise ConditioningNumericalError("Location precision is unavailable") + offset = information / precision + center, sigma = baseline + offset, 1. / math.sqrt(precision) + peak = constant + .5 * information * offset + if not all(math.isfinite(v) for v in (center, sigma, peak)) or sigma <= 0: + raise ConditioningNumericalError("Location summary overflow") + return ConstantOutputLikelihood(center, sigma, peak) + + @dataclass(frozen=True) class ErrorFilterStep: """Causal error moments before and after this step's optional reading. diff --git a/tests/code_sim_learning/test_inference_output_error.py b/tests/code_sim_learning/test_inference_output_error.py index 15dfbef78..09c26905c 100644 --- a/tests/code_sim_learning/test_inference_output_error.py +++ b/tests/code_sim_learning/test_inference_output_error.py @@ -8,7 +8,8 @@ from predicators.code_sim_learning.inference_conditioning import \ ConditioningNumericalError from predicators.code_sim_learning.inference_output_error import \ - GaussianOutputError, output_error_likelihood + ConstantOutputLikelihood, GaussianOutputError, \ + constant_output_likelihood, output_error_likelihood @pytest.mark.parametrize("persistence", [0., -.4, .8, 1.]) @@ -104,3 +105,79 @@ def test_invalid_inputs_and_numeric_overflow_are_explicit() -> None: output_error_likelihood(process, (-1e308, ), (1e308, ), .1) assert process.digest != GaussianOutputError(.7, .1).digest assert process.digest != GaussianOutputError(.8, .2).digest + + +@pytest.mark.parametrize("persistence", [-1., 0., .9, 1.]) +@pytest.mark.parametrize("initial_sigma", [0., .12]) +def test_constant_output_matches_dense_gaussian(persistence: float, + initial_sigma: float) -> None: + """Unknown static location has the independently integrated GLS density.""" + process = GaussianOutputError(persistence, .08, initial_sigma) + readings = (None, .7, .8, None, .6, 1.) + sensor_sigma = .05 + size = len(readings) + transition = np.array( + [[persistence**(t - k) if k <= t else 0. for k in range(size)] + for t in range(size)]) + covariance = transition @ np.diag([initial_sigma**2] + [.08**2] * + (size - 1)) @ transition.T + indices = [i for i, v in enumerate(readings) if v is not None] + observed = np.array([readings[i] for i in indices], dtype=float) + covariance = covariance[np.ix_(indices, indices)] + \ + sensor_sigma**2 * np.eye(len(indices)) + ones = np.ones(len(indices)) + precision = ones @ np.linalg.solve(covariance, ones) + center = ones @ np.linalg.solve(covariance, observed) / precision + result = constant_output_likelihood(process, readings, sensor_sigma) + assert result.center == pytest.approx(center, abs=1e-12) + assert result.sigma == pytest.approx(precision**-.5, abs=1e-12) + for location in (-.2, result.center, 1.3): + residual = observed - location + expected = -.5 * (len(indices) * math.log(2 * math.pi) + + np.linalg.slogdet(covariance)[1] + + residual @ np.linalg.solve(covariance, residual)) + assert result.log_likelihood(location) == pytest.approx(expected, + abs=1e-10) + original = output_error_likelihood(process, (location, ) * size, + readings, sensor_sigma) + assert result.log_likelihood(location) == pytest.approx( + original.log_likelihood, abs=1e-10) + + +def test_constant_output_independent_readings_and_translation() -> None: + """The independent special case is an average; offsets preserve its law.""" + readings = (.9, 1.1, 1., 1.2) + process = GaussianOutputError(.9, 0., 0.) + result = constant_output_likelihood(process, readings, .2) + assert result.center == pytest.approx(1.05) + assert result.sigma == pytest.approx(.1) + shifted = constant_output_likelihood(process, + tuple(v + 1e6 for v in readings), .2) + assert shifted.center - 1e6 == pytest.approx(result.center, abs=1e-10) + assert shifted.sigma == result.sigma + assert shifted.log_peak == pytest.approx(result.log_peak, abs=1e-8) + correlated = constant_output_likelihood(GaussianOutputError(.9, .2, 0.), + readings, .2) + assert correlated.sigma > result.sigma + + +def test_constant_output_requires_usable_noisy_evidence() -> None: + """Exact constraints and unsupported numeric scales get explicit errors.""" + process = GaussianOutputError(.9, .005, 0.) + for readings in ((), (None, None), (float("nan"), )): + with pytest.raises(ValueError): + constant_output_likelihood(process, readings, .005) + for sigma in (0., -.1, float("inf")): + with pytest.raises(ValueError): + constant_output_likelihood(process, (1., ), sigma) + for sigma in (1e-300, 1e300): + with pytest.raises(ConditioningNumericalError): + constant_output_likelihood(process, (1., ), sigma) + with pytest.raises(ConditioningNumericalError): + constant_output_likelihood(process, (1e308, -1e308), .005) + with pytest.raises(ValueError): + ConstantOutputLikelihood(0., 0., 1.) + with pytest.raises(ValueError): + ConstantOutputLikelihood(0., 1., 0.).log_likelihood(float("inf")) + with pytest.raises(ConditioningNumericalError): + ConstantOutputLikelihood(0., 1e-300, 0.).log_likelihood(1e300) From df2d0ff4defaef2af5afb23e9d20bd13a5453e77 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:23:07 -0400 Subject: [PATCH 28/94] Record full Fan population disagreement and total costs --- docs/uncertainty/fan-joint-inference.md | 22 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/fan-joint-inference.md b/docs/uncertainty/fan-joint-inference.md index 448ffeebe..e52912874 100644 --- a/docs/uncertainty/fan-joint-inference.md +++ b/docs/uncertainty/fan-joint-inference.md @@ -186,3 +186,25 @@ Report `22678127` now exercises complete-checkpoint recovery successfully and in There are eight distinct positive-weight speeds, with 0.75 of empirical mass at one exact speed value. This concentration does not establish precise identification; the paired result remains incomplete until seed 200 finishes. Including the original 28,812-second timeout, this seed has consumed 59,602 allocation CPU-seconds across the two attempts. + +## Completed full-recording pair + +Both recoveries and final report `22678058` have completed. +The report verifies exact complete-checkpoint recovery for both fits, independently reconstructs their fan-speed summaries, and checks identical full target identities, priors, configurations and recorded runtimes. + +| Numerical seed | Fan-speed 5th / 50th / 95th percentiles | Retained ancestors | Largest exact-value mass | Target evaluations | Recovery allocation seconds | +| --- | --- | ---: | ---: | ---: | ---: | +| 200 | 0.08994192 / 0.08996900 / 0.08999337 | 1 | 0.859375 | 15,116 | 31,612 | +| 201 | 0.09034013 / 0.09036351 / 0.09053017 | 1 | 0.750000 | 15,092 | 30,790 | + +The two weighted empirical CDFs differ by 1.0 at their maximum, and their one-dimensional Wasserstein distance is 0.00040176. +Their means differ by approximately 0.000402, while their empirical standard deviations are only 0.0000148 and 0.0000439. +The small absolute mean separation does not validate these much narrower, nonoverlapping empirical uncertainty distributions. +Both populations remain numerically unassessed. +No independent-sample p-value or post-hoc acceptance threshold is attached to these distances. + +The recovery attempts record 1,987,128 and 1,983,960 native actions and 31,591.27 and 30,769.80 worker seconds. +Including each original timeout, the per-seed allocation costs are 60,424 and 59,602 CPU-seconds. +The final report uses four allocation seconds on one CPU and no native simulation. +There is no reserved future in this full-recording study, so these results do not establish predictive improvement or an agent advantage. +The separate prefix budget comparisons remain active; the [constant-output guide](constant-output-guides.md) prepares a density-corrected proposal investigation without changing their target models or running configurations. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index fe188ec3c..3814436b6 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -30,7 +30,8 @@ Fixed centers preserve every diagnostic report across all three repetitions; car This supplies active-carry repeated-data coverage and supports an immutable original prior, but does not establish unchanged future predictions or planning decisions. The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. -The existing long-prefix Balloons and full-recording Fan fits remain separate ongoing diagnostics. +The long-prefix Balloons fits remain ongoing. +Both full-recording Fan fits and their verified report have completed, but their narrow empirical speed distributions do not overlap and each retains one initial ancestor; they remain unassessed. The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has completed all 512 fits. All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. This supplies larger-budget synthetic prior-predictive reference evidence, while physical-domain calibration and live-agent acceptance remain separate. From 51ce7119767ea72255cbc9cbf67289d881fa22c6 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:41:24 -0400 Subject: [PATCH 29/94] Validate fixture proposal correction and launch matched Fan fits --- docs/uncertainty/constant-output-guides.md | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index 6c16d5289..33f38d2cb 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -51,11 +51,39 @@ The correlated guide's center differs from the first reading by as much as 6.067 Using the independent-reading formula would substantially overstate the information under the current output model. The new calculation retains the correlation instead of introducing a new noise assumption. -## Next comparison +## Guided proposal validation The audit bundle is `logs/uncertainty_fan_constant_guide_audit_20260913`, with report checksum `5e0298336461e9ea09639a7828abbb20dd91ae4d7c17b06db3151051388b10a0`. Before launching a guided fit, verify its complete proposal density, retain original support and demonstrate target equality at identical physical candidates. A mixture with the original proposal can preserve support while the guide focuses fixture locations. Keep the original canonical scene coordinates in saved joint samples, so proposal changes do not silently change their meanings. Any new fit must retain the same fitting prefix, original scene prior, physical program and output model, and remain separate from the ongoing particle-count comparison. -No guided physical inference run has been launched by this change, and no production fitter or planning rule uses the helper yet. +The guided comparison below is offline; no production fitter or planning rule uses the helper. + +The isolated implementation is in `logs/uncertainty_fan_fixture_guide_20260913`. +It samples a mixture with 10% original proposal mass and 90% fixture-guided mass, changing only the 30 fixture position coordinates. +Both components map into the same original 120-coordinate scene representation; an additional proposal-only coordinate selects the component. +The target subtracts the log density of the complete mixture in the original coordinate representation, regardless of the selected component. +The physical simulator program, original scene prior, sensor/output law and 64-action fitting prefix remain fixed. +The additional 68 actions remain reserved for future prediction assessment. + +Compute reference `22678659` verified proposal normalization, corrected zeroth/first/second moments and component-independent correction for identical represented points. +The normalization integral was 1.0000000000006977, and the corrected moments matched 1, 1/2 and 1/3 within 3.2e-10. +These are analytic proposal checks with zero native simulator actions, not a physical posterior result. +Native preflight `22678746` completed in 198 allocation seconds on four compute CPUs, evaluating 10,240 native actions. +It preserved all 12 original support/target witnesses and checked four guided candidates, including two supported and two unsupported cases. +Their original physical target factors were unchanged apart from the declared full-mixture correction. +The complete 64-particle serial and parallel initial sampler states were exactly equal. +Serial initialization took 99.88 seconds and parallel initialization took 24.61 seconds within this same allocation. + +Initialization is a caution, not a success claim: 15 of 64 candidates had finite support, but their weight effective sample size was only 2.010, compared with approximately 9.1 for the original proposal. +A proposal informed by the entire prefix can concentrate on locations that receive little mass at the initial tempered stage. +The correction preserves the target mathematically but does not ensure efficient initialization or later exploration. +Do not use supported-candidate counts alone to claim improvement. + +Array `22678810` launches two exploratory guided fits with sampler seeds 302 and 303, matching the original 64-particle Fan prefix pair. +Each uses four compute CPUs on `mit_preemptable`, with 32 temperatures, eight moves, the original 16,448-evaluation limit and a four-hour allocation limit. +The launch manifest pins the tested preflight, scripts, model and data/proposal plan. +These runs are separate from the unchanged 128-particle budget comparison. +Final population diversity, independent-run agreement, reserved-future predictions and total inference cost remain required before judging this guide. +The forecast adapter must replay saved canonical scene samples while verifying the proposal correction against the augmented sampler coordinates; the original forecast driver assumes these two coordinate arrays are identical and cannot be reused unchanged. From 9838557d410ddf9efc45dc6dab4ca583ff6db777 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:42:56 -0400 Subject: [PATCH 30/94] Prepare canonical-scene forecast validation for guided fits --- docs/uncertainty/constant-output-guides.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index 33f38d2cb..6b8f80130 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -87,3 +87,11 @@ The launch manifest pins the tested preflight, scripts, model and data/proposal These runs are separate from the unchanged 128-particle budget comparison. Final population diversity, independent-run agreement, reserved-future predictions and total inference cost remain required before judging this guide. The forecast adapter must replay saved canonical scene samples while verifying the proposal correction against the augmented sampler coordinates; the original forecast driver assumes these two coordinate arrays are identical and cannot be reused unchanged. + + +The adapted forecast validation bundle is `logs/uncertainty_fan_guided_forecast_20260913`. +Short sampler fixture `22678838` is queued, followed on successful completion by forecast validation `22678841_0`. +The adapter first restores the completed augmented-coordinate checkpoint without additional target evaluations. +It transforms each positive-weight proposal into the saved original scene coordinates, then checks the original native prefix likelihood and the fully corrected base density during full-trajectory replay. +The first positive-weight history is repeated, and every history is saved with its unchanged population weight. +These jobs validate the adapter only; full guided-fit forecasts have not yet been submitted. From ec079f9610826e9e5530c60498523ea75d7fe156 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 09:50:24 -0400 Subject: [PATCH 31/94] Validate matched Fan proposal comparison and gate forecasts --- docs/uncertainty/constant-output-guides.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index 6b8f80130..557ee912a 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -94,4 +94,24 @@ Short sampler fixture `22678838` is queued, followed on successful completion by The adapter first restores the completed augmented-coordinate checkpoint without additional target evaluations. It transforms each positive-weight proposal into the saved original scene coordinates, then checks the original native prefix likelihood and the fully corrected base density during full-trajectory replay. The first positive-weight history is repeated, and every history is saved with its unchanged population weight. -These jobs validate the adapter only; full guided-fit forecasts have not yet been submitted. +These jobs validate the adapter only; the full forecasts below depend on the additional saved-artifact verification passing. + + +## Matched prediction comparison + +The comparison reader is in `logs/uncertainty_fan_guided_summary_20260913`. +It permits the intended proposal/conditioning identity change while requiring identical data, observation model, physical program, original scene prior and sampler settings apart from the additional proposal coordinate. +The original simulator and scene mapping are byte-identical between arms. +The only changed inference overlay adds the two constant-output helper definitions; removing those additions leaves the entire original module syntax tree unchanged. +The original target evaluator is preserved byte-for-byte in the guided wrapper. + +Validation `22678900` completed in 11 allocation seconds on one compute CPU, independently checking all 128 positive-weight original forecast histories and their unchanged weights. +It rejected eight incompatible comparisons covering data, sensor, program, prior, sampler configuration, coordinate meanings, source runtime and target factorization. +This establishes comparison checks, not guided prediction quality. +The guided saved-artifact verifier `22678924` remains dependent on the native forecast fixture. +It checks every canonical scene and full-mixture correction, then tests rejection of altered or missing coordinate checks, corrections, weights, predictions, particles and checkpoint identities. + +Full guided forecasts `22678925_1` and `22678926_2` are queued behind that verifier and their respective completed fits. +Summary `22678943` waits for the validation jobs and both forecast terminal states, retaining missing or unavailable outcomes explicitly. +The summary reports both original and both guided populations, all available pairwise prediction disagreements, parameter mass concentration and inference/forecast costs. +No guided prediction result or adequacy conclusion is available yet. From 34ca72d3c4eee688a3b3d5b22d4efe4d2be41df1 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 10:08:18 -0400 Subject: [PATCH 32/94] Validate alternative tempering and recover guided forecast runtime --- docs/uncertainty/constant-output-guides.md | 29 ++++++++- docs/uncertainty/guided-tempering.md | 65 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 9 ++- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 docs/uncertainty/guided-tempering.md diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index 557ee912a..cbe4c1564 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -111,7 +111,32 @@ This establishes comparison checks, not guided prediction quality. The guided saved-artifact verifier `22678924` remains dependent on the native forecast fixture. It checks every canonical scene and full-mixture correction, then tests rejection of altered or missing coordinate checks, corrections, weights, predictions, particles and checkpoint identities. -Full guided forecasts `22678925_1` and `22678926_2` are queued behind that verifier and their respective completed fits. -Summary `22678943` waits for the validation jobs and both forecast terminal states, retaining missing or unavailable outcomes explicitly. +Full guided forecasts `22678925_1` and `22678926_2` were initially queued behind that verifier and their respective completed fits. +Summary `22678943` was initially dependent on the validation jobs and both forecast terminal states; this first chain was cancelled after the setup failure below. The summary reports both original and both guided populations, all available pairwise prediction disagreements, parameter mass concentration and inference/forecast costs. No guided prediction result or adequacy conclusion is available yet. + + +## Forecast validation recovery + +The short sampler fixture `22678838` completed in 84 allocation seconds, evaluating 7,808 native actions. +The first forecast `22678841_0` failed during setup because its fit-only snapshot omitted `log_future_likelihood`; it performed zero native forecast actions. +Its dependent artifact verifier, full forecasts and summary were cancelled without starting. +This is a setup failure, not a posterior or agent failure. + +The corrected bundle `logs/uncertainty_fan_guided_forecast_v2_20260913` reuses the completed fixture checkpoint. +Its observation module retains every original density method unchanged and adds the three future-scoring/sampling methods from the previously validated forecast implementation. +The comparison checks verify the original syntax tree after removing only those explicit additions and their imports. +Native forecast `22679381_0` completed in 69 allocation seconds on four CPUs, reconstructing all 64 positive-weight histories plus the repeated first history in 8,580 native actions. + +An independent reader on node3103 failed strict coordinate reconstruction, while the same reader passed every coordinate and density correction on the source node1412 in `22679467`. +A separate audit on node1459 found no coordinate mismatch; the diagnostic on node3103 remains queued as `22679468`. +The cross-machine discrepancy is not yet explained. +The strict verifier now explicitly requires the recorded source node, and its validation and final-summary allocations are pinned to that node. +Final checks `22679489` and `22679490` passed in 8 and 16 allocation seconds, verifying the updated comparison reader and all 64 native artifacts while rejecting eight comparison mismatches and seven artifact corruptions. + +The replacement full forecasts are `22679493_1` and `22679494_2`, gated on those checks and their respective completed fits. +Summary `22679496` retains all planned populations and waits for the forecast terminal states. +The v2 reader and submission records are in `logs/uncertainty_fan_guided_summary_v2_20260913`. +These validations establish forecast reconstruction, not numerical adequacy of a completed guided posterior. +The separate [tempering comparison](guided-tempering.md) tests the initial-weight concentration problem without changing these running fits. diff --git a/docs/uncertainty/guided-tempering.md b/docs/uncertainty/guided-tempering.md new file mode 100644 index 000000000..d0bac2dae --- /dev/null +++ b/docs/uncertainty/guided-tempering.md @@ -0,0 +1,65 @@ +# Guided proposals and the tempering sequence + +The first Fan fixture guide preserves the original physical posterior but has an inefficient initial numerical distribution. +Its native initial population contains 15 supported candidates out of 64, with weight effective sample size only 2.010. +The first completed resampling stage of seed 302 retains two original ancestors. +These observations motivate changing the numerical bridge while retaining the original prior, likelihood, proposal and final target. +They do not establish that the first guided fit will fail its final prediction checks. + +## Target and intermediate distributions + +Let `u` denote the original canonical scene coordinates, `b(u)` their existing initial log factor, and `l(u)` the remaining prefix log likelihood. +Let `q(u)` be the complete mixture density induced by the augmented guided proposal in the original coordinate representation. +The mixture retains 10% original proposal mass, so its density is strictly positive throughout the original support. +Both constructions below retain all existing exact-observation and geometry support checks. + +```text +Current bridge: b(u) - log q(u) + beta * l(u) +Alternative bridge: b(u) + beta * (l(u) - log q(u)) +Final target: b(u) + l(u) - log q(u), at beta = 1 +``` + +Both final augmented-coordinate targets induce the same original physical posterior after accounting for the proposal transformation. +The alternative initial distribution is a guided numerical reference, not the original prior or the posterior after only the first observation. +It avoids immediately undoing the guide through the initial importance weights. +No guide likelihood is added as extra observation evidence, and the original physical prior is not narrowed. +The stored annealed factor now includes the finite mixture correction; a forecast reader must reconstruct the actual prefix likelihood explicitly rather than interpret that stored factor as a raw likelihood. +Intermediate distributions are numerical choices; their final density correction and exact support remain mandatory. + +## Exact reference + +The reference has independent standard normal priors, Gaussian observations, and analytically available Gaussian posteriors. +One additional coordinate is unobserved and must retain its original prior. +The guide equals the exact posterior for the observed coordinates and retains the same defensive mixture with the original proposal. +Each cell uses four sampler seeds, 64 particles, 32 temperatures, four moves per temperature and the same numerical budget. +The two bridges use identical original priors, observations, proposals and final target factors. + +| Observed coordinates | Correction placement | Mean marginal CDF error, four fits | Mean posterior-mean RMSE, four fits | Surviving original ancestors | +| --- | --- | ---: | ---: | ---: | +| 1 | Initial base | 0.03503 | 0.07824 | 29-64 | +| 1 | Annealed factor | 0.02798 | 0.04664 | 64 | +| 30 | Initial base | 0.14675 | 0.38403 | 3-4 | +| 30 | Annealed factor | 0.03633 | 0.08177 | 64 | + +CDF error averages absolute discrepancies at reference probabilities 0.05, 0.25, 0.5, 0.75 and 0.95 over every coordinate, including the uninformed coordinate. +This is an exploratory numerical comparison on exact references, not a physical-domain calibration or agent-performance result. +It does not prove that the alternative bridge is uniformly better for approximate or misleading guides. +Array `22679038` completed all 16 fits in 11 and 8 allocation seconds on one CPU per shard. +Independent verification `22679339` completed in 14 seconds, deriving posterior parameters from prior and observation precisions and recomputing every saved population's summaries. +The reference uses zero native simulator actions. +Artifacts are in `logs/uncertainty_guided_tempering_reference_20260913`. + +## Native Fan preflight and comparison + +Preflight `22679317` completed in 176 allocation seconds on four compute CPUs, evaluating 10,240 native actions. +It verifies the changed factorization at the same supported and unsupported physical candidates and preserves the complete final target. +The complete serial and parallel initial sampler states agree exactly. +All 15 supported initial candidates now receive equal weights, giving effective sample size 15.0 instead of 2.010 under the first bridge. +That initialization result does not establish final parameter or prediction accuracy. + +Array `22679393` runs the alternative bridge with sampler seeds 302 and 303, matching the existing guided pair's proposal, 64 particles, 32 temperatures, eight moves, 16,448-evaluation limit and four-hour allocation limit. +Each fit uses four CPUs on `mit_preemptable`. +The original guided fits remain frozen as the comparison arm. +The native bundle is `logs/uncertainty_fan_tempered_guide_20260913`, with complete-stage checkpoints and a launch manifest that pins the passed preflight and exact-reference verification. +Reserved-future forecasts still need an adapter that accounts for the correction in the annealed factor. +Neither bridge is used by the acting agent. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 3814436b6..6df0fcd5a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -36,7 +36,14 @@ The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has com All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. This supplies larger-budget synthetic prior-predictive reference evidence, while physical-domain calibration and live-agent acceptance remain separate. The new [constant-output guide calculation](constant-output-guides.md) passes 33 tests and focused static checks, and matches all checked Fan fixture likelihoods while retaining correlated discrepancy. -A prefix-only saved-history audit verifies the 30 fixture coordinates and a 3.312 mm Gaussian guide scale; proposal-density and identical-target checks are required before any guided physical fit. +A prefix-only saved-history audit verifies the 30 fixture coordinates and a 3.312 mm Gaussian guide scale. +The guided proposal subsequently passed density correction, original physical target and serial/parallel initialization checks. +Two guided 64-particle fits and gated reserved-future forecasts are submitted, with the original prior, program and observation model retained. +Initial weight effective sample size is worse than the original proposal, so this guide remains an exploratory comparison rather than an improvement claim. +The [guided-tempering reference](guided-tempering.md) completed 16 exact Gaussian fits; annealing the finite mixture correction reduces error in these references while preserving the final target. +Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010; two matched alternative-bridge fits are running. +The first guided forecast snapshot failed before native actions because it lacked the future-likelihood method. +The corrected snapshot reuses the completed fixture and passes full native replay; strict coordinate verification is pinned to the source node while a cross-machine mismatch remains under investigation. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. From 3045226ad9a37c630baedc25b5d4df2b50155d18 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 10:19:03 -0400 Subject: [PATCH 33/94] Validate tempered forecasts and queue six-population comparison --- docs/uncertainty/guided-tempering.md | 29 ++++++++++++++++++++- docs/uncertainty/implementation-progress.md | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/guided-tempering.md b/docs/uncertainty/guided-tempering.md index d0bac2dae..43033e166 100644 --- a/docs/uncertainty/guided-tempering.md +++ b/docs/uncertainty/guided-tempering.md @@ -61,5 +61,32 @@ Array `22679393` runs the alternative bridge with sampler seeds 302 and 303, mat Each fit uses four CPUs on `mit_preemptable`. The original guided fits remain frozen as the comparison arm. The native bundle is `logs/uncertainty_fan_tempered_guide_20260913`, with complete-stage checkpoints and a launch manifest that pins the passed preflight and exact-reference verification. -Reserved-future forecasts still need an adapter that accounts for the correction in the annealed factor. +The reserved-future adapter below accounts for the correction in the annealed factor. Neither bridge is used by the acting agent. + + +## Reserved-future forecast validation + +The forecast bundle is `logs/uncertainty_fan_tempered_forecast_20260913`. +It uses the verified forecast-only observation extension, preserving the original density methods and fitting data. +The short numerical fixture `22680058` completed in 79 allocation seconds on four CPUs, evaluating 7,808 native actions. +Its full-trajectory forecast `22680068_0` completed in 72 allocation seconds, evaluating 8,580 native actions across all 64 positive-weight scenes and one repeated history. +This fixture validates the adapter and is not an assessed posterior or agent seed. + +For every scene, the native replay checks the original initial factor against the stored base and the remaining prefix likelihood minus the complete mixture correction against the stored annealed factor. +The report retains the raw prefix likelihood, initial likelihood and canonical prior/proposal ratio separately. +The independent reader reconstructs these factors from the saved report and checkpoint while also checking original weights, complete histories, native events and goals. +Its first attempt `22680091` failed because it compared numeric factors directly with their checkpoint string encoding. +The corrected reader decodes those values and reuses the completed native artifacts; no simulator work was repeated to fix this reporting error. + +Final comparison validation `22680173` passed in 19 allocation seconds on one CPU, verifying 128 original histories and rejecting eight incompatible comparison contracts. +Final native-artifact validation `22680174` passed in 20 seconds on the source node, verifying all 64 tempered histories and rejecting nine corruptions, including changed raw likelihood and initial density factors. +The corrected source hashes match both validation reports. +Incomplete-report validation `22680175` passed in nine seconds, retaining all six planned populations while only the two original forecasts are available. + +Full tempered forecasts `22680182_1` and `22680183_2` depend on the passed checks and their respective completed fits. +Summary `22680197` also waits for the original guided forecasts `22679493_1` and `22679494_2`. +It compares two original, two base-corrected guided and two tempered-correction populations, retaining all 15 pairwise comparisons when available. +Original physical prior, data, program and output law remain fixed; the reader explicitly verifies the algebraic factorization change instead of requiring the numerical bridge identities to be identical. +The reporting bundle is `logs/uncertainty_fan_tempered_summary_20260913`. +No completed tempered posterior forecast or live-agent acceptance result is available yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 6df0fcd5a..b9d16b5a8 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -42,6 +42,7 @@ Two guided 64-particle fits and gated reserved-future forecasts are submitted, w Initial weight effective sample size is worse than the original proposal, so this guide remains an exploratory comparison rather than an improvement claim. The [guided-tempering reference](guided-tempering.md) completed 16 exact Gaussian fits; annealing the finite mixture correction reduces error in these references while preserving the final target. Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010; two matched alternative-bridge fits are running. +The short native tempered forecast and independent raw-factor/coordinate verification now pass, with full forecasts and a six-population comparison gated on the completed fits. The first guided forecast snapshot failed before native actions because it lacked the future-likelihood method. The corrected snapshot reuses the completed fixture and passes full native replay; strict coordinate verification is pinned to the source node while a cross-machine mismatch remains under investigation. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. From c857fa3c5a39b55b06be7fd5eb032a86ada7ceca Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 10:32:26 -0400 Subject: [PATCH 34/94] Identify numerical dispatch behind coordinate verification mismatch --- docs/uncertainty/constant-output-guides.md | 8 ++++++-- docs/uncertainty/implementation-progress.md | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index cbe4c1564..226a31eaa 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -130,8 +130,12 @@ The comparison checks verify the original syntax tree after removing only those Native forecast `22679381_0` completed in 69 allocation seconds on four CPUs, reconstructing all 64 positive-weight histories plus the repeated first history in 8,580 native actions. An independent reader on node3103 failed strict coordinate reconstruction, while the same reader passed every coordinate and density correction on the source node1412 in `22679467`. -A separate audit on node1459 found no coordinate mismatch; the diagnostic on node3103 remains queued as `22679468`. -The cross-machine discrepancy is not yet explained. +A separate audit on node1459 found no coordinate mismatch. +The targeted node3103 audit `22679468` completed in four seconds and reproduced mismatches in all 64 stored rows, with maximum absolute difference 1.1102230246251565e-16 in canonical unit coordinates. +The controlled follow-up `22680353` ran the same script on node3103 with the additional NumPy AVX-512 dispatch options disabled and reproduced all canonical coordinates exactly. +Both runs use the same saved fixture and source-runtime metadata, with zero native simulator actions. +This identifies a numerical-dispatch cause for the coordinate check failure; it does not establish cross-hardware native trajectory parity or change the prior, observation noise, or posterior target. +The checked artifacts and feature-mask configuration are recorded in `dispatch-verification.json` in the v2 summary bundle. The strict verifier now explicitly requires the recorded source node, and its validation and final-summary allocations are pinned to that node. Final checks `22679489` and `22679490` passed in 8 and 16 allocation seconds, verifying the updated comparison reader and all 64 native artifacts while rejecting eight comparison mismatches and seven artifact corruptions. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index b9d16b5a8..722ca3e3a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -44,7 +44,9 @@ The [guided-tempering reference](guided-tempering.md) completed 16 exact Gaussia Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010; two matched alternative-bridge fits are running. The short native tempered forecast and independent raw-factor/coordinate verification now pass, with full forecasts and a six-population comparison gated on the completed fits. The first guided forecast snapshot failed before native actions because it lacked the future-likelihood method. -The corrected snapshot reuses the completed fixture and passes full native replay; strict coordinate verification is pinned to the source node while a cross-machine mismatch remains under investigation. +The corrected snapshot reuses the completed fixture and passes full native replay. +A controlled same-node audit attributes the 1.11e-16 cross-machine coordinate mismatch to NumPy instruction dispatch; disabling the extra AVX-512 dispatch options restores exact coordinates for the saved fixture. +Strict physical verification remains pinned to the validated source runtime; this does not certify general cross-hardware trajectory replay. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. From 9aeac7b9343b140324225d924e4d51a238a43649 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 10:53:19 -0400 Subject: [PATCH 35/94] Record completed Domino budget comparison and unresolved instability --- docs/uncertainty/domino-budget-sensitivity.md | 54 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 ++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/domino-budget-sensitivity.md b/docs/uncertainty/domino-budget-sensitivity.md index 1bcf5b102..658f29307 100644 --- a/docs/uncertainty/domino-budget-sensitivity.md +++ b/docs/uncertainty/domino-budget-sensitivity.md @@ -87,3 +87,57 @@ For the point-start replicas, median lateral friction is 0.2734 versus 0.4649 an Their largest exact-value masses reach 33.84% and 56.02%, respectively, across the five reported parameters. This does not prove that the parameter distributions are wrong, but it prevents interpreting agreement on one action sequence as established uncertainty over other plans. The comparison retains all empirical quantiles and concentration measures for the larger-population check. + + +## Completed 128-particle results + +All four fits and their forecasts have completed. +The final report `logs/uncertainty_domino_budget_summary_20260913/summary-22675955.json` retains all eight populations and all twelve within-target pairs. +Its existing history verifier checks all 768 positive-weight saved histories across the 64- and 128-particle settings. +Independent scalar verification `22680987` recomputes every reported parameter quantile, exact-value concentration and pairwise prediction difference from the saved populations and forecasts. +It confirms the report checksum `74b778eae972534e83bb9397951f8ef82dc25fa84b7604516f5a7f9937a67612`. +The initial joint-only snapshot and verification remain archived separately. + +| State treatment | Numerical seed | Conditional position RMSE | Toppling Brier error | Final domino 1 toppling probability | Surviving initial ancestors | Target evaluations | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Joint, 128 particles | 100 | 11.534 mm | 0.001360 | 0.8944 | 1 | 30,889 | +| Joint, 128 particles | 101 | 10.953 mm | 0.001072 | 0.8899 | 1 | 30,760 | +| Point start, 128 particles | 100 | 11.401 mm | 0.0002245 | 0.6981 | 10 | 31,907 | +| Point start, 128 particles | 101 | 11.427 mm | 0.00003125 | 0.8672 | 14 | 31,910 | + +These are predictions on the fixed 97-action suffix, not solve rates or additional agent seeds. +The reference suffix records domino 1 as toppled at its final frame. +Individual forecast errors on this one suffix do not determine which approximation is closer to the full posterior. + +| Replica pair | Position-mean RMS difference | Maximum toppling-curve gap | Maximum final toppling gap | Passed screens | +| --- | ---: | ---: | ---: | --- | +| Joint, 64 particles | 3.038 mm | 0.18385 | 0.01645 | Curve and final | +| Joint, 128 particles | 4.580 mm | 0.20429 | 0.02236 | Final only | +| Point start, 64 particles | 0.880 mm | 0.01215 | 0.01215 | All three | +| Point start, 128 particles | 0.353 mm | 0.17530 | 0.16910 | Position and curve | + +None of the six joint-state replica pairs passes all three screens. +Of the four joint cross-budget pairs, position-mean differences range from 1.631 to 5.102 mm and toppling-curve gaps range from 0.19493 to 0.37369. +Every joint pair passes the final-toppling screen, but agreement at the last frame does not remove the disagreement in preceding predictions. + +Three of the six point-state pairs pass all screens: the original 64-particle pair and its two comparisons with 128-particle seed 101. +Both comparisons with 128-particle seed 100 fail the toppling-curve and final-toppling screens, with final gaps of 0.27208 and 0.25993. +The 128-particle point-state pair itself fails the final-toppling screen. +The smaller-budget point-state agreement therefore does not justify treating its uncertainty as numerically established. +Removing uncertain initial states is not a validated shortcut around the sampling problem. + +Parameter concentration remains material in the joint fits. +Their median lateral frictions are 0.34490 and 0.08197, and the second fit places 97.72% of rolling-friction mass on one retained value. +Its rolling-friction 5th, 50th and 95th percentiles all coincide at 0.00343717. +These empirical quantiles must not be interpreted as precise identification without numerical validation. +The point-state fits retain more ancestry and distinct parameter values, but their decision-relevant probabilities still change across numerical budgets. + +The joint fits used 3:01:04 and 3:00:02 of allocation time on four CPUs; the point-state fits used 3:06:44 and 3:08:55 on four CPUs. +Each forecast evaluated 20,769 native actions, including the repeated first history, and used 129-133 allocation seconds on four CPUs. +The final existing report took 31 seconds on one CPU; the independent scalar check took two seconds and zero native actions. +Accounting includes the fit allocation rather than only the last sampling stage. + +This closes the planned 64-versus-128 comparison, not Stage B. +Doubling population size did not resolve the declared stability failures, and the evidence does not support publishing either state treatment as an assessed physical posterior. +The next numerical investigation must address exploration of parameter and state tradeoffs, rather than assume initial-state uncertainty alone explains the problem or that another population increase will suffice. +The incumbent agent remains unchanged. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 722ca3e3a..bb1fd2356 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -16,7 +16,11 @@ The [combined Domino assessment](domino-comparison-summary.md) now has all six c Matched local-only proposals fail all three initial stability screens; mixed local/full-range proposals reduce disagreement and pass both toppling screens, but still fail the position-mean screen. The fixed-initial-state approximation passes all three screens while showing worse position error and better toppling error than legacy on this recording. These are numerical replicas on one development recording, not agent seeds or evidence of calibrated uncertainty across tasks. -Four [larger Domino fits](domino-budget-sensitivity.md) are running at 128 particles with unchanged target priors and proposal rules, with forecast and comparison follow-ups already queued. +All four [larger Domino fits](domino-budget-sensitivity.md), their forecasts and the eight-population comparison have completed. +The 128-particle joint pair fails the position and toppling-curve screens; none of its six within-target comparisons across budgets passes all screens. +The 128-particle fixed-state pair fails the final-toppling screen, and two cross-budget comparisons show final-probability gaps above 0.25. +The earlier fixed-state agreement did not persist across budgets, so removing initial-state uncertainty is not a validated shortcut. +Both treatments remain unassessed, with inference exploration and cost unresolved. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. From 14dd18560b4293d80b0265a38bb5165dc07d4c96 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 11:18:44 -0400 Subject: [PATCH 36/94] Diagnose Domino proposal movement across scales and jitter --- docs/uncertainty/domino-coupled-directions.md | 115 ++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 + 2 files changed, 119 insertions(+) create mode 100644 docs/uncertainty/domino-coupled-directions.md diff --git a/docs/uncertainty/domino-coupled-directions.md b/docs/uncertainty/domino-coupled-directions.md new file mode 100644 index 000000000..e6c329a53 --- /dev/null +++ b/docs/uncertainty/domino-coupled-directions.md @@ -0,0 +1,115 @@ +# Domino coupled-parameter direction diagnostic + +This is a Stage B investigation of numerical exploration after the [population-size comparison](domino-budget-sensitivity.md) failed to establish stable predictions. +It evaluates fixed directions under the original point-start target; it does not produce a posterior or an agent result. +The original joint-state inference problem remains unresolved. + +## Question and controlled inputs + +Could coordinated changes to the five dynamics parameters reach plausible candidates that separate coordinate changes cannot reach? +The diagnostic uses the two completed 128-particle point-start populations, with numerical seeds 100 and 101. +The frozen simulator, original parameter prior, first-observation state, observation model and 64 fitting actions are unchanged. +No reserved future observations select the anchors or directions. + +Eight equally spaced weighted quantiles select anchor rows from each population, retaining duplicate rows if selected. +For each anchor, two ordered pairs of distinct donor rows are sampled uniformly from its own population, excluding the anchor row. +The donor difference is scaled by 0.5 or 1.0 and receives the same fixed uniform jitter in [-0.001, 0.001] in unit-prior coordinates. +For each displacement, the audit evaluates the joint five-coordinate move and the five separate single-coordinate moves from the same anchor. +The proposal random seed is 931705. +All 384 candidates are retained in the report, including the 70 outside the unit box, which receive zero acceptance without a native rollout. + +The audit first requires exact reproduction of all 16 archived anchor states and likelihoods, plus serial/parallel equality for four anchors. +At the final temperature, the original parameter prior is uniform in these coordinates and the removed initial-state factor is constant. +The score proxy is therefore `min(1, exp(candidate_log_likelihood - anchor_log_likelihood))`. +Expected squared jump is that proxy times the squared displacement in unit-prior coordinates. +The report compares each joint proposal with the mean across its five scalar proposals, making the per-proposal calculation explicit. +It does not compare against five sequential accepted coordinate updates. + +These fixed-direction measurements do not establish detailed balance for an interacting ensemble sampler, posterior accuracy, or improvement in planning. +Any resulting sampler change would need its own invariant-target reference checks and matched inference comparisons. + +## Execution and evidence + +Bundle: `logs/uncertainty_domino_coupled_directions_20260913/`. +The manifest pins both source reports and checkpoints, the archived worker and setup files, every anchor, and every proposed displacement. +The driver uses the original snapshot preparation and target worker without editing them. +Job `22681514` runs on `mit_preemptable`, node1412, with four CPUs, 20 GB and a 30-minute allocation. +The independent summary reader checks score arithmetic, paired displacements, all proposal groups, source hashes and the evaluation count. +Job `22681514` completed in 156 allocation seconds (four CPUs), with 150.426 measured driver seconds. +It performed 334 native target evaluations, including 20 anchor checks, for 21,376 simulated fitting actions. +All 16 archived anchors reproduced exactly; the four serial/parallel comparisons were exact. +The reader verified all 384 proposal records and 64 matched direction groups and rejected five corrupted report variants. + +## Broad-direction result + +Expected squared jumps below are per proposed evaluation, including out-of-box rejection, in squared unit-prior coordinates. +The scalar column averages five alternative one-coordinate proposals from the original anchor, not a sequential sweep. + +| Numerical seed | Direction scale | Joint, all coordinates | Scalar, all coordinates | Joint excluding restitution | Scalar excluding restitution | +| --- | ---: | ---: | ---: | ---: | ---: | +| 100 | 0.5 | 3.553e-8 | 0.00476847 | 3.361e-8 | 2.986e-5 | +| 100 | 1.0 | 0.0155905 | 0.0190031 | 0.00175850 | 1.987e-5 | +| 101 | 0.5 | 1.503e-8 | 0.00431245 | 1.825e-9 | 5.933e-7 | +| 101 | 1.0 | 2.287e-9 | 0.0130366 | 2.291e-10 | 0.000136446 | + +Joint proposals exceed their corresponding five-scalar mean expected jump in only one of 64 directions. +However, all 50 in-box restitution-only proposals have exactly unchanged likelihood, so movement in that coordinate dominates the aggregate scalar mean. +This makes aggregate movement a poor proxy for movement in the remaining parameters. +Excluding restitution, the seed-100 scale-1 joint group has a larger mean jump than the scalar alternatives, but the effect does not recur in the other three groups. +These observations do not support deploying a broad donor-difference proposal as a general repair. +They also do not rule out useful coupled moves at smaller scales or other parameter directions. + +## Smaller-scale follow-up + +Job `22681592` repeats the exact anchors, donor pairs, jitter draws and archived target with scales 0.02 and 0.1. +Its separate bundle is `logs/uncertainty_domino_small_directions_20260913/`. +It retains 384 proposals, of which 376 are inside the unit box. +This isolates step scale without fitting new populations or altering the probability model. +The job completed in 178 allocation seconds (four CPUs), with 172.861 measured driver seconds. +All archive and serial/parallel checks passed again. +The report contains 396 native evaluations and 25,344 simulated fitting actions. +An independent affine-direction check confirms the same anchors, donors and jitter as the broad-scale audit. + +| Numerical seed | Direction scale | Joint, all coordinates | Scalar, all coordinates | Joint excluding restitution | Scalar excluding restitution | +| --- | ---: | ---: | ---: | ---: | ---: | +| 100 | 0.02 | 4.449e-9 | 1.087e-5 | 2.914e-9 | 7.055e-9 | +| 100 | 0.1 | 2.357e-8 | 0.000276109 | 1.630e-8 | 5.439e-9 | +| 101 | 0.02 | 2.886e-5 | 1.423e-5 | 2.766e-5 | 1.240e-6 | +| 101 | 0.1 | 0.000311976 | 0.000181571 | 0.000273650 | 3.930e-7 | + +Smaller joint moves produce more expected movement in population 101 than the scalar alternatives, including after excluding restitution. +Population 100 still barely moves, so a common smaller scale does not establish a general remedy. +The fixed jitter is not scaled with the donor difference and may disrupt narrow correlated directions. + +## Jitter isolation + +Job `22681711` repeats scales 0.02 and 0.1 with exactly the same anchors and donor pairs, removing only the additive jitter. +Its bundle is `logs/uncertainty_domino_unjittered_directions_20260913/`. +All 384 proposed points have been independently reconstructed from the archived donor rows and verified exactly. +There are 22 zero-displacement proposals; these contribute zero expected jump even if accepted and must not be interpreted as exploration. +The job completed in 177 allocation seconds (four CPUs), with 171.382 measured driver seconds. +All archive and serial/parallel checks passed, with 396 native evaluations and 25,344 simulated fitting actions. + +| Numerical seed | Direction scale | Joint, all coordinates | Scalar, all coordinates | Joint excluding restitution | Scalar excluding restitution | +| --- | ---: | ---: | ---: | ---: | ---: | +| 100 | 0.02 | 1.821e-7 | 1.120e-5 | 8.222e-8 | 1.187e-7 | +| 100 | 0.1 | 1.831e-6 | 0.000277716 | 9.560e-7 | 1.754e-7 | +| 101 | 0.02 | 1.250e-8 | 1.299e-5 | 6.593e-11 | 7.039e-9 | +| 101 | 0.1 | 3.822e-6 | 0.000196525 | 2.632e-6 | 2.037e-5 | + +Removing jitter improves expected joint movement in population 100 but reduces it in population 101 at both scales. +The earlier gain from smaller joint moves is therefore not robust to this controlled perturbation. +This is consistent with the previous irregular local-target findings, but does not identify their physical cause. +The three audits together cover 1,152 candidate proposals and 72,064 simulated fitting actions, costing 511 allocation seconds on four CPUs. +All three final report readers reproduce their complete records and reject five corrupted variants each. +No sampler or production agent was changed. + +## Next decision + +Do not select a new donor-direction kernel from these results alone. +Separate the score differences into observation channels and contact/event differences for matched nearby candidates to identify what prevents movement. +That attribution must retain the existing likelihood and all fitted observations; it must not remove inconvenient factors or select a discrepancy model using the reserved future suffix. +Any justified inference change still requires numerical reference validation and repeated physical prediction comparisons before Stage C. + +The executable audit drivers, native reports, full per-parameter summaries and reader checks are retained in their three named bundles. +Each `reader-validation.json` records the final script, manifest, native-result and summary hashes. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index bb1fd2356..5b54d8c16 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -21,6 +21,10 @@ The 128-particle joint pair fails the position and toppling-curve screens; none The 128-particle fixed-state pair fails the final-toppling screen, and two cross-budget comparisons show final-probability gaps above 0.25. The earlier fixed-state agreement did not persist across budgets, so removing initial-state uncertainty is not a validated shortcut. Both treatments remain unassessed, with inference exploration and cost unresolved. +The [coupled-direction audit](domino-coupled-directions.md) reproduces 16 archived point-state anchors exactly and evaluates 384 matched joint/scalar proposals. +Broad donor-difference moves usually have very low acceptance; all 50 supported restitution-only moves leave the likelihood unchanged, exposing a misleading source of aggregate parameter movement. +Smaller steps improve joint movement in one population while the other barely moves; removing jitter reverses which population benefits, so the three completed audits do not establish a general sampler repair. +All 1,152 proposals and native replay checks are verified; likelihood-factor attribution is the next diagnostic before selecting another inference change. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. From bf8ca2a04ba4b651442fa44e42a93a2abadc53cc Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 11:20:32 -0400 Subject: [PATCH 37/94] Record first completed guided Fan forecast --- docs/uncertainty/implementation-progress.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 5b54d8c16..cb49a2c1c 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -47,6 +47,8 @@ The new [constant-output guide calculation](constant-output-guides.md) passes 33 A prefix-only saved-history audit verifies the 30 fixture coordinates and a 3.312 mm Gaussian guide scale. The guided proposal subsequently passed density correction, original physical target and serial/parallel initialization checks. Two guided 64-particle fits and gated reserved-future forecasts are submitted, with the original prior, program and observation model retained. +The first guided fit `22678810_0` and forecast `22679493_1` have completed; archived source hashes and the forecast's exact checkpoint/coordinate checks are verified. +That fit retains one initial ancestor, and its paired fit and final comparison remain unfinished. Initial weight effective sample size is worse than the original proposal, so this guide remains an exploratory comparison rather than an improvement claim. The [guided-tempering reference](guided-tempering.md) completed 16 exact Gaussian fits; annealing the finite mixture correction reduces error in these references while preserving the final target. Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010; two matched alternative-bridge fits are running. From 2922f14167aa1421ff244579d5f679d79c242454 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 11:40:00 -0400 Subject: [PATCH 38/94] Record completed Fan budget and proposal comparisons --- docs/uncertainty/constant-output-guides.md | 32 ++++++++++++++++++++-- docs/uncertainty/fan-prefix-comparison.md | 25 +++++++++++++++++ docs/uncertainty/guided-tempering.md | 28 ++++++++++++++++++- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/docs/uncertainty/constant-output-guides.md b/docs/uncertainty/constant-output-guides.md index 226a31eaa..b0f2dd5b4 100644 --- a/docs/uncertainty/constant-output-guides.md +++ b/docs/uncertainty/constant-output-guides.md @@ -114,7 +114,7 @@ It checks every canonical scene and full-mixture correction, then tests rejectio Full guided forecasts `22678925_1` and `22678926_2` were initially queued behind that verifier and their respective completed fits. Summary `22678943` was initially dependent on the validation jobs and both forecast terminal states; this first chain was cancelled after the setup failure below. The summary reports both original and both guided populations, all available pairwise prediction disagreements, parameter mass concentration and inference/forecast costs. -No guided prediction result or adequacy conclusion is available yet. +The completed v2 prediction results are recorded below; numerical adequacy remains unestablished. ## Forecast validation recovery @@ -143,4 +143,32 @@ The replacement full forecasts are `22679493_1` and `22679494_2`, gated on those Summary `22679496` retains all planned populations and waits for the forecast terminal states. The v2 reader and submission records are in `logs/uncertainty_fan_guided_summary_v2_20260913`. These validations establish forecast reconstruction, not numerical adequacy of a completed guided posterior. -The separate [tempering comparison](guided-tempering.md) tests the initial-weight concentration problem without changing these running fits. +The separate [tempering comparison](guided-tempering.md) tests the initial-weight concentration problem without changing these fits. + +## Completed matched prediction results + +Both guided fits and their reserved-action forecasts have completed. +Summary `22679496` completed in 11 allocation seconds on one CPU and verifies all 256 positive-weight histories from the four populations, unchanged weights, native predictions and the intended physical-target contract. +The summary's plan and comparison-script hashes match the final files. +These numerical seeds use one development recording, not independent agent tasks. + +| Proposal | Numerical seed | Conditional position RMSE (m) | Goal Brier score | Fan-speed 5%, 50%, 95% quantiles | Fit allocation seconds (4 CPUs) | +| --- | ---: | ---: | ---: | --- | ---: | +| Original | 302 | 0.00631118 | 0.0424194 | 0.0662, 0.0835, 0.9137 | 5,521 | +| Original | 303 | 0.00612967 | 0.0207869 | 0.0773, 0.1099, 0.9324 | 5,568 | +| Guided | 302 | 0.00557643 | 0.0120993 | 0.8924, 0.9021, 0.9220 | 5,407 | +| Guided | 303 | 0.00540711 | 0.0300042 | 0.0728, 0.0959, 0.8316 | 5,265 | + +The guided pair has lower conditional position RMSE in both replicas, while the goal Brier score improves in one and worsens in the other. +Between the original replicas, position means differ by 2.866 mm RMS and goal-probability curves by up to 0.332124. +Between the guided replicas, those disagreements are 1.340 mm and 0.3125. +Across the four original-versus-guided pairs, position-mean disagreements range from 1.962 to 4.528 mm, and maximum goal-curve gaps range from 0.255093 to 0.5. +The substantial goal disagreement and very different empirical speed quantiles remain unresolved. +Lower position error on this suffix does not establish posterior calibration or a reliable planning improvement. + +The original empirical full-future densities each have only one supported particle; the guided densities have only two and three respectively. +All other positive-weight particles retain their weights and contribute zero to that complete-future density. +This limited support is reported separately from the position and goal summaries. +Each guided forecast performs 8,580 native actions, taking 73 and 72 allocation seconds on four CPUs. +No result in this comparison approves the physical posterior for the acting agent. +The separate [alternative-tempering comparison](guided-tempering.md#completed-six-population-comparison) is also complete and does not resolve the physical prediction instability. diff --git a/docs/uncertainty/fan-prefix-comparison.md b/docs/uncertainty/fan-prefix-comparison.md index 9f9b9382e..0440aa688 100644 --- a/docs/uncertainty/fan-prefix-comparison.md +++ b/docs/uncertainty/fan-prefix-comparison.md @@ -218,3 +218,28 @@ Independent comparison confirms the original prior and target identity are uncha The initial population contains 24 finite particles with weight effective sample size 20.19; these are initialization diagnostics, not a completed posterior. Serial initialization takes 170.43 seconds and parallel initialization 44.72 seconds, excluding the separate fixed-row comparisons and startup. The validated preflight checksum is `1d90e36965fff29d925e87fe93717f50586720cc598175836c80593c31090b11`. + +## Completed population-size comparison + +Both 128-particle fits, their forecasts and summary `22677060` have completed under the same original target as the 64-particle pair. +The summary verifies all 384 positive-weight histories across four populations, their unchanged weights and all six pairwise comparisons. +Its plan and comparison-script hashes match the report, and all four forecast hashes were checked against their current files. + +| Particles | Numerical seed | Conditional position RMSE (m) | Goal Brier score | Surviving initial ancestors | Fit allocation seconds (4 CPUs) | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 64 | 302 | 0.00631118 | 0.0424194 | 1 | 5,521 | +| 64 | 303 | 0.00612967 | 0.0207869 | 2 | 5,568 | +| 128 | 302 | 0.00685677 | 0.0493406 | 2 | 10,854 | +| 128 | 303 | 0.00546470 | 0.0399988 | 1 | 11,087 | + +The 128-particle pair differs by 5.043 mm RMS in conditional position means and by up to 0.298481 in its native goal-probability curves. +The corresponding 64-particle disagreements are 2.866 mm and 0.332124. +Across budgets, position-mean disagreements range from 2.077 to 7.317 mm and maximum goal-curve gaps range from 0.269680 to 0.549676. +Doubling the population therefore does not establish stable, budget-insensitive predictions. + +The 128-particle seed-302 empirical fan-speed distribution places 50.77% of its weight at one exact value, 0.9168676. +Its 50% and 95% quantiles consequently coincide; this is not evidence of precise identification. +The two larger populations retain five and two particles with positive complete-future density; the remaining 123 and 126 particles contribute zero without having their population weights discarded. +The larger forecasts each use 17,028 native actions, taking 110 and 114 allocation seconds on four CPUs. +The final summary takes nine seconds on one CPU without native actions. +These remain offline numerical experiments, not new agent seeds or an accepted posterior replacement. diff --git a/docs/uncertainty/guided-tempering.md b/docs/uncertainty/guided-tempering.md index 43033e166..7238b1076 100644 --- a/docs/uncertainty/guided-tempering.md +++ b/docs/uncertainty/guided-tempering.md @@ -89,4 +89,30 @@ Summary `22680197` also waits for the original guided forecasts `22679493_1` and It compares two original, two base-corrected guided and two tempered-correction populations, retaining all 15 pairwise comparisons when available. Original physical prior, data, program and output law remain fixed; the reader explicitly verifies the algebraic factorization change instead of requiring the numerical bridge identities to be identical. The reporting bundle is `logs/uncertainty_fan_tempered_summary_20260913`. -No completed tempered posterior forecast or live-agent acceptance result is available yet. +The completed physical comparison is recorded below; no live-agent acceptance result is established. + +## Completed six-population comparison + +Both alternative-tempering fits, both forecasts and summary `22680197` have completed. +The summary verifies the unchanged physical target, all 384 positive-weight histories, and all 15 pairwise comparisons across the original, base-corrected guided and tempered-correction proposals. +Its final plan and comparison-script hashes match the saved report. + +| Tempered numerical seed | Conditional position RMSE (m) | Goal Brier score | Fan-speed 5%, 50%, 95% quantiles | Fit allocation seconds (4 CPUs) | +| --- | ---: | ---: | --- | ---: | +| 302 | 0.00527377 | 0.0170186 | 0.0849, 0.9138, 0.9157 | 5,415 | +| 303 | 0.00870651 | 0.0644868 | 0.0803, 0.0869, 0.1037 | 5,387 | + +| Replica pair | Position-mean disagreement (mm RMS) | Maximum goal-probability curve gap | +| --- | ---: | ---: | +| Original proposal | 2.866 | 0.332124 | +| Base-corrected guide | 1.340 | 0.312500 | +| Tempered correction | 6.656 | 0.371003 | + +The tempered pair disagrees more than either earlier pair on these prediction summaries. +Its empirical full-future density has no supported particle in seed 302 and one in seed 303; the zero density in seed 302 is retained, not replaced with a finite value. +Each forecast uses 8,580 native actions and 76 allocation seconds on four CPUs. +The final summary takes 16 seconds on one CPU with no native actions. + +Better initial effective sample size and the successful exact Gaussian references did not translate into stable physical forecasts in this comparison. +The alternative bridge is not accepted for posterior use or deployment. +These are numerical replicas on one development recording, not agent solve-rate results. From 56d98e3c8c12f42f62a17c84dc08f8f13d06b3fd Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 11:40:00 -0400 Subject: [PATCH 39/94] Attribute Domino score instability to robot transients --- docs/uncertainty/domino-factor-attribution.md | 90 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 19 ++-- 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 docs/uncertainty/domino-factor-attribution.md diff --git a/docs/uncertainty/domino-factor-attribution.md b/docs/uncertainty/domino-factor-attribution.md new file mode 100644 index 000000000..6c045a4a5 --- /dev/null +++ b/docs/uncertainty/domino-factor-attribution.md @@ -0,0 +1,90 @@ +# Domino likelihood-factor attribution + +This Stage B diagnostic follows the [coupled-direction comparisons](domino-coupled-directions.md). +It identifies which existing likelihood factors cause large score changes without modifying the inference target, priors, program, observations or acting agent. + +## Controlled replay + +For each of the same 16 weighted anchor rows, select the finite joint-direction pair with the largest fitting-score difference between the small jittered and unjittered audits. +Replay its anchor and both candidates, giving 48 cases, plus one repeated complete replay. +Selection uses only the existing 64-action fitting prefix and its scores. +These deliberately selected contrasts are diagnostic examples, not a representative calibration sample or agent seeds. + +The original native initialization, first-observation state, action sequence and output likelihood are retained. +The instrumentation records the 65 predicted and observed frames, per-factor likelihood contributions, and native contact records at primitive-step boundaries. +Every remaining likelihood exactly reproduces its previously saved target value. +The repeated case also reproduces its complete predictions, contacts and factor decomposition exactly. + +Compute job `22681870` completed in 42 allocation seconds on four CPUs in `mit_preemptable`. +The 49 replays use 3,136 simulated actions. +Artifacts, frozen input identities and scripts are in `logs/uncertainty_domino_factor_attribution_20260913/`. +The summary verifies the complete factors, initial-observation removal and all 48 score contrasts, including identical cases with zero change. +An independent variance-form Kalman calculation verifies all 93,600 scalar time factors, with maximum absolute log-factor discrepancy 1.422e-14. +The independently checked scalar terms account for the dominant contributions below. + +## What changes the scores + +Robot proprioception and robot pose account for 59.9% to 99.7% of the sum of absolute channel contributions, with median 96.8%, for the 16 jittered-versus-unjittered contrasts. +Controlled joint-position channel 6 is the largest individual contribution in 15 contrasts; channel 5 is largest in the remaining contrast. +Its maximum predicted difference between matched candidates ranges from 0.0007623 to 0.012082 radians. +The existing proprioceptive discrepancy process has persistence 0.9 and innovation scale 0.001 per primitive step. +Large likelihood ratios from these transient joint differences therefore need not involve a different final object outcome. + +| Anchor | Numerical seed | Jittered minus unjittered log score | Robot share of absolute channel changes | Largest channel | Peak step | +| ---: | ---: | ---: | ---: | --- | ---: | +| 0 | 100 | -21.756 | 70.8% | Joint 6 | 51 | +| 1 | 100 | 44.075 | 95.4% | Joint 6 | 50 | +| 2 | 100 | 1.343 | 83.3% | Joint 5 | 57 | +| 3 | 100 | -83.775 | 99.5% | Joint 6 | 53 | +| 4 | 100 | -36.089 | 99.2% | Joint 6 | 52 | +| 5 | 100 | -53.496 | 94.8% | Joint 6 | 50 | +| 6 | 100 | 36.524 | 98.1% | Joint 6 | 49 | +| 7 | 100 | 98.293 | 98.8% | Joint 6 | 52 | +| 8 | 101 | -15.327 | 74.0% | Joint 6 | 52 | +| 9 | 101 | -9.783 | 59.9% | Joint 6 | 53 | +| 10 | 101 | 55.613 | 98.1% | Joint 6 | 54 | +| 11 | 101 | 129.646 | 99.7% | Joint 6 | 49 | +| 12 | 101 | -117.307 | 99.2% | Joint 6 | 48 | +| 13 | 101 | -26.525 | 98.1% | Joint 6 | 52 | +| 14 | 101 | -22.103 | 84.9% | Joint 6 | 52 | +| 15 | 101 | 14.113 | 67.1% | Joint 6 | 52 | + +All 16 matched pairs have the same sets of contacting body/link pairs at the recorded primitive-step boundaries. +This does not establish identical contact forces, locations, multiplicities or contact behavior inside an environment step. +Most dominant score changes occur during steps 48-54; the remaining channel-5 case peaks at step 57. +This narrows the investigation to the robot's transient response rather than establishing a categorical contact-event mismatch. + +The attribution is not evidence that joint observations should be dropped or their error scale increased. +Those observations may legitimately constrain contact parameters through the robot's response. +The joint and Cartesian channels also cannot be merged by assuming fresh forward kinematics reproduces the recorded Cartesian pose: the [earlier observation-phase audit](observation-reductions.md#cartesian-robot-pose-is-different) already disproved that shortcut. +The likelihood still needs a principled account of their actual timing and dependencies. + +## Consequence for the plan + +Neither broad nor smaller donor-direction moves provide a consistent numerical repair across the retained populations. +The new attribution explains why apparent object-level similarity does not imply similar fitting probability: the score changes mainly come from robot joint transients. +It does not establish that the current probability model is calibrated or that a particular alternative sampler will explore it adequately. + +The next numerical investigation should test local sensitivity of those transient joint predictions and distinguish a narrow, reproducible parameter constraint from irregular numerical or model behavior. +Any sensitivity-informed proposal must retain the original prior and complete likelihood and pass independent numerical reference checks before a new native inference comparison. +Keep the original fitter in control until the Stage A/B evidence requirements and later planning comparisons are met. + +## Completed local-sensitivity test + +Job `22682046` evaluates anchors 0, 7, 8 and 15 at symmetric changes of 1e-6, 1e-5 and 1e-4 in each of the five unit-prior parameters. +The 124 cases plus an exact repeated replay use 8,000 native actions and complete in 81 allocation seconds on four CPUs. +Every original anchor score reproduces exactly, and a separate scalar check verifies all 241,800 time factors with maximum absolute error 1.422e-14. +The complete results are in `logs/uncertainty_domino_transient_sensitivity_20260913/`. + +For each parameter and scale, the diagnostic estimates the central-difference vector of joint-6 predictions across the fitting prefix. +The relative vector difference is the norm of the difference divided by the larger vector norm. +For the 16 non-restitution anchor/parameter combinations, changing the scale from 1e-6 to 1e-5 gives relative derivative differences of approximately 0.847 to 1.408. +Changing from 1e-6 to 1e-4 gives differences of approximately 0.965 to 1.024. +These are large discrepancies, not stable local sensitivities suitable for an unvalidated gradient or curvature proposal. +All tested restitution derivatives remain exactly zero. +Exact repetition rules out nondeterministic evaluation as the explanation for these particular differences; it does not establish a smooth simulator map. + +The evidence does not justify simply increasing particles again, choosing a gradient proposal, or deleting the informative joint channels. +A possible next model comparison is to place explicitly modeled joint discrepancy in physical transitions, condition those transitions on exact joint readings and retain their densities once, using the existing transition-conditioning machinery. +That would be a different probability model from the current output-error law and must be labeled and validated separately, including its observation phase and unconditional future generation. +It must not be presented as a numerical sampler improvement under the unchanged target. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index cb49a2c1c..426ad5003 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -24,13 +24,16 @@ Both treatments remain unassessed, with inference exploration and cost unresolve The [coupled-direction audit](domino-coupled-directions.md) reproduces 16 archived point-state anchors exactly and evaluates 384 matched joint/scalar proposals. Broad donor-difference moves usually have very low acceptance; all 50 supported restitution-only moves leave the likelihood unchanged, exposing a misleading source of aggregate parameter movement. Smaller steps improve joint movement in one population while the other barely moves; removing jitter reverses which population benefits, so the three completed audits do not establish a general sampler repair. -All 1,152 proposals and native replay checks are verified; likelihood-factor attribution is the next diagnostic before selecting another inference change. +All 1,152 proposals and native replay checks are verified. +The [factor attribution](domino-factor-attribution.md) now reproduces 48 matched cases: joint channel 6 dominates 15 of 16 selected score changes, with median 96.8% of absolute channel changes coming from robot observations. +All sampled boundary contact-pair sets match between those pairs, and an independent Gaussian calculation verifies 93,600 scalar time factors. +The subsequent 124-case local-sensitivity test also completes: non-restitution joint-response derivatives vary strongly across 1e-6 to 1e-4 parameter changes, while 241,800 additional scalar factors verify independently; a simple gradient proposal is not supported. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. The larger native preflight `22676726` matches the original prior and target identity and verifies exact serial/parallel initialization at 128 particles. -Both larger fits `22676775_0` and `_1` are running, with same-suffix forecasts `22677013_0` and `22677014_1` queued behind their individual fits. -The cross-budget reader passes compute validation `22677052`, reproducing 128 complete baseline histories and rejecting seven incomparable or corrupted inputs; finite summary `22677060` will retain all six replica pairs. +Both larger fits, forecasts and cross-budget summary `22677060` are complete, verifying 384 complete histories and all six pairs. +The larger pair still differs by 5.043 mm in position means and up to 0.298481 in goal-probability curves, with cross-budget goal gaps as large as 0.549676; numerical adequacy remains unestablished. The [isolated carried-center comparison](carried-center-comparison.md) has completed both arms and report `22676128`. All six fits on identical data select the same parameters and produce exactly identical complete predictions. @@ -47,12 +50,14 @@ The new [constant-output guide calculation](constant-output-guides.md) passes 33 A prefix-only saved-history audit verifies the 30 fixture coordinates and a 3.312 mm Gaussian guide scale. The guided proposal subsequently passed density correction, original physical target and serial/parallel initialization checks. Two guided 64-particle fits and gated reserved-future forecasts are submitted, with the original prior, program and observation model retained. -The first guided fit `22678810_0` and forecast `22679493_1` have completed; archived source hashes and the forecast's exact checkpoint/coordinate checks are verified. -That fit retains one initial ancestor, and its paired fit and final comparison remain unfinished. +Both guided fits, forecasts and the four-population summary `22679496` are complete, with all 256 positive-weight histories verified. +The guided pair lowers position errors but retains a 0.3125 maximum goal-curve disagreement and very different empirical fan-speed quantiles; posterior adequacy remains unestablished. Initial weight effective sample size is worse than the original proposal, so this guide remains an exploratory comparison rather than an improvement claim. The [guided-tempering reference](guided-tempering.md) completed 16 exact Gaussian fits; annealing the finite mixture correction reduces error in these references while preserving the final target. -Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010; two matched alternative-bridge fits are running. -The short native tempered forecast and independent raw-factor/coordinate verification now pass, with full forecasts and a six-population comparison gated on the completed fits. +Its native Fan preflight preserves the target and serial/parallel state, with initial effective sample size 15 instead of 2.010. +Both alternative-bridge fits and forecasts are now complete: summary `22680197` verifies all six populations and 15 pairs, but the tempered pair differs by 6.656 mm in position means and up to 0.371003 in goal curves. +The tempered seed-302 empirical complete-future density has zero supported particles; this is retained as a failure of that density estimate, not converted to a finite value. +The completed Fan budget, guide and tempering comparisons do not close Stage B. The first guided forecast snapshot failed before native actions because it lacked the future-likelihood method. The corrected snapshot reuses the completed fixture and passes full native replay. A controlled same-node audit attributes the 1.11e-16 cross-machine coordinate mismatch to NumPy instruction dispatch; disabling the extra AVX-512 dispatch options restores exact coordinates for the saved fixture. From 5abcae5d3ae9f04fb3cd6ff44c2fc51633e451c4 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 12:05:49 -0400 Subject: [PATCH 40/94] Validate Domino joint-transition model and launch matched inference --- docs/uncertainty/domino-joint-transition.md | 95 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 + 2 files changed, 98 insertions(+) create mode 100644 docs/uncertainty/domino-joint-transition.md diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md new file mode 100644 index 000000000..a78efb175 --- /dev/null +++ b/docs/uncertainty/domino-joint-transition.md @@ -0,0 +1,95 @@ +# Domino joint-transition discrepancy comparison + +This is a different probability model under the [simplification proposal](simplification-proposal.md), not a sampler change under the previous target. +The [transient diagnostics](domino-factor-attribution.md) found that robot joints dominate selected score changes and that their local sensitivities are irregular. +This comparison tests physical joint-transition discrepancy instead of the previous joint output-error process. +The old model and all of its results remain the controls. +No acting agent uses this alternative. + +## Declared model + +After each native primitive action, let the simulator predict controlled joint positions `q_native`. +The alternative draws each next physical joint position independently from `Normal(q_native, 0.001**2)` in that joint's coordinate units. +For these Fetch configurations, that means radians for the seven revolute joints and meters for the two finger joints. +This is a declared transition-error scale, not sensor noise or a numerical tolerance. +Joint velocities are retained, and positions are neither wrapped nor clipped. +The Gaussian law does not itself certify mechanical bounds, collision support or an accurate contact-force model. + +During fitting, the exact observed joint positions analytically determine the transition innovations. +Their Gaussian densities remain in the likelihood once per joint per primitive step. +The corrected physical positions propagate into the following native step. +Every other body state, event, attachment and program memory continues through the simulator. + +The nine former Gaussian AR output factors for joint positions are removed in this model. +Their exact sensor checks remain and must agree with the corrected joint readings. +All other output factors, sensor measurements and the checked finger readout remain unchanged. +Adding the former joint output factors after conditioning would count another density at zero residual; the audit explicitly detects that extra normalization factor. + +The robot Cartesian fields `x`, `y`, `z`, `roll`, `tilt` and `wrist` retain their native predicted cached-link values from before the correction. +They are never replaced by observed Cartesian values. +This follows the [verified native observation phase](transition-discrepancy.md#preserve-the-native-robot-observation-phase), rather than assuming fresh forward kinematics reproduces historical observations. + +The original parameter prior, simulator program, initial point state and 64-action fitting prefix are fixed. +The first-observation factor is unchanged; the parameter-independent initial factor removed by the point-start comparison remains parameter-independent here. +The inference identity includes both the new transition law and the remaining output model. +These point-start fits do not establish an uncertain-initial-state posterior. + +## Conditional scoring and future generation + +Future generation draws the joint innovations without access to future observations. +The existing normalized output model then generates readings, including temporally correlated discrepancy and exact derived displays. + +Future-density evaluation is a separate computation. +Exact future joint readings determine the corresponding innovations and physical continuation, and their densities are retained. +Remaining future observations enter the output likelihood, not the physical correction. +A density-evaluation trajectory conditioned on future joints must never be reused as an unconditional goal prediction. +The joint-only transition law introduces no unobserved direction integral when every corrected joint coordinate is observed exactly. +Other latent initial-state uncertainty still belongs in the posterior population. + +## Native validation + +Job `22683003` completed in 91 allocation seconds on four CPUs in `mit_preemptable`, using 9,152 native actions. +Its bundle is `logs/uncertainty_domino_joint_transition_20260913/`. +The two observation-module extensions used for future generation preserve the old target exactly at four archived anchors, each checked twice. + +All 124 fixed sensitivity cases have finite scores under the alternative model. +One complete case repeats exactly, including predictions, corrected joints and transition factors. +The independent reader checks 74,304 Gaussian joint factors to maximum absolute log-density error 3.553e-15. +It verifies complete joint coverage and the sum of transition and remaining-output likelihoods. +The audit also rejects a deliberately inconsistent joint reading and verifies the extra constant that would arise from incorrectly retaining the old joint output factors. + +Four generated 16-action futures reproduce exactly when their generated readings are passed back through conditional density evaluation. +This includes their complete physical predictions and transition/output densities, not just final states. +These are fixed-candidate component checks, not calibrated forecasts or agent seeds. +The largest angular correction in these records is 0.014080 radians at the wrist-roll joint; the largest finger correction is 0.002393 meters. + +The native pre-correction joint-response derivatives remain irregular across perturbation scales 1e-6, 1e-5 and 1e-4. +Corrected joint positions equal their observations by construction and are not used to claim zero parameter sensitivity. +The component checks therefore justify a labeled inference experiment, not a claim that the model fixes the numerical problem. + +## Bounded inference comparison + +Array `22683118` runs two new replicas, seeds 100 and 101, from the original prior. +Both startup checks pass and both runs have initialized their 64-particle populations. +Each uses the same 32 cubic-spaced temperatures, eight moves per temperature, five single-coordinate blocks, scale 0.05, 50/50 local/full-range proposal mixture and 16,448-evaluation budget as the earlier 64-particle point-start comparison. +The only intended statistical change is the explicitly identified joint-transition model. +Each allocation requests four CPUs, 20 GB and four hours on node1412 in `mit_preemptable`. +Complete-stage checkpoints preserve weights, random state, ancestry and cumulative evaluation counts. +The bundle is `logs/uncertainty_domino_joint_transition_fit_20260913/`. + +The full-suffix fixture tests generation and density evaluation over all 97 reserved actions before posterior forecasts are trusted. +It also checks that changing future object-position readings while retaining future joint readings does not change the density evaluator's physical history. +The separate bundle is `logs/uncertainty_domino_joint_transition_future_20260913/`. +The first job, `22683253`, completed its native work but failed a reporting assertion that compared in-memory tuples with lists loaded from JSON. +The corrected check canonicalizes that representation while retaining exact numeric equality and saves completed native records before subsequent assertions. +Replacement `22683284` passes in 37 allocation seconds on four CPUs, using 1,771 native actions; the earlier 38-second allocation and its 1,771 actions remain additional compute cost. + +All four generated complete futures replay exactly when their generated readings are used for conditional scoring. +Their fitting prefixes and both recorded-suffix density cases match the archived preflight exactly. +Changing future object-position readings while retaining future joints leaves the evaluator's physical predictions and transitions unchanged. +The independent reader verifies all six retained complete histories and 8,694 joint factors to maximum absolute log-density error 8.882e-16, and rejects altered prefix values. +Both fixed-candidate recorded-suffix densities are finite. +This establishes a supported, reproducible full-suffix calculation for those candidates, not a comparison of posterior predictions or model evidence. + +After the fits complete, their forecasts must preserve original posterior weights, separate unconditional predictions from conditioned density trajectories, and assess numerical replication and simulator cost. +No fit or forecast in this experiment has been approved for planning or deployment. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 426ad5003..fdf53e003 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -28,6 +28,9 @@ All 1,152 proposals and native replay checks are verified. The [factor attribution](domino-factor-attribution.md) now reproduces 48 matched cases: joint channel 6 dominates 15 of 16 selected score changes, with median 96.8% of absolute channel changes coming from robot observations. All sampled boundary contact-pair sets match between those pairs, and an independent Gaussian calculation verifies 93,600 scalar time factors. The subsequent 124-case local-sensitivity test also completes: non-restitution joint-response derivatives vary strongly across 1e-6 to 1e-4 parameter changes, while 241,800 additional scalar factors verify independently; a simple gradient proposal is not supported. +The separate [joint-transition model comparison](domino-joint-transition.md) now passes 124-case probability/replay checks and full 97-action future-generation/density validation. +It retains every joint transition density and cached-link observation timing while removing only the former joint AR output factors as an explicit model change. +Two fresh 64-particle point-start inference replicas, `22683118_0` and `_1`, are running under the original prior and unchanged sampler settings; their posterior prediction comparison remains pending. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. From 298523d1d2d97c17a7459177d86605e1f7878245 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 12:24:30 -0400 Subject: [PATCH 41/94] Validate weighted Domino transition forecasts and queue comparison --- docs/uncertainty/domino-joint-transition.md | 37 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 ++++ 2 files changed, 43 insertions(+) diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md index a78efb175..fc7ef0501 100644 --- a/docs/uncertainty/domino-joint-transition.md +++ b/docs/uncertainty/domino-joint-transition.md @@ -93,3 +93,40 @@ This establishes a supported, reproducible full-suffix calculation for those can After the fits complete, their forecasts must preserve original posterior weights, separate unconditional predictions from conditioned density trajectories, and assess numerical replication and simulator cost. No fit or forecast in this experiment has been approved for planning or deployment. + +## Checkpoint-driven posterior forecasts + +The posterior forecast adapter is implemented in `logs/uncertainty_domino_transition_posterior_forecast_20260913/`. +A short native sampler fixture, `22683423_0`, completed with 16 particles, two temperatures and at most 48 evaluations. +It is a checkpoint and forecast integration fixture, not an assessed posterior or a new agent seed. +Restoring its completed checkpoint reproduces the complete saved sampler result without another target evaluation. + +For each positive-weight fitted particle, the adapter generates four unconditional futures in each of two independent random-number banks. +It retains the original particle weight, divided only by the number of draws within each bank. +Position means and variances integrate the scalar output-error process conditional on the fitting prefix; native toppling events come from the generated physical trajectories. +A separate history conditions on the actual future joint readings solely to evaluate the recorded suffix density. +Those future-conditioned histories never contribute to unconditional position or toppling predictions, and suffix observations never reweight the fitted particles. +Zero particle weights and zero predictive densities remain explicit. + +Forecast fixture `22684064_0` completed in 150 allocation seconds on four CPUs, with 23,506 native actions including two exact complete-history repeats. +Independent verification `22684436` completed in 31 allocation seconds on one CPU. +It reads all 144 saved histories, checks the complete checkpoint population and original weights, reconstructs the scalar Gaussian filtering and forecast moments, and recomputes all weighted summaries. +It independently checks 208,656 Gaussian joint factors, with maximum absolute log-factor discrepancy 8.882e-16. +Deliberately changed weights, generation/density roles, prefix values, forecast moments, future joints and joint-density factors are all rejected. +These checks establish artifact consistency and arithmetic for the fixture, not statistical adequacy of its short fit. + +The two full forecasts are queued as `22684522_1` and `22684523_2`, each dependent on successful completion of its corresponding fit. +Each checks the frozen fixture-validation gate before generating predictions and runs the independent artifact verifier after finishing. +The declared forecast budget is two banks of four draws per positive-weight particle plus one separate density history per particle, using four CPUs for at most 45 minutes. +With all 64 weights positive, this requires 93,058 native actions including the two complete-history repeats. +Between-bank disagreement measures future-simulation Monte Carlo variability; it does not measure posterior uncertainty or establish convergence of parameter inference. + +The paired report retains the earlier exploratory screens: 2.5 mm RMS disagreement in conditional position means, 0.20 maximum toppling-curve gap and 0.15 maximum final toppling gap. +It compares the new replicas against the original 64-particle point-start replicas while requiring identical prior, fitting data, program and sampler settings. +The joint-discrepancy law is explicitly required to differ. +Missing forecasts or verification reports leave the comparison incomplete. +Uncertain initial-state inference, wider development coverage, acceptable inference cost and closed-loop acceptance remain separate requirements. + +Comparison-reader validation `22684684` reproduces identical fixture summaries, detects injected position and event regressions, rejects misaligned coordinates and preserves an incomplete status when either new forecast is missing. +Finite report job `22684725` is queued after both full forecast jobs reach terminal states; it is not a notification monitor. +The bundle's `verified-inputs.json` pins the tested scripts, fixture reports and job mapping. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index fdf53e003..f638a843d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -31,6 +31,10 @@ The subsequent 124-case local-sensitivity test also completes: non-restitution j The separate [joint-transition model comparison](domino-joint-transition.md) now passes 124-case probability/replay checks and full 97-action future-generation/density validation. It retains every joint transition density and cached-link observation timing while removing only the former joint AR output factors as an explicit model change. Two fresh 64-particle point-start inference replicas, `22683118_0` and `_1`, are running under the original prior and unchanged sampler settings; their posterior prediction comparison remains pending. +The checkpoint-driven forecast adapter now passes a native 16-particle integration fixture and an independent reader over all 144 saved histories. +The reader verifies original weights, separates unconditional generation from future-conditioned density evaluation, and rejects six deliberately corrupted inputs. +Full forecasts `22684522_1` and `22684523_2` are queued behind their respective fits, with fixture validation required before execution and complete artifact verification afterward. +These remain offline diagnostics; neither the short fixture nor a completed sampler is an approved posterior. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. Position errors are lower than legacy on this recording, but goal-probability curves disagree by up to 0.33212, and each empirical full-future density depends on only one supported particle. @@ -45,6 +49,8 @@ This supplies active-carry repeated-data coverage and supports an immutable orig The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. The long-prefix Balloons fits remain ongoing. +After Slurm confirmed `22671041_0` timed out, its seed-300 fit continued as `22684078_0` from completed stage 28 and 15,107 evaluations under the same frozen runtime, prior and numerical budget. +Its two previous eight-hour allocations remain part of the cost; seed 301 continues in its existing allocation. Both full-recording Fan fits and their verified report have completed, but their narrow empirical speed distributions do not overlap and each retains one initial ancestor; they remain unassessed. The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has completed all 512 fits. All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. From d0c70680eca5b10e14f46554ceed23b507c5b474 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 12:38:47 -0400 Subject: [PATCH 42/94] Audit historical Bridge and Boil dynamics as separate inference controls --- docs/uncertainty/historical-model-controls.md | 85 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 + 2 files changed, 89 insertions(+) create mode 100644 docs/uncertainty/historical-model-controls.md diff --git a/docs/uncertainty/historical-model-controls.md b/docs/uncertainty/historical-model-controls.md new file mode 100644 index 000000000..dfc3be847 --- /dev/null +++ b/docs/uncertainty/historical-model-controls.md @@ -0,0 +1,85 @@ +# Historical Bridge and Boil model controls + +This is an additional Stage B development audit under the [simplification proposal](simplification-proposal.md). +It does not replace the frozen incomplete-model controls or change the acting agent. +No parameter fitting, posterior assessment or new agent run occurs here. + +## Why these controls are needed + +The saved Bridge and Boil programs from all three original noisy-sweep seeds declare no learnable parameters. +The audit covers thirteen versioned files across the six runs, including later learning cycles. +Their hidden-dynamics hooks are empty or diagnostic-only. +Those artifacts remain useful tests of explicit model inadequacy and parameter-free dispatch, but cannot measure an advantage of one parameter estimator over another. + +Earlier saved noisy runs contain learned programs with explicit filling/heating and glue/bond mechanisms. +This audit freezes the seed-0 cycle-000 version-002 program from `boil-agent_continual_noise_p12_r07` and from `bridge-agent_continual_cross_noise_on`. +The source programs were created on September 8; the target observations are the already-frozen September 10 first-training recordings used by the existing controls. +Selection is based on the presence of the missing dynamics, before evaluating transfer errors. +Defaults, parameter bounds, model memory and command rules are retained from those historical artifacts. +The source program and its chosen bounds are learned artifacts, not evidence of an original prior predating their source data. +A subsequent posterior experiment must declare and freeze its prior separately. + +The native replay uses the same frozen `b09217bb3` runtime and public noisy initial state as the original incomplete-model controls. +Historical rules run through their existing post-step interface: feature updates affect reported predictions, while emitted physics commands affect subsequent native steps. +This is a historical-model control, not a conversion of those programs to the current subclass contract or evidence of conversion parity. +The audit retains complete trajectories and hidden model memory and never injects later observations into the rollout. + +Each domain has five fresh replays: the original no-op, historical defaults twice, and one parameter at its lower and upper declared bounds. +The perturbed parameter is `fill_rate` for Boil and `bond_dist` for Bridge. +The original no-op trajectory must match its archived control exactly; the two default histories, commands and model memories must match each other exactly. +Errors are reported for all post-action readings and for the suffix after action 64. +The suffix is a diagnostic partition; there is no fit on its preceding prefix in this audit. + +## Boil outcome + +Job `22685010_1` completes all five 264-action replays in 30 allocation seconds on one CPU, using 1,320 native actions. +Independent reader `22685333` verifies the archived control, complete repetitions, all scalar error calculations, parameter variants and exact-output contradictions. + +| Reading | No-op all-action RMSE | Historical defaults all-action RMSE | Historical defaults suffix RMSE | +| --- | ---: | ---: | ---: | +| Water volume | 0.7600 | 0.08506 | 0.08976 | +| Bubbling level | 0.4260 | 0.06786 | 0.06695 | +| Spilled level | 0.07104 | 0.07104 | 0.06909 | + +The declared sensor standard deviation for these three channels is 0.07. +The transferred filling/heating program supplies substantially better scalar predictions on this recording without refitting. +Changing fill rate to either declared bound changes 210 predicted frames; the upper bound produces large water-volume error rather than being silently clipped to an acceptable result. + +However, the full sensor-only replay still contradicts eighteen exact observed channels. +Joint and related robot readout mismatches begin at action 9; the faucet and its switch disagree on their on/off state at two actions beginning at action 55. +Lower scalar error therefore does not establish a supported complete likelihood or justify dropping the exact observations from inference. +The program is a useful candidate for a separately declared physical-discrepancy comparison, not an approved posterior model. + +## Bridge outcome + +The first Bridge job, `22685010_0`, failed after 55 allocation seconds because its model memory contains a set and tuple-keyed dictionary that the report serializer could not encode. +This was a diagnostic reporting failure, not a failed agent seed or a physical replay contradiction. +The original script and failed output are preserved. +The replacement serializer preserves dictionaries, sets, tuples and lists with explicit type tags, including tuple keys; a JSON roundtrip verifies those distinctions. +The replacement Bridge job, `22685141_0`, completes five 1,186-action replays in 133 allocation seconds on one CPU, using 5,930 native actions. +The failed allocation remains additional cost; its last saved progress report is not a complete accounting of native work. + +| Replay | Glue reading mismatches across all faces and actions | Steps emitting attachment commands | +| --- | ---: | ---: | +| No-op | 2,272 | 0 | +| Historical defaults | 3,695 | 347 | +| Lower bond-distance bound | 4,042 | 0 | +| Upper bond-distance bound | 3,695 | 347 | + +The learned program makes six glue channels vary and can emit attachment commands, but its default glue predictions transfer poorly. +The lower bond-distance bound changes 347 complete predicted frames and removes the attachment commands; the upper bound leaves the default history unchanged. +These results expose parameter sensitivity and a flat region, without showing that fitting can recover the observed mechanics. +All exact-output contradictions remain visible; meaningful dynamics alone do not establish model adequacy. +Independent reader `22685371` verifies all five Bridge histories in 23 allocation seconds and retains contradictions in 26 exact observed channels for the default transferred model. + +## Artifacts and next decision + +The frozen bundle is `logs/uncertainty_bridge_boil_model_controls_20260913/`. +Its plan records the thirteen-file inventory, selected source programs, target data, runtime, variants and original-control hashes. +The reports preserve generated observations and memory; independent verification reloads the original public evidence and checks both noisy errors and exact-channel mismatches. +The existing no-op controls remain unchanged. +The bundle's `verified-inputs.json` pins the completed reports, verification artifacts and script versions, including the original reporting failure. + +Use Boil's improved scalar dynamics to define a supported full observation/transition model before attempting a parameter-inference comparison. +For Bridge, first resolve whether model revision or a declared discrepancy model can explain the glue and bond transitions; do not assume a larger sampler will repair the transferred program. +Neither audit completes Stage B, resolves uncertain initial-state inference or permits retiring the incumbent estimator. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index f638a843d..840125693 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -71,6 +71,10 @@ The first guided forecast snapshot failed before native actions because it lacke The corrected snapshot reuses the completed fixture and passes full native replay. A controlled same-node audit attributes the 1.11e-16 cross-machine coordinate mismatch to NumPy instruction dispatch; disabling the extra AVX-512 dispatch options restores exact coordinates for the saved fixture. Strict physical verification remains pinned to the validated source runtime; this does not certify general cross-hardware trajectory replay. +The [historical Bridge/Boil model audit](historical-model-controls.md) now expands the incomplete-model analysis beyond the original parameter-free artifacts. +All thirteen saved version files across the six original sweep runs remain parameter-free; two earlier learned programs are therefore tested as separate frozen transfer controls. +Boil's transferred filling/heating dynamics improve scalar errors, while Bridge's transferred glue dynamics produce more exact glue-reading mismatches than its no-op control. +Both complete replays retain exact-output contradictions, so neither is an approved complete probability model or a replacement posterior. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. From 0e2e614f1cd8730c7e1d8b264dbe6eacf96c54fc Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 13:06:17 -0400 Subject: [PATCH 43/94] Validate Boil joint discrepancy and prefix fixture-state controls --- .../boil-transition-and-initial-state.md | 94 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 +- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/boil-transition-and-initial-state.md diff --git a/docs/uncertainty/boil-transition-and-initial-state.md b/docs/uncertainty/boil-transition-and-initial-state.md new file mode 100644 index 000000000..909796ffd --- /dev/null +++ b/docs/uncertainty/boil-transition-and-initial-state.md @@ -0,0 +1,94 @@ +# Boil transition discrepancy and initial fixture state + +This extends the [historical-model control](historical-model-controls.md) under Stage A/B of the [simplification proposal](simplification-proposal.md). +The learned filling/heating program, parameter defaults and September 10 training recording stay fixed. +The acting agent and original parameter-free controls remain unchanged. +All corrected trajectories below condition on recorded joints throughout the episode; they are likelihood diagnostics, not unconditional forecasts. + +## Joint correction alone does not resolve the failure + +The model applies the same explicit joint-transition construction tested in [Domino](domino-joint-transition.md). +After each native action, each controlled physical joint has a Gaussian transition centered at its native prediction with standard deviation 0.001 in that joint's coordinate units. +Exact observed joints determine the corrections, while their normalized densities remain in the likelihood once. +Native joint velocities and the pre-correction cached Cartesian robot fields remain unchanged. +Joint AR output-error factors are absent from corrected trajectories; the other declared position/orientation discrepancies, sensor factors and checked finger readout remain. +Switch observations retain their exact semantics. + +Native job `22685601_0` completes six 264-action replays in 75 allocation seconds on one CPU, using 1,584 native actions. +The uncorrected no-op and historical-default histories reproduce their archived controls exactly. +Two corrected default histories repeat exactly, including their model memory and all density factors. +Varying the fill rate changes the learned scalar dynamics but leaves the physical switch mismatch unresolved. + +| Source or replay | Faucet turns on at action | Faucet turns off at action | Mismatched faucet readings | +| --- | ---: | ---: | ---: | +| Recording | 56 | 121 | 0 | +| Uncorrected historical model | 55 | 122 | 2 | +| Joint-corrected historical model at original point state | 55 | No switch-off through 264 | 145 | + +The faucet object and its switch expose the same event, so the two 145-count witnesses are duplicate readouts of one physical mismatch, not 290 independent events. +The complete likelihood is zero in all six cases. +The output likelihood is finite through action 54 and first becomes zero at action 55. +Independent verification `22685666` checks 9,504 Gaussian joint factors to maximum absolute log-factor discrepancy 8.882e-16, recomputes the complete output likelihood, and rejects four deliberately corrupted inputs. +It takes 79 allocation seconds on one CPU. + +The largest correction is 0.016831 radians in an angular joint and 0.008100 meters in a finger joint, both at action 56. +These are physical interventions, not harmless numerical adjustments. +Changing the Gaussian scale alone cannot alter this conditioned path: the exact observed positions determine every correction at any positive scale. +The scale changes density weights, not the missed switch-off for the fixed starting state. + +## A control for cache and native-reset effects + +Job `22685736_0` separates three operations while retaining the same starting state and historical program. +It completes six full histories in 34 allocation seconds on one CPU, using 1,584 native actions. + +| Intervention | Native joint-reset calls | Predicted frames different from unmodified historical replay | +| --- | ---: | ---: | +| Refresh the observation cache only | 0 | 0 | +| Reset every joint to its identical native position and velocity | 2,376 | 0 | +| Repeat those identical resets in a fresh world | 2,376 | 0 | +| Correct joints to recorded positions | 936 | 256 | + +Complete predictions, commands and model memory match for both identity-reset runs and the cache-only run. +The recorded-joint run reproduces the earlier failed corrected history exactly. +An independent JSON reader checks all six histories and every intervention, and rejects altered identity positions, reset counts and cache predictions. +This isolates the failure to changed joint positions among these interventions; it does not establish general engine-checkpoint portability or identify a particular contact-force error. + +## Initial slider state versus initial fixture position + +A bounded search varies the unobserved initial faucet-slider position and velocity while preserving the exact displayed off state. +The three positions cover fractions 0, 0.25 and 0.75 of the off interval, and velocities are -0.02, 0 and 0.02 meters per second. +An additional original-state repetition makes ten histories. +The native slider range is 0 to 0.0296 meters, with the off/on boundary at 0.0148 meters. +These grid points are candidate witnesses, not samples from a declared prior. + +Job `22685891_0` completes all ten histories in 117 allocation seconds on one CPU, using 2,640 native actions. +All public initial observations remain identical, the original-state histories repeat exactly, and every candidate retains the same 145-reading faucet mismatch. +Independent reader `22686067` checks all ten histories, invariant initial readings and 23,760 Gaussian joint factors in 117 allocation seconds. +This bounded search does not prove the full initial-state prior infeasible. + +A separate point-state test estimates fixed fixture x/y coordinates using only the first 65 public observations. +It changes neither height nor orientation, verifies that each adjusted body has a fixed base, and leaves movable bodies and initial joint settings unchanged. +The two alternatives average only the faucet switch, or all four fixed faucet/burner fixtures. +The resulting switch x position is 0.027114 meters from the noisy first-frame point estimate; its y shift is 0.001085 meters. +That displacement is substantial relative to switch contact geometry. + +Job `22686011_0` completes the original point twice and both alternatives in 54 allocation seconds on one CPU, using 1,056 native actions. +Both prefix-mean alternatives eliminate the remaining exact switch witnesses and have finite complete likelihoods under the declared transition/output model. +The original point still has zero likelihood and repeats the earlier failed history exactly. +Independent reader `22686210` verifies the four histories, the first-65-frame mean calculations, unchanged non-target initial features, full observation likelihoods and 9,504 Gaussian joint factors in 62 allocation seconds. +It rejects the same four corrupted-input classes as the transition verifier. +Both supported histories reproduce every exact switch observation; Gaussian-factor roundoff is at most 3.553e-15 in this check. +The arithmetic mean is a diagnostic point estimate, not a posterior mean under the temporally correlated output-error model. +Its likelihood values are conditional on the chosen fixture state, not model evidence integrated over initial-state uncertainty. + +## Implication for the implementation + +The joint-transition model alone did not repair the point-state Boil replay. +A better fixture-state estimate supplies supported histories without changing the learned dynamics or relaxing exact switch observations. +This provides a useful starting point for explicit joint parameter/state inference and supports the plan's requirement to account for uncertain initial states. +It does not justify treating fixture positions as known, accepting a posterior, or enabling the replacement in planning. + +Next define a normalized fixture-state prior and a density-corrected proposal using the available observation prefix, retain the initial-state likelihood and original parameter prior, and test independent inference replicas and causal future predictions. +The full source and artifacts are in `logs/uncertainty_boil_joint_transition_20260913/`, `logs/uncertainty_boil_joint_reset_control_20260913/`, `logs/uncertainty_boil_slider_initial_control_20260913/`, and `logs/uncertainty_boil_fixture_pose_control_20260913/`. + +Each bundle's `verified-inputs.json` records the tested source, plans, reports and verification artifacts. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 840125693..923dff475 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -50,7 +50,8 @@ The original harness reproduction also confirms that applying fitted values to a The long-prefix Balloons fits remain ongoing. After Slurm confirmed `22671041_0` timed out, its seed-300 fit continued as `22684078_0` from completed stage 28 and 15,107 evaluations under the same frozen runtime, prior and numerical budget. -Its two previous eight-hour allocations remain part of the cost; seed 301 continues in its existing allocation. +Its two previous eight-hour allocations remain part of the cost. +Seed 301 subsequently reached a confirmed allocation timeout and continued as `22686274_1` from completed stage 24 and 13,819 evaluations, preserving its existing numerical budget and prior sixteen allocation hours. Both full-recording Fan fits and their verified report have completed, but their narrow empirical speed distributions do not overlap and each retains one initial ancestor; they remain unassessed. The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has completed all 512 fits. All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. @@ -75,6 +76,9 @@ The [historical Bridge/Boil model audit](historical-model-controls.md) now expan All thirteen saved version files across the six original sweep runs remain parameter-free; two earlier learned programs are therefore tested as separate frozen transfer controls. Boil's transferred filling/heating dynamics improve scalar errors, while Bridge's transferred glue dynamics produce more exact glue-reading mismatches than its no-op control. Both complete replays retain exact-output contradictions, so neither is an approved complete probability model or a replacement posterior. +The subsequent [Boil transition/initial-state audit](boil-transition-and-initial-state.md) verifies that joint corrections alone worsen the fixed-point switch history, while cache refresh and identity joint resets preserve every predicted frame. +Ten initial-slider candidates retain the failure, but estimating static fixture x/y positions from the first 65 observations yields two supported complete conditional histories with the same learned dynamics and exact switch checks. +These point-state cases are independently verified and supply candidates for explicit initial-state inference; they are not posterior forecasts or Stage B acceptance. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. From 48547473444495429df3bed2ea6f7b723b2ba90f Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 13:20:03 -0400 Subject: [PATCH 44/94] Add density-corrected fixture proposals for offline uncertainty inference --- docs/uncertainty/boil-fixture-proposal.md | 59 ++++++++ docs/uncertainty/implementation-progress.md | 4 + .../inference_box_guidance.py | 127 ++++++++++++++++++ .../test_inference_box_guidance.py | 106 +++++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 docs/uncertainty/boil-fixture-proposal.md create mode 100644 predicators/code_sim_learning/inference_box_guidance.py create mode 100644 tests/code_sim_learning/test_inference_box_guidance.py diff --git a/docs/uncertainty/boil-fixture-proposal.md b/docs/uncertainty/boil-fixture-proposal.md new file mode 100644 index 000000000..db3093bf2 --- /dev/null +++ b/docs/uncertainty/boil-fixture-proposal.md @@ -0,0 +1,59 @@ +# Boil fixture-state proposal + +September 13, 2026. +This extends the [supported point-state audit](boil-transition-and-initial-state.md) toward uncertain initial-state inference. +It is a restricted component experiment with the historical filling/heating parameters and all other initial-state coordinates held fixed. +It does not complete the full scene prior or establish a replacement posterior. + +## Prior and proposal are separate + +The eight coordinates are x/y for `faucet_switch`, `faucet`, `burner_switch0`, and `burner0`. +Their declared development prior is a product of uniforms with x in `[0.35, 1.15]` and y in `[1.05, 1.65]` meters. +These bounds use the public workspace with five centimeters of padding; recorded fixture centers and exact task-generator placements do not define them. +The component is conditional on the diagnostic's fixed heights, orientations, and remaining point state. +It must not be presented as the original prior over the whole physical scene. + +For each coordinate, the guide uses only the first 65 public observations and the existing constant-output likelihood helper. +It retains the declared sensor noise and AR output discrepancy with persistence 0.9, innovation standard deviation 0.005 meters, and zero initial error. +Its center is generally different from the arithmetic mean used in the earlier point-state probe. +The guide is a Gaussian truncated to the declared prior bounds. + +The proposal is a mixture of the original uniform vector with probability 0.25 and the guided vector with probability 0.75. +One mixture indicator selects the whole vector; the density is a mixture of products, not a product of coordinate-wise mixtures. +The new offline `GaussianBoxProposal` component returns both the physical candidate and `log(prior/proposal)`. +The uniform branch preserves the complete box support and bounds this importance ratio above by four. + +Every original output observation and joint-transition factor remains in the likelihood once. +Using the prefix to construct a proposal does not remove those observations or multiply their likelihood a second time. +Proposal identities change when the guide changes; the original prior identity remains fixed. + +## Native audit + +The frozen bundle is `logs/uncertainty_boil_fixture_proposal_20260913`. +The audit retains the original point, the two supported arithmetic-mean controls, and an exact repeat of the original point. +It adds the correlated-likelihood guide center and 24 stratified mixture draws: six from the uniform component and eighteen from the guided component. +All 29 cases replay the same 264 recorded actions, totaling 7,656 native actions. +These are conditional histories using the recorded exact joints throughout, not unconditional future predictions or new agent seeds. + +Each history records the requested and actual fixture positions, complete output likelihood, nine joint-transition factors per action, exact-output contradictions, and initial fixture-pair intersections. +Intersection checks describe sampled geometry without silently rejecting candidates and changing the normalized box prior. +Passing fixture-pair checks alone does not establish whole-scene feasibility: moving bodies, table contacts, attachments, orientations, and articulated state still need their own supported composition. +If later inference conditions on a feasible scene, its normalization and any parameter dependence must be handled explicitly. + +The independent reader reconstructs the 65-frame Gaussian covariance to verify the guide centers, scales, and likelihood constants. +It checks mixture densities and inverse quantiles without calling the proposal mapper, then verifies every output score and joint factor from saved histories. +Corrupted importance weights, proposal densities, joint readings, likelihood factors, exact-switch witnesses, and model identities must be rejected. +The point and repeat controls must retain the previously verified histories exactly. + +## Validation and remaining gate + +Independent quadrature tests check proposal normalization, the joint mixture law, recovery of original-prior moments, and fixed-prior posterior evidence and moments under three different guide centers. +Additional tests cover finite support endpoints, deterministic mapping, the defensive weight bound, identities, and invalid inputs. +The first compute check passed all 27 proposal/output-error tests and focused type checking; its two test-file lint findings were corrected before resubmission. +Final compute checks `22687335` completed successfully: 27 functional tests, two-file type checking and lint, and pinned formatting. +Native audit `22687337_0` and independent reader `22687339` are queued in dependency order on `mit_preemptable`. +The cancelled dependent jobs from the first lint failure performed no native actions and are setup outcomes, not failed inference trials. + +The next gate is to inspect support across the sampled fixture positions, then incorporate this component into the remaining initial-state composition with the fixed parameter prior and complete likelihood accounting. +Neither a supported draw nor a successful numerical component test establishes posterior adequacy, held-out prediction quality, or unchanged agent performance. +Production fitting, planning, and execution estimation remain unchanged. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 923dff475..ddf7a951e 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -79,6 +79,10 @@ Both complete replays retain exact-output contradictions, so neither is an appro The subsequent [Boil transition/initial-state audit](boil-transition-and-initial-state.md) verifies that joint corrections alone worsen the fixed-point switch history, while cache refresh and identity joint resets preserve every predicted frame. Ten initial-slider candidates retain the failure, but estimating static fixture x/y positions from the first 65 observations yields two supported complete conditional histories with the same learned dynamics and exact switch checks. These point-state cases are independently verified and supply candidates for explicit initial-state inference; they are not posterior forecasts or Stage B acceptance. +The [Boil fixture proposal](boil-fixture-proposal.md) now separates an explicit uniform fixture-position prior from a first-65-observation Gaussian guide, retaining the complete prior/proposal density correction. +Independent quadrature checks recover original-prior moments and posterior reference integrals under different guides. +Twenty-seven functional tests and focused static/format checks pass; the native 29-history audit and independent reader are queued. +Full initial-state composition and posterior adequacy remain open. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. diff --git a/predicators/code_sim_learning/inference_box_guidance.py b/predicators/code_sim_learning/inference_box_guidance.py new file mode 100644 index 000000000..cf1f33a2a --- /dev/null +++ b/predicators/code_sim_learning/inference_box_guidance.py @@ -0,0 +1,127 @@ +"""Normalized defensive proposals for a fixed uniform inference prior. + +Guidance changes where candidates are evaluated, not the prior or the +likelihood. The caller must retain all observation factors used to build +the guide and separately account for physical support or conditioning. +""" +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass +from typing import Tuple + +import numpy as np +from scipy.stats import truncnorm + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_data import content_digest +from predicators.code_sim_learning.inference_sampling import BoxPrior, \ + PriorPoint + + +@dataclass(frozen=True) +class GaussianBoxProposal: + """Mix a box prior with an independent truncated Gaussian guide. + + The guide has the same box support as the original prior. A single + mixture draw selects the entire vector, so the proposal density is a + mixture of products, not a product of marginal mixtures. Centers may + come from a declared observation prefix; they never redefine the + prior. + """ + prior: BoxPrior + centers: Tuple[float, ...] + scales: Tuple[float, ...] + prior_mass: float = .1 + + def __post_init__(self) -> None: + centers = tuple(float(v) for v in self.centers) + scales = tuple(float(v) for v in self.scales) + if len(centers) != len(self.prior.names) or \ + len(scales) != len(centers): + raise ValueError("Guide coordinates must match the prior") + if not 0 < self.prior_mass < 1 or any( + not math.isfinite(v) + for v in centers) or any(not math.isfinite(v) or v <= 0 + for v in scales): + raise ValueError("Finite guide centers, positive scales and a " + "strictly interior mixture mass are required") + for (lo, hi), center, scale in zip(self.prior.bounds, centers, scales): + if not all(math.isfinite((v - center) / scale) for v in (lo, hi)): + raise ValueError("Standardized guide bounds must be finite") + object.__setattr__(self, "centers", centers) + object.__setattr__(self, "scales", scales) + + @property + def digest(self) -> str: + """Identify proposal choices separately from the original prior.""" + return content_digest( + json.dumps({ + "schema": 1, + "proposal": asdict(self) + }, sort_keys=True).encode("utf-8")) + + def log_density(self, point: Tuple[float, ...]) -> float: + """Return the normalized mixture density in physical coordinates.""" + if len(point) != len(self.prior.names) or any(not math.isfinite(v) + for v in point): + raise ValueError("A finite point matching the prior is required") + if any(not lo <= v <= hi + for v, (lo, hi) in zip(point, self.prior.bounds)): + return -math.inf + gaussian_terms = [] + for value, (lo, hi), center, scale in zip(point, self.prior.bounds, + self.centers, self.scales): + term = float( + truncnorm.logpdf(value, (lo - center) / scale, + (hi - center) / scale, + loc=center, + scale=scale)) + if not math.isfinite(term): + raise ConditioningNumericalError("Guide density overflow") + gaussian_terms.append(term) + uniform = -math.fsum(math.log(hi - lo) for lo, hi in self.prior.bounds) + result = float( + np.logaddexp( + math.log(self.prior_mass) + uniform, + math.log1p(-self.prior_mass) + math.fsum(gaussian_terms))) + if not math.isfinite(result): + raise ConditioningNumericalError("Mixture density overflow") + return result + + def transform(self, unit: Tuple[float, ...]) -> PriorPoint: + """Map independent uniforms to a candidate and log(prior/proposal). + + One extra coordinate selects the mixture component. This + returned weight contains no observation likelihood and no scene- + feasibility normalizer. Those factors remain the caller's + responsibility. + """ + if len(unit) != len(self.prior.names) + 1 or any( + not math.isfinite(v) or not 0 <= v <= 1 for v in unit): + raise ValueError("Expected one unit coordinate per dimension " + "plus one mixture coordinate") + values = [] + for value, (lo, hi), center, scale in zip(unit, self.prior.bounds, + self.centers, self.scales): + if unit[-1] < self.prior_mass: + physical = lo + value * (hi - lo) + else: + physical = float( + truncnorm.ppf(value, (lo - center) / scale, + (hi - center) / scale, + loc=center, + scale=scale)) + # Exact unit endpoints denote the declared finite box endpoints. + if value == 0: + physical = lo + elif value == 1: + physical = hi + if not math.isfinite(physical) or not lo <= physical <= hi: + raise ConditioningNumericalError("Guide quantile left support") + values.append(physical) + joint = tuple(values) + uniform = -math.fsum(math.log(hi - lo) for lo, hi in self.prior.bounds) + return PriorPoint(joint, uniform - self.log_density(joint)) diff --git a/tests/code_sim_learning/test_inference_box_guidance.py b/tests/code_sim_learning/test_inference_box_guidance.py new file mode 100644 index 000000000..97882717f --- /dev/null +++ b/tests/code_sim_learning/test_inference_box_guidance.py @@ -0,0 +1,106 @@ +"""Independent quadrature checks for observation-guided box proposals.""" +import math + +import numpy as np +import pytest +from scipy.integrate import quad + +from predicators.code_sim_learning.inference_box_guidance import \ + GaussianBoxProposal +from predicators.code_sim_learning.inference_sampling import BoxPrior + + +def test_joint_mixture_density_and_normalization() -> None: + """The joint density uses one mixture indicator for the whole vector.""" + prior = BoxPrior(("x", "y"), ((-1., 2.), (.1, .8))) + guide = GaussianBoxProposal(prior, (.2, .9), (.3, .4), .2) + point = (.1, .5) + normal = 1. + for value, center, scale, (lo, hi) in zip(point, guide.centers, + guide.scales, prior.bounds): + mass = .5 * (math.erf( + (hi - center) / (scale * math.sqrt(2))) - math.erf( + (lo - center) / (scale * math.sqrt(2)))) + normal *= math.exp(-.5 * ((value - center) / scale)**2) / ( + scale * math.sqrt(2 * math.pi) * mass) + expected = .2 / (3 * .7) + .8 * normal + assert math.exp(guide.log_density(point)) == pytest.approx(expected) + integral = quad( + lambda x: quad(lambda y: math.exp(guide.log_density( + (x, y))), .1, .8)[0], -1., 2.)[0] + assert integral == pytest.approx(1., abs=1e-10) + assert guide.log_density((-1.1, .5)) == -math.inf + + +@pytest.mark.parametrize("center", [-.8, .5, 2.5]) +def test_corrected_proposal_recovers_original_prior_and_posterior( + center: float) -> None: + """Integrate both proposal branches and recover fixed-prior evidence. + + The likelihood deliberately resembles the guide. Replacing the prior + with that guide or omitting the correction changes these integrals. + """ + prior = BoxPrior(("location", ), ((-1., 2.), )) + guide = GaussianBoxProposal(prior, (center, ), (.4, ), .2) + + def expectation(power: int, with_likelihood: bool) -> float: + + def integrand(u: float, branch: float) -> float: + point = guide.transform((u, branch)) + x = point.joint[0] + likelihood = math.exp(-.5 * ((x - .3) / .6)**2) \ + if with_likelihood else 1. + return math.exp(point.log_weight) * x**power * likelihood + + return .2 * quad( + lambda u: integrand(u, 0.), 0., 1., epsabs=1e-9)[0] + .8 * quad( + lambda u: integrand(u, 1.), 0., 1., epsabs=1e-9)[0] + + assert expectation(0, False) == pytest.approx(1., abs=2e-8) + assert expectation(1, False) == pytest.approx(.5, abs=2e-8) + assert expectation(2, False) == pytest.approx(1., abs=2e-8) + for power in (0, 1, 2): + expected = quad( + lambda x, exponent=power: x**exponent * math.exp(-.5 * ( + (x - .3) / .6)**2) / 3., + -1., + 2.)[0] + assert expectation(power, True) == pytest.approx(expected, abs=2e-8) + + +def test_endpoints_determinism_support_and_identity() -> None: + """The defensive branch covers the original support with bounded weight.""" + prior = BoxPrior(("x", "y"), ((-.3, 1.1), (2., 5.))) + guide = GaussianBoxProposal(prior, (.3, 4.), (.01, .02), .1) + for branch in (0., 1.): + assert guide.transform((0., 1., branch)).joint == (-.3, 5.) + rng = np.random.default_rng(821) + for _ in range(50): + unit = tuple(float(v) for v in rng.uniform(size=3)) + point = guide.transform(unit) + assert point == guide.transform(unit) + assert point.log_weight <= -math.log(.1) + 1e-12 + other = GaussianBoxProposal(prior, (.4, 3.), (.1, .2), .3) + assert other.prior.digest == guide.prior.digest + assert other.digest != guide.digest + assert GaussianBoxProposal(prior, (.3, 4.), (.01, .02), .1) == guide + + +def test_invalid_guides_and_coordinates() -> None: + """Invalid input cannot masquerade as an ordinary rejected candidate.""" + prior = BoxPrior(("x", ), ((0., 1.), )) + for centers, scales, mass in [((0., ), (0., ), .1), + ((math.inf, ), (1., ), .1), + ((0., ), (1., ), 0.), ((0., ), (1., ), 1.), + ((0., ), (1., ), math.nan), + ((0., 1.), (1., ), .1), + ((1e308, ), (1e-308, ), .1)]: + with pytest.raises(ValueError): + GaussianBoxProposal(prior, centers, scales, mass) + guide = GaussianBoxProposal(prior, (.5, ), (.1, )) + for unit in [(), (.5, ), (.5, math.nan), (-.1, .5), (.5, 1.1)]: + with pytest.raises(ValueError): + guide.transform(unit) + for point in [(), (math.inf, ), (.1, .2)]: + with pytest.raises(ValueError): + guide.log_density(point) From fc728cb8fd87580e323b2b1749ae2d45855015d1 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 13:35:23 -0400 Subject: [PATCH 45/94] Preserve unobserved joint uncertainty and verify Boil fixture support --- docs/uncertainty/boil-articulated-prior.md | 62 ++++++++++++++++ docs/uncertainty/boil-fixture-proposal.md | 11 ++- docs/uncertainty/implementation-progress.md | 5 +- docs/uncertainty/initial-state-inventory.md | 5 ++ .../code_sim_learning/inference_joints.py | 32 +++++++++ .../test_unobserved_joints.py | 71 +++++++++++++++++++ 6 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 docs/uncertainty/boil-articulated-prior.md create mode 100644 tests/code_sim_learning/test_unobserved_joints.py diff --git a/docs/uncertainty/boil-articulated-prior.md b/docs/uncertainty/boil-articulated-prior.md new file mode 100644 index 000000000..7409e34fb --- /dev/null +++ b/docs/uncertainty/boil-articulated-prior.md @@ -0,0 +1,62 @@ +# Boil articulated initial-state component + +September 13, 2026. +This extends the [fixture-position work](boil-fixture-proposal.md) toward a complete episode initialization model. +The fixed historical Boil program and production agent remain unchanged. + +## Three nonrobot movable joints + +The public asset definitions contain one slider in each of the two switches and one revolute joint in the faucet itself. +The faucet joint is separate from the switch that controls the recorded faucet `is_on` value. +Its position and velocity are not observed by that Boolean reading. +The remaining joints in those assets are fixed links. +The native audit verifies this inventory against the instantiated runtime instead of relying only on the asset XML. + +Each joint has an explicitly declared mixture: probability 0.8 split equally between rest at its two interval endpoints, and probability 0.2 on independent uniform position and velocity. +These engineering choices do not follow from absent velocity observations. +Switch position support uses the native travel limit multiplied by the visible `switch_joint_scale`; velocity support is `[-0.02, 0.02]` meters per second. +Faucet position support uses the native revolute interval, nominally `[0, pi/2]`; velocity support is `[-0.2, 0.2]` radians per second. +The proposal reads and records the actual instantiated bounds. + +Each switch law is conditioned on its actual initial exact flag. +The probability of that event is retained once per underlying slider, even though the corresponding faucet or burner repeats the same switch state in its public features. +For the midpoint threshold and symmetric endpoint masses, the event probability is one half. +The unobserved faucet joint uses its original mixture directly and contributes no invented observation factor. + +`RestingJointPrior` now exposes an unconditional coordinate map alongside its existing Boolean-conditioning interface. +The new map preserves both resting atoms and continuous motion; unused unit coordinates remain harmless auxiliaries in a rest case. +It does not modify the old conditional map or its prior identity. +Zero-measure coordinate seams and invalid numerical inputs remain explicit errors. + +## Native validation + +Bundle `logs/uncertainty_boil_articulated_prior_20260913` freezes the component, source program, recording inputs, and job scripts. +The experiment keeps the verified first-65-frame mean fixture point and the same historical parameter values. +It draws 16 joint-state triples with seed 401 and includes the original supported point twice as a repeat control. +Each of the 18 cases replays all 264 actions, for 4,752 native actions. +These are conditional histories with the recorded robot joints used throughout, not unconditional forecasts or agent seeds. + +Before stepping, every requested joint position and velocity must match its native readback exactly, and every conditioned switch flag must match its actual observation. +The report records the original joint inventory, sampled units, original prior identities, event factors, complete trajectory likelihoods, and exact-output contradictions. +Initial joint-event factors are stored separately from the conditional trajectory likelihood and explicitly combined by the independent reader. +The reader independently reconstructs the mixture maps and rejects altered readbacks, missing event factors, and invented faucet-joint observations. +It also verifies all existing output and transition factors and both reference histories. + +The driver saves each native observation before the historical rules execute. +A separate reader will reconstruct those states and run the literal historical rules, requiring exact agreement with all saved output predictions and model memory and rejecting emitted physical commands. +This checks a possible reuse boundary for cheaper later parameter evaluations. +It does not authorize reuse for other programs or establish that the entire joint-state inference problem is solved. + +## Status and remaining composition + +Compute checks `22687962` passed 16 functional tests, focused type checking, lint, and pinned formatting. +These include independent piecewise quadrature for the original mixed prior, both pure-component limits, unchanged conditional behavior, and invalid-coordinate cases. +The first check's untyped quadrature-library call was replaced by the analytic two-node Gaussian quadrature rule; the statistical reference itself had already passed. + +Native job `22688041_0` is running on `mit_preemptable`, followed by queued independent reader `22688102` and literal-rule reader `22688171`. +The instantiated inventory confirms two sliders with URDF interval `[0, 0.296]` and the separate faucet hinge with interval `[0, pi/2]`; the effective slider interval is `[0, 0.0296]` meters. +No native result is accepted merely because these jobs have been submitted. + +The full scene model must still combine these joints with uncertain fixture height/orientation, jug pose and motion, robot nuisance state, physical support, and the fixed program's liquid/memory initialization contract. +Any feasibility conditioning must preserve its normalization and the original parameter prior. +The current fixture and articulated component studies do not replace that full composition, independent-fit agreement, held-out prediction checks, or live non-regression experiments. diff --git a/docs/uncertainty/boil-fixture-proposal.md b/docs/uncertainty/boil-fixture-proposal.md index db3093bf2..dc87c4374 100644 --- a/docs/uncertainty/boil-fixture-proposal.md +++ b/docs/uncertainty/boil-fixture-proposal.md @@ -51,9 +51,16 @@ Independent quadrature tests check proposal normalization, the joint mixture law Additional tests cover finite support endpoints, deterministic mapping, the defensive weight bound, identities, and invalid inputs. The first compute check passed all 27 proposal/output-error tests and focused type checking; its two test-file lint findings were corrected before resubmission. Final compute checks `22687335` completed successfully: 27 functional tests, two-file type checking and lint, and pinned formatting. -Native audit `22687337_0` and independent reader `22687339` are queued in dependency order on `mit_preemptable`. +Native audit `22687337_0` and independent reader `22687339` completed on `mit_preemptable`, using respectively 379 and 327 allocation seconds on one CPU. +All 18 guided random draws have finite complete conditional likelihoods and no detected initial fixture-pair intersections. +All six uniform-component draws have zero complete likelihood; two also have fixture-pair intersections. +The guide center and both previous mean controls remain supported, and the original failed point repeats exactly. +The reader verifies all 68,904 joint-transition factors, with maximum independent calculation difference 3.638e-12, and rejects all six corrupted-input classes. +Its dense Gaussian reference validates all eight guide summaries and all 24 proposal densities and inverse quantiles. +The coordinate guide standard deviation is 4.74818 millimeters under the retained temporally correlated output model. The cancelled dependent jobs from the first lint failure performed no native actions and are setup outcomes, not failed inference trials. -The next gate is to inspect support across the sampled fixture positions, then incorporate this component into the remaining initial-state composition with the fixed parameter prior and complete likelihood accounting. +These support results justify incorporating this component into the remaining initial-state composition with the fixed parameter prior and complete likelihood accounting. +The 18 supported conditional draws establish local support under this proposal, not an 18-seed agent solve rate, a posterior estimate, or independent calibration. Neither a supported draw nor a successful numerical component test establishes posterior adequacy, held-out prediction quality, or unchanged agent performance. Production fitting, planning, and execution estimation remain unchanged. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index ddf7a951e..30bd1c85a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -81,8 +81,11 @@ Ten initial-slider candidates retain the failure, but estimating static fixture These point-state cases are independently verified and supply candidates for explicit initial-state inference; they are not posterior forecasts or Stage B acceptance. The [Boil fixture proposal](boil-fixture-proposal.md) now separates an explicit uniform fixture-position prior from a first-65-observation Gaussian guide, retaining the complete prior/proposal density correction. Independent quadrature checks recover original-prior moments and posterior reference integrals under different guides. -Twenty-seven functional tests and focused static/format checks pass; the native 29-history audit and independent reader are queued. +Twenty-seven functional tests and focused static/format checks pass. +The native 29-history audit and independent reader are complete: all 18 guided fixture draws have supported full conditional histories, all six broad draws retain zero likelihood, and the reader verifies 68,904 joint factors plus all guide densities and inverse quantiles. Full initial-state composition and posterior adequacy remain open. +The subsequent [Boil articulated-state component](boil-articulated-prior.md) adds a direct original-prior map for the unobserved faucet joint while preserving the existing conditioning interface for the two switches. +Sixteen functional tests and focused static/format checks pass; the native 18-history audit is running and its readers are queued. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. diff --git a/docs/uncertainty/initial-state-inventory.md b/docs/uncertainty/initial-state-inventory.md index 7f76de008..826818803 100644 --- a/docs/uncertainty/initial-state-inventory.md +++ b/docs/uncertainty/initial-state-inventory.md @@ -137,6 +137,11 @@ The [scalar inadequacy control](boil-incomplete-control.md) now quantifies the f Even the best unrestricted constants leave bubbling and water-volume RMSE at 5.51 and 5.93 times the declared sensor sigma, while spill remains near the noise scale. This is a noisy predictive failure, not an exact contradiction or a reason to invent hidden heat initialization from evaluator metadata. +The later [Boil articulated-state component](boil-articulated-prior.md) identifies an additional unobserved revolute joint in the faucet asset, separate from the two observed on/off switches. +Its original position/motion law must remain unconditioned by those flags; the switch laws retain their actual event probabilities. +The component implementation passes numerical references and the native inventory confirms all three joints; restoration and full-history checks are running. +This adds explicit missing joint coordinates without declaring the rest of the Boil scene inventory complete. + ## Original non-hatch balloons The recording contains one box, three balloons, three clips, one band, and the robot. diff --git a/predicators/code_sim_learning/inference_joints.py b/predicators/code_sim_learning/inference_joints.py index 26570d1a4..0941ff1d6 100644 --- a/predicators/code_sim_learning/inference_joints.py +++ b/predicators/code_sim_learning/inference_joints.py @@ -295,6 +295,38 @@ def condition_above(self, threshold: float, """Condition on position > threshold without selecting one angle.""" return ThresholdJointPrior(self, threshold, observed) + @property + def coordinates(self) -> BoxPrior: + """Unit coordinates for the original, unobserved joint law.""" + return BoxPrior( + (self.name + ".position_mixture", self.name + ".velocity"), + ((0., 1.), ) * 2) + + def lift(self, point: np.ndarray) -> Tuple[float, float]: + """Draw the original rest/motion prior without inventing a reading. + + The second coordinate is an unused uniform auxiliary in either + rest case. The original law is already normalized, so this map + contributes no observation or proposal-density correction. + """ + values = np.asarray(point, dtype=float) + if values.shape != (2, ) or not np.isfinite(values).all(): + raise ValueError("Invalid original joint coordinates") + if np.any(values <= 0) or np.any(values >= 1): + raise JointCoordinateBoundary( + "Original joint coordinates must be interior") + for index, position in enumerate(self.rest_positions): + if values[0] < self.rest_probability * (index + 1) / 2: + return position, 0. + position = self.lower + (self.upper - self.lower) * \ + (float(values[0]) - self.rest_probability) / \ + (1 - self.rest_probability) + if not self.lower < position < self.upper: + raise JointCoordinateBoundary( + "Interior position rounded to an original-prior boundary") + velocity = self.velocity_half_width * (2 * float(values[1]) - 1) + return position, velocity + @dataclass(frozen=True) class ThresholdJointPrior: diff --git a/tests/code_sim_learning/test_unobserved_joints.py b/tests/code_sim_learning/test_unobserved_joints.py new file mode 100644 index 000000000..7156a26d8 --- /dev/null +++ b/tests/code_sim_learning/test_unobserved_joints.py @@ -0,0 +1,71 @@ +"""Original articulated-joint priors without synthetic Boolean readings.""" +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_joints import \ + JointCoordinateBoundary, RestingJointPrior + + +@pytest.mark.parametrize("rest", [0., .7, 1.]) +def test_original_joint_moments_and_atoms(rest: float) -> None: + """Piecewise quadrature recovers the declared mixed prior exactly.""" + prior = RestingJointPrior("faucet.hinge", -1., 2., (-.3, .8), rest, .2) + points = np.array([-1., 1.]) / np.sqrt(3.) + weights = np.ones(2) + moments = np.zeros(6) + edges = (0., rest / 2, rest, 1.) + for lower, upper in zip(edges[:-1], edges[1:]): + if upper == lower: + continue + for point, weight in zip(points, weights): + u = lower + (point + 1) * (upper - lower) / 2 + for other, other_weight in zip(points, weights): + q, v = prior.lift(np.array([u, (other + 1) / 2])) + mass = weight * other_weight * (upper - lower) / 4 + moments += mass * np.array( + [1., q, q * q, v, v * v, + float(v == 0.)]) + # Uniform[-1,2] has first/second moments 1/2 and 1. + expected = [ + 1., rest * .25 + (1 - rest) * .5, + rest * (.3**2 + .8**2) / 2 + (1 - rest), 0., (1 - rest) * .2**2 / 3, + rest + ] + np.testing.assert_allclose(moments, expected, rtol=0, atol=1e-14) + assert prior.coordinates.names == ("faucet.hinge.position_mixture", + "faucet.hinge.velocity") + assert prior.coordinates.bounds == ((0., 1.), (0., 1.)) + + +def test_unobserved_sampling_preserves_existing_conditioning() -> None: + """Both original atoms stay available and actual readings still + condition.""" + prior = RestingJointPrior("slider", 0., 2., (0., 2.), .8, .1) + identity = prior.digest + off = prior.condition_above(1., False) + on = prior.condition_above(1., True) + before = [law.lift(np.array([.9, .75])) for law in (off, on)] + assert prior.lift(np.array([.2, .25])) == (0., 0.) + assert prior.lift(np.array([.6, .25])) == (2., 0.) + q, v = prior.lift(np.array([.9, .75])) + assert q == pytest.approx(1.) and v == pytest.approx(.05) + assert [law.lift(np.array([.9, .75])) for law in (off, on)] == before + assert prior.digest == identity + assert off.log_observation_factor == pytest.approx(np.log(.5)) + assert on.log_observation_factor == pytest.approx(np.log(.5)) + + +def test_original_joint_invalid_coordinates_and_seams() -> None: + """Malformed coordinates and zero-measure seams stay explicit.""" + prior = RestingJointPrior("hinge", 0., 1., (0., 1.), .8, .1) + for point in [np.array([.5]), np.array([np.nan, .5])]: + with pytest.raises(ValueError): + prior.lift(point) + for point in [ + np.array([0., .5]), + np.array([1., .5]), + np.array([.5, 1.]), + np.array([.8, .5]) + ]: + with pytest.raises(JointCoordinateBoundary): + prior.lift(point) From f1d7fa3a21b27c0eb1fb84072b6fbfd83365025b Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 13:43:19 -0400 Subject: [PATCH 46/94] Record verified Boil joint-state support and literal rule reuse --- docs/uncertainty/boil-articulated-prior.md | 11 ++++++++--- docs/uncertainty/implementation-progress.md | 3 ++- docs/uncertainty/initial-state-inventory.md | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/uncertainty/boil-articulated-prior.md b/docs/uncertainty/boil-articulated-prior.md index 7409e34fb..5012ef959 100644 --- a/docs/uncertainty/boil-articulated-prior.md +++ b/docs/uncertainty/boil-articulated-prior.md @@ -43,7 +43,7 @@ The reader independently reconstructs the mixture maps and rejects altered readb It also verifies all existing output and transition factors and both reference histories. The driver saves each native observation before the historical rules execute. -A separate reader will reconstruct those states and run the literal historical rules, requiring exact agreement with all saved output predictions and model memory and rejecting emitted physical commands. +A separate reader reconstructs those states and runs the literal historical rules, requiring exact agreement with all saved output predictions and model memory and rejecting emitted physical commands. This checks a possible reuse boundary for cheaper later parameter evaluations. It does not authorize reuse for other programs or establish that the entire joint-state inference problem is solved. @@ -53,9 +53,14 @@ Compute checks `22687962` passed 16 functional tests, focused type checking, lin These include independent piecewise quadrature for the original mixed prior, both pure-component limits, unchanged conditional behavior, and invalid-coordinate cases. The first check's untyped quadrature-library call was replaced by the analytic two-node Gaussian quadrature rule; the statistical reference itself had already passed. -Native job `22688041_0` is running on `mit_preemptable`, followed by queued independent reader `22688102` and literal-rule reader `22688171`. +Native job `22688041_0`, independent reader `22688102`, and literal-rule reader `22688171` all completed on `mit_preemptable`. +Their one-CPU allocation times are respectively 232, 208, and 22 seconds. The instantiated inventory confirms two sliders with URDF interval `[0, 0.296]` and the separate faucet hinge with interval `[0, pi/2]`; the effective slider interval is `[0, 0.0296]` meters. -No native result is accepted merely because these jobs have been submitted. +All 16 sampled joint triples have finite full conditional likelihoods, and both reference histories repeat exactly. +The samples include nonzero initial velocity in four faucet-switch draws, one burner-switch draw, and five faucet-hinge draws. +The independent reader verifies all 48 initial joint draws and 42,768 robot transition factors, with maximum arithmetic difference 3.553e-15, and rejects seven corrupted-input classes. +The literal-rule reader reproduces all 4,752 post-rule output frames and every recorded model-memory state exactly, with no new native actions. +This reuse check covers the tested parameter values; validating reuse across parameter changes remains a prerequisite for an optimized fitter. The full scene model must still combine these joints with uncertain fixture height/orientation, jug pose and motion, robot nuisance state, physical support, and the fixed program's liquid/memory initialization contract. Any feasibility conditioning must preserve its normalization and the original parameter prior. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 30bd1c85a..7b09600b3 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -85,7 +85,8 @@ Twenty-seven functional tests and focused static/format checks pass. The native 29-history audit and independent reader are complete: all 18 guided fixture draws have supported full conditional histories, all six broad draws retain zero likelihood, and the reader verifies 68,904 joint factors plus all guide densities and inverse quantiles. Full initial-state composition and posterior adequacy remain open. The subsequent [Boil articulated-state component](boil-articulated-prior.md) adds a direct original-prior map for the unobserved faucet joint while preserving the existing conditioning interface for the two switches. -Sixteen functional tests and focused static/format checks pass; the native 18-history audit is running and its readers are queued. +Sixteen functional tests and focused static/format checks pass. +All 16 sampled joint triples have supported full conditional histories; independent readers verify 48 initial joint draws, 42,768 transition factors, and exact literal-rule reconstruction of all 4,752 saved output frames and model-memory states. The next gate is trustworthy decision-relevant prediction within the declared computation budget, followed by saved-decision shadow comparisons and matched live use. No current physical posterior has been approved for the acting agent. diff --git a/docs/uncertainty/initial-state-inventory.md b/docs/uncertainty/initial-state-inventory.md index 826818803..72fc94c65 100644 --- a/docs/uncertainty/initial-state-inventory.md +++ b/docs/uncertainty/initial-state-inventory.md @@ -139,7 +139,7 @@ This is a noisy predictive failure, not an exact contradiction or a reason to in The later [Boil articulated-state component](boil-articulated-prior.md) identifies an additional unobserved revolute joint in the faucet asset, separate from the two observed on/off switches. Its original position/motion law must remain unconditioned by those flags; the switch laws retain their actual event probabilities. -The component implementation passes numerical references and the native inventory confirms all three joints; restoration and full-history checks are running. +The component implementation passes numerical references, the native inventory confirms all three joints, and independent checks verify 48 initial joint draws plus all 18 complete conditional histories. This adds explicit missing joint coordinates without declaring the rest of the Boil scene inventory complete. ## Original non-hatch balloons From 4affabe15072a0677c08e29deb4759c9f9ffd039 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 14:22:16 -0400 Subject: [PATCH 47/94] Record Domino transition-model disagreement and budget follow-up --- docs/uncertainty/domino-joint-transition.md | 43 +++++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md index fc7ef0501..d0927604a 100644 --- a/docs/uncertainty/domino-joint-transition.md +++ b/docs/uncertainty/domino-joint-transition.md @@ -69,8 +69,8 @@ The component checks therefore justify a labeled inference experiment, not a cla ## Bounded inference comparison -Array `22683118` runs two new replicas, seeds 100 and 101, from the original prior. -Both startup checks pass and both runs have initialized their 64-particle populations. +Array `22683118` completed both new replicas, seeds 100 and 101, from the original prior. +Their allocations took 1:50:43 and 1:50:29 respectively on four CPUs. Each uses the same 32 cubic-spaced temperatures, eight moves per temperature, five single-coordinate blocks, scale 0.05, 50/50 local/full-range proposal mixture and 16,448-evaluation budget as the earlier 64-particle point-start comparison. The only intended statistical change is the explicitly identified joint-transition model. Each allocation requests four CPUs, 20 GB and four hours on node1412 in `mit_preemptable`. @@ -115,7 +115,7 @@ It independently checks 208,656 Gaussian joint factors, with maximum absolute lo Deliberately changed weights, generation/density roles, prefix values, forecast moments, future joints and joint-density factors are all rejected. These checks establish artifact consistency and arithmetic for the fixture, not statistical adequacy of its short fit. -The two full forecasts are queued as `22684522_1` and `22684523_2`, each dependent on successful completion of its corresponding fit. +The two full forecasts, `22684522_1` and `22684523_2`, completed together with their independent verifiers in 11:50 and 11:49 respectively on four CPUs. Each checks the frozen fixture-validation gate before generating predictions and runs the independent artifact verifier after finishing. The declared forecast budget is two banks of four draws per positive-weight particle plus one separate density history per particle, using four CPUs for at most 45 minutes. With all 64 weights positive, this requires 93,058 native actions including the two complete-history repeats. @@ -128,5 +128,40 @@ Missing forecasts or verification reports leave the comparison incomplete. Uncertain initial-state inference, wider development coverage, acceptable inference cost and closed-loop acceptance remain separate requirements. Comparison-reader validation `22684684` reproduces identical fixture summaries, detects injected position and event regressions, rejects misaligned coordinates and preserves an incomplete status when either new forecast is missing. -Finite report job `22684725` is queued after both full forecast jobs reach terminal states; it is not a notification monitor. +Finite report job `22684725` completed after both full forecasts and verifiers; it is not a notification monitor. The bundle's `verified-inputs.json` pins the tested scripts, fixture reports and job mapping. + +## Completed comparison and larger-budget follow-up + +The new replicas agree to 0.215 mm in position means, but their maximum toppling-curve and final-toppling gaps are both 0.166830. +The final gap exceeds the predeclared 0.15 limit, so the alternative still fails its numerical replication screen. +The older joint-output model's matched 64-particle pair has 0.880 mm position disagreement and a 0.012149 toppling gap; its separate larger-budget failures remain recorded in the earlier comparison. +Agreement at this single budget does not approve either model. + +| Model and numerical seed | Position RMSE (m) | Toppling Brier score | Final-toppling Brier score | Forecast native actions | +| --- | ---: | ---: | ---: | ---: | +| Joint output error, 100 | 0.0114410 | 0.000002870 | 0.000148375 | 10,465 | +| Joint output error, 101 | 0.0113989 | 0.000003864 | 0.000293803 | 10,465 | +| Joint transition, 100 | 0.0113551 | 0.000119152 | 0.011549145 | 93,058 | +| Joint transition, 101 | 0.0113573 | 0.000018461 | 0.001548140 | 93,058 | + +These are held-out suffix predictions on one development recording, not solve rates or agent seeds. +The new model's lower position error accompanies worse toppling scores on this recording. +Its two future banks differ by 0.084 and 0.158 mm in position means and by 0.013326 and 0.006608 in toppling curves, substantially less than the disagreement between fitted replicas. +The two full verifiers each check 834,624 joint factors with maximum error 8.882e-16, preserve original weights and reject all six corruption classes. + +The new fits retain only six and five initial ancestors. +Mass medians differ, approximately 0.176 versus 0.113, as do spinning-friction medians, approximately 1.263 versus 0.614. +The next experiment tests budget sensitivity rather than assuming that more accepted proposals imply reliable uncertainty. + +Array `22688869`, in `logs/uncertainty_domino_joint_transition_128_20260913/`, runs matched seeds 100 and 101 with 128 particles and a 32,896-evaluation cap. +The original prior, law, program, data, temperature schedule and proposal settings are retained. +Forecast jobs `22689058_0` and `22689059_1` depend on their respective successful fits and reuse the exact verified forecast and reader implementations. +Each requests four CPUs for at most one hour and checks all generated artifacts after completion. +The expected cost with 128 positive weights is 185,794 native actions per forecast, including two repeats. + +The budget report `22689095` retains all six within-budget and cross-budget comparisons among the four new-model populations. +Its reader checks matching prior, complete inference identity and all sampler settings except particle/evaluation budgets. +Reader test `22689104` passes identical-summary, injected-regression and misaligned-coordinate controls and correctly reports the current two-population comparison as incomplete. +The follow-up forecast/report bundle is `logs/uncertainty_domino_transition_128_forecast_20260913/`. +All results remain a fixed-initial-state ablation and do not close Stage B. From 4a1c6f5cb202bc3735f4816824e071cf952dbb36 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 14:22:16 -0400 Subject: [PATCH 48/94] Document verified Boil full-scene support and parameter-independent replay --- docs/uncertainty/boil-full-scene-prior.md | 100 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 17 +++- docs/uncertainty/initial-state-inventory.md | 6 ++ 3 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 docs/uncertainty/boil-full-scene-prior.md diff --git a/docs/uncertainty/boil-full-scene-prior.md b/docs/uncertainty/boil-full-scene-prior.md new file mode 100644 index 000000000..5d92b3d2c --- /dev/null +++ b/docs/uncertainty/boil-full-scene-prior.md @@ -0,0 +1,100 @@ +# Boil complete unheld-scene prior audit + +This extends the independently verified [fixture proposal](boil-fixture-proposal.md) and [articulated-state component](boil-articulated-prior.md) into a declared full scene for the fixed historical Boil program. +It is part of Stage A physical support work needed for Stage B inference comparisons. +The production agent and its uncertainty handling remain unchanged. + +## Information and model boundary + +The development input is the first 264-action training recording under `logs/uncertainty_recording_audit_20260912/inputs/Boil/L01/`. +Only reconstructed public observations and actions enter inference. +The frozen runtime is `b09217bb38f2c3082136994ae43fbef2eb590e82`, with explicitly captured inference-component overlays. +The historical learned filling/heating program has SHA-256 `b123f544d835e745d2c740d527ebb6b80d20e9b8bb412b1c6429f8de11afe172`. +Its eight original development parameter bounds define independent uniform priors; an optimizer's logarithmic coordinate flag does not change that prior. +This is a transferred historical program, separate from the later sweep's parameter-free Boil model. + +The initial case has one unheld jug, conditioned on its public exact held-state reading. +Known shapes, object identities and colors, fixed robot base placement, and the two-table geometry are conditioned interface inputs. +Missing joint velocities, object orientations and motion do not imply rest. +The fresh native initializer supplies cold heat and zero spill, and the literal learned program starts with `LATENT_INIT={}`. +No evaluator-private heat or saved inferred memory initializes the candidate. +Jug volume is sampled explicitly and initializes the learned volume memory through the ordinary rule path. + +## Original scene components + +| Component | Declared distribution | +| --- | --- | +| Four fixed fixture base positions | Independent uniform x in [0.35, 1.15], y in [1.05, 1.65], z in [0.35, 0.8] meters | +| Fixture orientations | Probability 0.8: all upright with independent uniform yaw; probability 0.2: independent uniform SO(3) orientations | +| Robot joint positions | Original zero-centered Gaussian reset law; exact controlled positions are conditioned with their density retained, four unobserved positions remain uncertain | +| Robot motion | Probability 0.8: all joint velocities zero; otherwise each of thirteen movable joint velocities uniform in [-0.1, 0.1] native units | +| Jug pose and motion | Probability 0.8: upright, resting on the support plane with uncertain xy/yaw; otherwise free position/orientation with linear velocities in [-0.1, 0.1] m/s and angular velocities in [-0.2, 0.2] rad/s | +| Jug water | Probability 0.5 at zero; otherwise uniform over [0, 1.3] | +| Two switch sliders | Validated endpoint-rest/uniform-motion law, conditioned on exact initial flags with both event probabilities retained | +| Unobserved faucet hinge | Validated endpoint-rest/uniform-motion law over [0, pi/2], without an invented switch observation | + +The jug placement cell is x in [0.3, 1.2], y in [1.1, 1.85], z in [0.4, 1.2] meters. +The existing rigid-assembly component erodes its free position bounds by a conservative enclosing radius derived from native geometry. +The resting component uses the actual collision support depth. +The geometry predicate rejects the whole candidate if the robot or jug penetrates another collision body or any pair of fixed fixtures interpenetrates. +Fixed wheel/ground contacts, fixture mounting intersections with tables, and fixed table overlap are explicit exceptions. +Visual-only liquid bodies do not participate in this predicate. + +The target is proportional to `p(theta) p0(scene) I[C(scene)]` times the full observation/transition likelihood. +The geometric normalizer is independent of theta for this fixed program: the eight parameters modify only reported features and latent memory, with no physical commands or parameter-dependent collision shape. +Its unknown common normalizer can therefore be omitted from posterior ratios, but not from absolute model evidence. +This justification does not apply automatically to other programs. + +## Observation-guided proposals + +All proposal guides retain a 0.25 original-prior component and a 0.75 normalized truncated-Gaussian component. +Every guide retains the full original-prior/proposal density correction. +Fixture xyz and observed upright yaw use the first 65 public observations; jug position, resting yaw and positive water volume use the first observation. +Fixture xyz guidance includes the declared correlated output discrepancy. +Unobserved fixture angles and moving SO(3) branches do not acquire a fabricated observation guide. +All original observation factors remain in the likelihood exactly once. +Unused coordinates in mixture cases integrate out under the original unit-coordinate representation. + +## Native support correction + +The first complete-scene attempt, job `22688683_0`, used the jug AABB lower edge as its support depth. +That edge includes 1 mm of collision padding: 0.061 m instead of the actual 0.060 m support depth. +The job was cancelled after identifying this mistake; its partial records remain in `logs/uncertainty_boil_full_scene_20260913/` as an invalid-prior diagnostic. +They are not an agent failure or accepted posterior input. + +The corrected initializer queries native jug-to-table separation from an elevated identity-orientation pose, derives the support depth, and checks it again at zero separation before applying the candidate. +The corrected native audit, `22688726_0`, completed in 7:02 on one compute CPU. +Its bundle is `logs/uncertainty_boil_full_scene_v2_20260913/`. + +Sixteen independently drawn scenes use seed 402, with two parameter settings per scene: historical defaults and an independent original-uniform parameter draw. +Two identical point controls provide the previously verified reference trajectory. +Three scenes fail the declared initial geometry predicate and are rejected before any action. +Thirteen scenes are geometrically feasible; two have finite full-recording likelihood under both parameter settings, while the other eleven retain exact-event contradictions. +That gives 28 completed histories and 7,392 native actions including the point controls. +These counts measure support in this proposal audit, not agent solve rates or numerically adequate posterior coverage. + +Every paired physical history is identical across its two parameter settings, including native pre-rule frames and joint-transition factors. +The rule-generated observations and model memory may differ with parameters. +This is the required physical independence check before considering reuse of native histories during parameter inference. + +Fresh geometry verification `22689157` completed on one compute CPU in 24 seconds. +It reconstructs all sixteen saved scenes in new native worlds, independently enumerates 464 eligible body pairs, and reproduces all rejected and explicitly allowed intersections. +It also repeats both support-depth probes for each scene without executing environment actions. + +Independent density verification `22689045` completed in 5:51 on one CPU. +It verifies all 66,528 joint factors to maximum absolute error 3.638e-12, the Gaussian guide calculations and inverse proposal coordinates, all conditioned initial-state factors and complete output likelihoods. +Ten corrupted transition, output, proposal and initial-state inputs are rejected. +Additional coordinate verification `22689232` checks all sixteen scenes against the original bounds, auxiliary-coordinate layout, moving-jug orientation/velocity map and mixture branches, rejecting five corruption classes. + +Literal-rule verification `22689096` completed in 28 allocation seconds, with 15.143 seconds in its reader and no native actions. +All 7,392 rule-generated frames and model-memory states match exactly under both default and independently sampled parameter settings. +The native physical histories are therefore reusable across the tested parameter changes, while each parameter-dependent observation and memory trajectory is still evaluated anew. +The bundle's `verified-inputs.json` pins the source report, scripts and all four verification reports. + +## Remaining gate + +A finite candidate establishes likelihood support, not useful parameter uncertainty. +The next experiment must cover the original scene and parameter law, preserve all initial conditioning factors, and assess independent-run agreement, held-out predictions and compute cost. +Cached physical histories may reduce repeated work only under the fixed program's verified parameter-independence boundary. +No arbitrary reuse across physical parameters, state changes, different programs or different observation phases is justified. +Stage A closure, Stage B acceptance and live-agent non-regression remain unproven. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 7b09600b3..d635b76fa 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -30,10 +30,11 @@ All sampled boundary contact-pair sets match between those pairs, and an indepen The subsequent 124-case local-sensitivity test also completes: non-restitution joint-response derivatives vary strongly across 1e-6 to 1e-4 parameter changes, while 241,800 additional scalar factors verify independently; a simple gradient proposal is not supported. The separate [joint-transition model comparison](domino-joint-transition.md) now passes 124-case probability/replay checks and full 97-action future-generation/density validation. It retains every joint transition density and cached-link observation timing while removing only the former joint AR output factors as an explicit model change. -Two fresh 64-particle point-start inference replicas, `22683118_0` and `_1`, are running under the original prior and unchanged sampler settings; their posterior prediction comparison remains pending. -The checkpoint-driven forecast adapter now passes a native 16-particle integration fixture and an independent reader over all 144 saved histories. -The reader verifies original weights, separates unconditional generation from future-conditioned density evaluation, and rejects six deliberately corrupted inputs. -Full forecasts `22684522_1` and `22684523_2` are queued behind their respective fits, with fixture validation required before execution and complete artifact verification afterward. +Both 64-particle point-start fits and their independently verified full-suffix forecasts have completed. +The new pair agrees to 0.215 mm in position means but differs by 0.166830 in final-toppling probability, failing the declared 0.15 limit. +Future-bank variability is much smaller than fitted-replica disagreement, and the new model has worse toppling scores than the matched older model on this recording despite slightly better position error. +The complete artifacts retain original weights, separate generation from conditioned density evaluation and reject six deliberately corrupted inputs. +Matched 128-particle follow-ups `22688869_0` and `_1` are running, with dependent forecast/verifier jobs `22689058_0` and `22689059_1` and a six-pair budget report `22689095`. These remain offline diagnostics; neither the short fixture nor a completed sampler is an approved posterior. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. @@ -536,6 +537,14 @@ Four cold-fit comparisons are submitted for the two 64-action windows and the tw The Fan audit separates its earliest broad support from later data-derived bounds and declares a fixed uniform prior for future posterior experiments without changing the latest program dynamics. These are comparison inputs and submitted experiments, not completed Stage B evidence or a production replacement. +The [complete Boil scene audit](boil-full-scene-prior.md) now composes fixture poses, robot nuisance joints and motion, jug pose/motion/water and all articulated components under an explicit geometry-conditioned law. +The corrected native audit completed 28 histories and 7,392 actions; two of sixteen sampled scenes have finite complete-recording likelihood, three are geometrically infeasible and eleven retain exact-event contradictions. +Paired default/random parameter cases preserve identical physical histories for the fixed feature-only program. +Fresh independent geometry reconstruction verifies all sixteen scenes, 464 body pairs and both native support probes per scene. +Independent density and coordinate checks pass, including 66,528 transition factors, original-prior corrections and fifteen corruption controls. +Literal rule evaluation also reproduces all 7,392 frames and model-memory states exactly across both parameter settings without additional native actions. +This validates a fixed-program reuse boundary for future inference, while remaining a prior/support audit rather than a posterior or agent result. + ## Next gate The [parameter consumer boundary](parameter-consumers.md) now derives parameter quantiles and weighted or resampled ensembles from the same assessed joint approximation. diff --git a/docs/uncertainty/initial-state-inventory.md b/docs/uncertainty/initial-state-inventory.md index 72fc94c65..df82c8d1a 100644 --- a/docs/uncertainty/initial-state-inventory.md +++ b/docs/uncertainty/initial-state-inventory.md @@ -142,6 +142,12 @@ Its original position/motion law must remain unconditioned by those flags; the s The component implementation passes numerical references, the native inventory confirms all three joints, and independent checks verify 48 initial joint draws plus all 18 complete conditional histories. This adds explicit missing joint coordinates without declaring the rest of the Boil scene inventory complete. +The subsequent [full unheld-scene composition](boil-full-scene-prior.md) declares fixture xyz/orientation, all robot nuisance joints and motion, jug pose/motion/water and the three articulated components together. +Its 84 unit inputs include mixture selectors and unused case auxiliaries as well as eight parameter inputs; this is a computational map size, not a claim of 84 independent physical degrees of freedom. +The whole-scene geometry predicate has a parameter-independent normalizer only for the frozen historical feature-only program. +Independent density, coordinate and fresh-native geometry checks pass, including two full-recording supported sampled scenes and exact rule reuse across changed parameters. +This closes a declared prior/support construction for that unheld development case, while broader attachment cases, posterior adequacy and all-domain runtime closure remain separate requirements. + ## Original non-hatch balloons The recording contains one box, three balloons, three clips, one band, and the robot. From 1f36f9977a2e3c3de0b4a76030487f29965ed77f Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 14:55:17 -0400 Subject: [PATCH 49/94] Check budget-specific Domino runtime identities independently --- docs/uncertainty/domino-joint-transition.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md index d0927604a..a3a54bd2d 100644 --- a/docs/uncertainty/domino-joint-transition.md +++ b/docs/uncertainty/domino-joint-transition.md @@ -161,7 +161,9 @@ Each requests four CPUs for at most one hour and checks all generated artifacts The expected cost with 128 positive weights is 185,794 native actions per forecast, including two repeats. The budget report `22689095` retains all six within-budget and cross-budget comparisons among the four new-model populations. -Its reader checks matching prior, complete inference identity and all sampler settings except particle/evaluation budgets. +Its reader checks matching prior, data, sensor and program identities and all sampler settings except particle/evaluation budgets. +The runtime identities legitimately differ because they include the budget-specific plan; each is independently checked against its frozen plan, scripts and preflight rather than requiring the two runtime hashes to match. +Follow-up reader check `22689407` passes, and both live 128-particle reports match their independently reconstructed expected runtime identity. Reader test `22689104` passes identical-summary, injected-regression and misaligned-coordinate controls and correctly reports the current two-population comparison as incomplete. The follow-up forecast/report bundle is `logs/uncertainty_domino_transition_128_forecast_20260913/`. All results remain a fixed-initial-state ablation and do not close Stage B. From 206a8648599ef7fc93d0671c383e11135c1a4628 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 14:55:19 -0400 Subject: [PATCH 50/94] Validate canonical Boil initialization and launch prefix inference --- docs/uncertainty/boil-full-scene-prior.md | 8 ++ docs/uncertainty/boil-joint-inference.md | 117 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 8 ++ 3 files changed, 133 insertions(+) create mode 100644 docs/uncertainty/boil-joint-inference.md diff --git a/docs/uncertainty/boil-full-scene-prior.md b/docs/uncertainty/boil-full-scene-prior.md index 5d92b3d2c..b18425750 100644 --- a/docs/uncertainty/boil-full-scene-prior.md +++ b/docs/uncertainty/boil-full-scene-prior.md @@ -98,3 +98,11 @@ The next experiment must cover the original scene and parameter law, preserve al Cached physical histories may reduce repeated work only under the fixed program's verified parameter-independence boundary. No arbitrary reuse across physical parameters, state changes, different programs or different observation phases is justified. Stage A closure, Stage B acceptance and live-agent non-regression remain unproven. + +## Subsequent runtime-closure finding + +The [prefix-inference integration](boil-joint-inference.md#initialization-order-obstruction-and-correction) exposes an additional limitation of this audit. +Removing its intermediate initialization operations changes some contact-rich continuations despite exact agreement in initial public state and saved scene geometry. +The original operation sequence still repeats exactly, and its density, geometry and literal-rule checks remain valid for that sequence. +Those checks do not establish that the final physical coordinates alone define an observation-independent generative initialization. +The subsequent fixed-template experiment removes the observation-derived intermediate numeric anchor and is being validated as a separately identified runtime before posterior fitting. diff --git a/docs/uncertainty/boil-joint-inference.md b/docs/uncertainty/boil-joint-inference.md new file mode 100644 index 000000000..3929a457d --- /dev/null +++ b/docs/uncertainty/boil-joint-inference.md @@ -0,0 +1,117 @@ +# Boil joint inference on a held-out-prefix split + +This follows the [verified full-scene support audit](boil-full-scene-prior.md). +The candidate retains the complete declared unheld-scene prior and the historical program's eight independent original-uniform parameter priors. +It is an offline Stage B experiment; the acting agent still uses the incumbent estimator. + +## Fitting and prediction boundary + +Only actions 0 through 131 and observations 0 through 132 enter the fitting target. +The remaining 132 recorded actions and observations are reserved for a later causal forecast comparison. +The target context retains only the fitting prefix, including its own immutable data identity. +Fixture proposal guidance uses observations 0 through 64, wholly inside that prefix. +Jug pose and water guidance use observation zero. +No future reading conditions the fitting trajectory, initializes memory or changes the proposal guide. + +The physical scene uses 76 unit inputs, including mixture selectors and unused case auxiliaries; eight more unit inputs select the original physical parameter values. +The joint sample retains the eight named parameter values together with the 76 scene auxiliaries, preserving parameter/state dependence. +This dimension describes the computational representation, not 84 independent physical degrees of freedom. + +The conditioned base contains the scene prior/proposal correction, geometric support indicator, initial exact-joint conditioning densities and the complete initial output factor exactly once. +The tempered likelihood contains subsequent joint-transition densities and observations 1 through 132. +The unknown geometric normalizer remains common to the parameters for this fixed feature-only program and is not used to claim absolute model evidence. + +## Exact physical-history reuse + +The frozen program changes only `jug0.water_volume`, `jug0.bubbling_level` and `faucet.spilled_level` in the public predictions. +All three use their original scalar sensor likelihoods; none belongs to a coupled Euler, temporal discrepancy or derived-readout factor. +Every other physical prediction and output factor is independent of the eight program parameters under the previously verified command-free boundary. + +A worker caches the native 132-action history by the complete 76-coordinate scene input. +Changing a physical scene coordinate requires a new native replay. +Changing only program parameters reuses that history, restarts the literal program memory, and recomputes all three variable observation channels. +The code checks every unchanged channel and refuses physical commands rather than assuming that every learned program permits this optimization. +The cache is bounded to 32 scenes per worker and does not modify target values or the sampler's random stream. + +The target sums the cached physical output/transition factors and newly evaluated scalar factors. +It never subtracts two very large full-history scores to obtain a remaining likelihood. +The initial scalar factor is included in the base, with only later scalar observations included in the parameter-dependent term. +Zero geometric support and exact-event contradictions remain explicit zero density. + +## Validation and numerical plan + +The preflight reconstructs all sixteen previously audited physical scenes and compares every native fitting-prefix frame with the saved 264-action histories. +For every geometrically feasible scene it evaluates both historical default and independently sampled parameter settings through the literal rules. +It compares the factored target against a direct complete-prefix output likelihood plus all original scene and transition factors. +It also checks unchanged targets across repeated calls, reuse after parameter changes, and serial/parallel agreement. +This is a target and cache validation, not a posterior assessment. + +The first preflight, `22689476`, stopped during import before simulation because the historical snapshot lacked the new evaluation module. +The second, `22689537`, exposed the missing causal future-likelihood method in the historical observation module. +Both failures remain recorded; their 13-second and 36-second allocations are setup costs, not failed inference or agent seeds. +The corrected snapshot explicitly pins the evaluation, checkpoint, sampler and observation modules alongside the scene-prior components. + +The planned pilots use numerical seeds 410 and 411, 32 particles, 32 cubic-spaced temperatures, eight moves per temperature and a cap of 8,224 target evaluations each. +Thirteen disjoint proposal blocks cover five scene groups and eight individual parameters. +Each block uses the existing 50/50 local/full-range proposal mixture with local scale 0.05. +Complete-stage checkpoints retain the original prior, data, random state, weights, ancestry and cumulative evaluation count on continuation. +Per-allocation counters report native actions, physical replays, cache hits, rule evaluations and worker time separately from sampler evaluations. +Each run requests four compute CPUs and at most eight hours in `mit_preemptable`. + +The experiment bundle is `logs/uncertainty_boil_joint_inference_20260913/`. +A passing preflight is required before either posterior pilot starts. +Neither a completed sampler nor agreement on this one recording establishes calibrated uncertainty, acceptable latency or live-agent non-regression. +Both posterior replication and the reserved-action prediction comparison remain required before advancing. + +## Initialization-order obstruction and correction + +The first native-prefix preflight failed after the target stopped reproducing intermediate initialization operations from the older audit. +Jobs `22689582` and `22689612` record that failure and its detailed reproduction. +Final initial observations and the entire saved scene geometry still match exactly, but contact-rich continuations differ. +Scene 3 first differs in jug rotation at action 26, scene 7 in robot/contact motion at action 61, and scene 13 in switch state and robot motion at action 23. +The two repeatedly supported scenes, 2 and 12, initially match under both tested orders. + +Controlled diagnostic `22689667` compares both initialization orders on all five selected scenes and repeats each fresh trajectory. +Restoring the archived intermediate fixture-position resets and observation queries restores exact agreement on every scene. +Both orders repeat exactly independently, so this is deterministic dependence on initialization operations, not evidence of random sensor error. +The experiment identifies the responsible operation sequence; it does not identify a particular internal engine cache. +Final body poses, joints and collision witnesses alone are therefore insufficient to identify this simulator's initial runtime state. + +Simply retaining observation-derived intermediate poses would leave an unaccounted path from noisy readings into engine initialization. +The corrected candidate uses a fixed numeric template before applying the sampled scene. +The template retains only conditioned object identities/types/colors and the exact fixed base pose, if supplied. +Controlled joint positions are initially zero, and non-position numeric feature values are zero. +Jug, faucet, faucet-switch, burner and burner-switch template positions are explicitly recorded constants in the new plan. +The complete uncertain scene, including conditioned actual robot joints and all sampled poses/motion, is then applied through the unchanged component map. +No noisy numeric reading selects an intermediate reset pose. + +An attempt to omit task-state setup entirely, `22689702`, failed because the native world had not activated its jug state. +The fixed-template approach preserves that required setup while removing the observation-derived numeric anchor. +Diagnostic `22689937` repeats all five selected fixed-template histories exactly, with identical initial observations and scene geometry to the older audit. +Both previously supported scenes retain finite fitting-prefix factors; some subsequent histories differ, as expected from the changed initialization rule. +This is a separately identified generative runtime, not an assertion that the older and corrected likelihoods are equivalent. + +The independent archived-history factorization reference, `22689973`, completes all sixteen roots and both parameter settings for each feasible root. +Its maximum finite discrepancy between direct and factored log targets is 1.456e-11. +Three archived scenes have finite 132-action prefix targets; failed exact-event cases retain zero density. +That reference validates arithmetic and literal-rule reuse only, and the fit launcher explicitly refuses to accept it as a native preflight certificate. + +Full canonical preflight `22690049` completed in 1:52 on four CPUs. +It passes all sixteen scene checks, exact fresh repeats, unchanged source initial-state/geometry components, direct likelihood factorization, a retained historical trajectory control and serial/parallel target equality. +It also changes every non-conditioned numeric initial reading and checks that the fixed template is unchanged. +The canonical target and planned pilots are isolated in `logs/uncertainty_boil_joint_inference_canonical_20260913/`. +The original experimental bundle and its failed traces remain available as controls. + +Three of the sixteen canonical candidates have finite prefix targets, with maximum finite factored-versus-direct log-target error 1.456e-11. +Eight of thirteen geometrically feasible candidates also retain the historical prefix trajectory exactly; the other five retain their explicitly changed canonical-runtime histories. +The corrected initialization remains separate from the old intermediate-reset control even where their predictions agree. + +Array `22690118` launches the two gated canonical posterior pilots, numerical seeds 410 and 411. +The launcher verifies the native preflight kind, exact script set and hashes, plan identity, fitting-data identity and output-model identity before sampling. +An archived-history arithmetic certificate cannot satisfy that gate. +The canonical bundle's `verified-inputs.json` records the completed preflight and submitted jobs. +Posterior numerical adequacy and the 132-action reserved suffix remain unevaluated. + +Both pilots are running and have written their initialization checkpoints. +Of 32 initial particles, seed 410 has eleven finite targets and seed 411 has seven. +These initialization counts establish that sampling started with support; they do not establish adequate posterior exploration. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index d635b76fa..fa2a9e8ef 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -545,6 +545,14 @@ Independent density and coordinate checks pass, including 66,528 transition fact Literal rule evaluation also reproduces all 7,392 frames and model-memory states exactly across both parameter settings without additional native actions. This validates a fixed-program reuse boundary for future inference, while remaining a prior/support audit rather than a posterior or agent result. +The [Boil prefix-inference integration](boil-joint-inference.md) implements a cache for parameter-independent physical histories and separates the first 132 fitting actions from 132 reserved actions. +The archived-history reference verifies all factored targets against direct likelihoods to 1.456e-11, but native integration exposes initialization-order dependence in three tested contact trajectories. +Restoring the old intermediate operation sequence restores exact agreement; this leaves an observation-derived numeric initialization path that must be removed before treating the scene construction as a generative prior. +The fixed-template initializer passes full sixteen-scene preflight `22690049`, including fresh repeats, serial/parallel target equality, unchanged initial geometry and likelihood factorization. +Three sampled candidates have finite fitting-prefix targets, and the maximum finite factorization discrepancy is 1.456e-11. +Both gated canonical Boil posterior pilots, `22690118_0` and `_1`, are running with initialization checkpoints and eleven/seven finite targets out of 32 respectively. +Their numerical adequacy and held-out predictions remain unassessed. + ## Next gate The [parameter consumer boundary](parameter-consumers.md) now derives parameter quantiles and weighted or resampled ensembles from the same assessed joint approximation. From cd29e8717e358c82bcfe8f59b32d904e51f9dac3 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 14:55:42 -0400 Subject: [PATCH 51/94] Record first completed Balloons joint-inference pilot --- docs/uncertainty/balloons-composed-inference.md | 14 ++++++++++++++ docs/uncertainty/implementation-progress.md | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/balloons-composed-inference.md b/docs/uncertainty/balloons-composed-inference.md index 90a811326..9ccdc4a1d 100644 --- a/docs/uncertainty/balloons-composed-inference.md +++ b/docs/uncertainty/balloons-composed-inference.md @@ -254,3 +254,17 @@ The successful report, source hashes, candidate coordinates and per-seed physica This audit supplies unconditional physical and observation generation, not the complete stochastic posterior forecast interface. Numerically assessed joint posterior weights and integration over future transitions remain required. In particular, future-density evaluation must use Gaussian joint and radial speed factors with conditional-history integration, rather than treating finitely many unconditional paths as exact-output equality components. + +## First completed full-recording pilot + +Seed 300 completed through continuation job `22684078_0` in an additional 2:25:46 allocation on September 13. +Its two previous eight-hour allocations remain part of the approximately 18:25:46 allocated runtime cost. +This is one continued numerical seed, not three experiments or an agent result. + +The final population has 128 particles after 17,200 evaluated proposals and all 32 temperature stages. +It accepted 8,150 of 32,768 attempted moves, with 29 resampling events and only one surviving initial ancestor. +The report and checksummed checkpoint agree exactly on all joint samples, weights, completed stage and evaluation count. +The consistency record is `logs/uncertainty_balloons_joint_pilot_20260912/seed300-completion-check.json`. +These checks establish a completed artifact, not adequate posterior exploration, calibrated uncertainty or trustworthy predictions. +Seed 301 remains running as `22686274_1`; a paired assessment is not yet available. +The complete 235-action fitting recording cannot also serve as an independent held-out prediction test for this fit. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index fa2a9e8ef..b5c53ab42 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -49,10 +49,13 @@ Fixed centers preserve every diagnostic report across all three repetitions; car This supplies active-carry repeated-data coverage and supports an immutable original prior, but does not establish unchanged future predictions or planning decisions. The original harness reproduction also confirms that applying fitted values to a reused subclass reference can change registry defaults even with explicit carrying off; the corrected comparison isolates that effect without changing production behavior. -The long-prefix Balloons fits remain ongoing. +The first full-recording Balloons fit, numerical seed 300, has completed; seed 301 remains ongoing. After Slurm confirmed `22671041_0` timed out, its seed-300 fit continued as `22684078_0` from completed stage 28 and 15,107 evaluations under the same frozen runtime, prior and numerical budget. Its two previous eight-hour allocations remain part of the cost. Seed 301 subsequently reached a confirmed allocation timeout and continued as `22686274_1` from completed stage 24 and 13,819 evaluations, preserving its existing numerical budget and prior sixteen allocation hours. +Seed 300 has now completed all 32 stages with 17,200 evaluations, 29 resampling events and one surviving initial ancestor. +Its report and checksummed checkpoint agree exactly on the saved joint population and weights; numerical availability remains unevaluated. +The final 2:25:46 allocation adds to its prior sixteen hours, and the full fitting recording is not a held-out prediction test. Both full-recording Fan fits and their verified report have completed, but their narrow empirical speed distributions do not overlap and each retains one initial ancestor; they remain unassessed. The [repeated-dataset Gaussian reference](repeated-dataset-reference.md) has completed all 512 fits. All three parameter-coordinate groups pass the declared CDF-error screens at 2,048 particles; both correlated coordinates fail at 256 particles despite apparently plausible coverage. From 4654d0660e32bdcaf8fd9a601c3889decb896bbd Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 15:19:39 -0400 Subject: [PATCH 52/94] Document validated canonical Boil forecasts and paired assessment --- docs/uncertainty/boil-canonical-forecasts.md | 86 ++++++++++++++++++++ docs/uncertainty/boil-joint-inference.md | 5 ++ docs/uncertainty/implementation-progress.md | 7 ++ 3 files changed, 98 insertions(+) create mode 100644 docs/uncertainty/boil-canonical-forecasts.md diff --git a/docs/uncertainty/boil-canonical-forecasts.md b/docs/uncertainty/boil-canonical-forecasts.md new file mode 100644 index 000000000..6c3f1030f --- /dev/null +++ b/docs/uncertainty/boil-canonical-forecasts.md @@ -0,0 +1,86 @@ +# Boil forecasts after canonical prefix inference + +September 13, 2026. +This continues the [canonical Boil inference experiment](boil-joint-inference.md) in Stage B of the [simplification plan](simplification-proposal.md). +The experiment remains offline, and the production agent keeps its existing estimator. + +## Prediction boundary + +The two canonical fits use the first 132 recorded actions and 133 observations of the fixed development episode. +The following 132 actions are reserved for prediction assessment. +Each continuation reconstructs the sampled initial scene from the validated fixed template and replays all 264 actions in one fresh world. +It preserves native contact history and the literal simulator program's memory across the fitting boundary. +It does not restore an arbitrary engine snapshot at action 132. + +During the fitting prefix, the declared conditional joint-transition model uses the observed joint positions and retains their density factors. +During forecast generation, it draws fresh joint innovations without consulting future observations. +The output-error process conditions on the fitting prefix and propagates forward; future sensor readings are sampled independently under that process. +The separate future-likelihood calculation conditions on the recorded future joints and retains their complete transition density. +Those observation-conditioned paths are used only to evaluate density, never to calculate forecast means or event probabilities. + +Every generated continuation checks exact agreement with the original fitting worker on its entire physical prefix, literal-rule prefix and target factors. +No future likelihood reweights the fitted particles. +Every positive-weight particle contributes to the forecast, and particles assigning zero density to the recorded future remain in the mixture with their original weights. +An entirely unsupported empirical future mixture is reported explicitly, without renormalizing a successful subset. + +Clean public recordings supply assessment targets only. +The generator receives the fitting observations and future actions, but no future noisy or clean readings. +The fixed scene candidates used to validate the machinery were selected during earlier development audits, so their fixture results are mechanical checks, not an independent predictive-performance result. + +## Verified native fixture + +Fixture job `22690359` completed in 1:57 on four compute CPUs. +Two supported scene candidates each have two generated futures and one recorded-future density calculation, with additional repeats and density round trips. +The fixture records 4,884 native action steps, including fitting-reference replays. +Generated futures repeat exactly and reproduce their trajectory and complete density when their generated observations are supplied to the density evaluator. +Changing a future scalar reading changes its likelihood without changing the native trajectory or carried model memory. + +Independent verifier `22690509` completed in 2:33. +It checks all twelve saved histories and 28,512 scalar joint-transition factors, recomputes literal model memory, independently calculates output moments, and compares event flags with the native environment predicates. +It rejects deliberate corruptions of the generation/density role, fitting observations, corrected joints, transition factors, model memory, forecast moments, goal events and initial likelihood factor. +An earlier verifier attempt, `22690433`, failed while serializing NumPy Boolean predicate outputs; explicit conversion to ordinary Booleans fixes the checker without changing any simulation or factor. + +These checks validate the forecast machinery on the selected candidates. +They do not establish posterior exploration, predictive adequacy or an agent solve rate. + +## Completed-population adapter + +The adapter checks the source program, data, prior, model, runtime and sampler configuration against the frozen fit identity. +It restores a completed checksummed sampler checkpoint through the sampler API with target evaluation prohibited. +The recovered result must exactly match the saved population, including weights and joint coordinates. +Incomplete populations cannot enter this path. + +For each positive-weight particle, the planned assessment generates two independent banks of four futures and separately evaluates the recorded future density. +Every history is saved, and every saved record is checked against the original fitting prefix, complete transition factors and model memory. +The first generated and recorded-density paths are also repeated in fresh worlds. +An independent reader reconstructs the weighted summary from the saved histories and original checkpoint. + +Reported metrics retain their physical units: + +- Output-mean error against noisy future observations for jug position, water volume, bubbling and spilled volume. +- Native prediction error against clean public future readings for those same six features. +- Probability curves and Brier scores for filled, boiled, no spill, burner off and the actual task goal. +- Complete future mixture log density, including unsupported particles. +- Differences between future banks and between independently fitted populations. +- Sampler diagnostics, native simulation steps and allocation time. + +This first pair is exploratory and has no declared Boil numerical acceptance margins. +Agreement between its fits would not establish calibration, a matched improvement over the legacy estimator, or live-agent non-regression. +Those remain subsequent Stage B/C/E requirements. + +The frozen scripts, inputs, reports and complete forecast histories are in `logs/uncertainty_boil_canonical_forecast_20260913/`. + +## Adapter validation and queued forecasts + +Adapter check `22690783` completed in 1:40 on a compute node. +Its generated and recorded-density continuations exactly reproduce the previously verified fixture, including fresh repeats and all saved fields. +It independently accumulates mixture means and variances, retains zero future support, and verifies exact recovery of a synthetic completed checkpoint without evaluating its target. +Nine negative controls reject missing or duplicated rows, changed weights, swapped roles, invalid bank indices, negative variance, unfinished checkpoints, altered checkpoint weights and changed inference identity. +The synthetic checkpoint is an API test, not a fitted physical posterior. +An earlier adapter check, `22690760`, failed because that synthetic fixture supplied an inconsistent prior identity; correcting the fixture allowed the unchanged identity guard to pass. + +The source identities and completed fixture certificates are pinned in `posterior-plan.json` before submission. +Forecast/verifier jobs `22690858_0` and `22690863_1` wait for successful completion of fits `22690118_0` and `_1` respectively. +Each requests four CPUs, 20 GB and at most two hours in `mit_preemptable`. +The paired report `22690864` waits for both forecast/verifier jobs to succeed. +Both fits were still running at submission, so there is no completed-population forecast result yet. diff --git a/docs/uncertainty/boil-joint-inference.md b/docs/uncertainty/boil-joint-inference.md index 3929a457d..50fe24cf9 100644 --- a/docs/uncertainty/boil-joint-inference.md +++ b/docs/uncertainty/boil-joint-inference.md @@ -115,3 +115,8 @@ Posterior numerical adequacy and the 132-action reserved suffix remain unevaluat Both pilots are running and have written their initialization checkpoints. Of 32 initial particles, seed 410 has eleven finite targets and seed 411 has seven. These initialization counts establish that sampling started with support; they do not establish adequate posterior exploration. + +The [canonical forecast continuation](boil-canonical-forecasts.md) now verifies fresh whole-history generation, separate future-conditioned density evaluation and literal memory across the fitting boundary. +Its completed-population adapter preserves original checkpoint weights and has passed native and mixture checks. +Full forecast/verifier jobs are queued behind the two fits, with a dependent paired stability report. +No completed-population prediction result is available yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index b5c53ab42..0ee053a88 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,13 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [canonical Boil forecast path](boil-canonical-forecasts.md) now passes its native fixture, independent full-history verifier and completed-population adapter checks. +The verifier checks twelve histories and 28,512 joint-transition factors, and rejects eight corrupted records. +The adapter independently checks weighted moments, zero future support and exact checkpoint recovery, rejects nine additional corruptions, and repeats generated and density-evaluation paths exactly. +The two Boil fits remain running; their reserved-132-action forecasts and independent verifiers are queued as `22690858_0` and `22690863_1`, followed by paired report `22690864`. +Forecast generation cannot read future observations, and the original fitted weights remain unchanged. +These are mechanical and offline comparison gates, not evidence of improved agent performance or permission to advance to live posterior use. + The [combined Domino assessment](domino-comparison-summary.md) now has all six completed 64-particle populations, their reserved-action forecasts and verified complete weighted histories. Matched local-only proposals fail all three initial stability screens; mixed local/full-range proposals reduce disagreement and pass both toppling screens, but still fail the position-mean screen. The fixed-initial-state approximation passes all three screens while showing worse position error and better toppling error than legacy on this recording. From 2a825540450490cf2f48a9b42557c208a62c793a Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 15:31:41 -0400 Subject: [PATCH 53/94] Add verified subclass Boil incumbent comparison protocol --- docs/uncertainty/boil-canonical-forecasts.md | 4 ++ docs/uncertainty/boil-incumbent-control.md | 72 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 ++ 3 files changed, 82 insertions(+) create mode 100644 docs/uncertainty/boil-incumbent-control.md diff --git a/docs/uncertainty/boil-canonical-forecasts.md b/docs/uncertainty/boil-canonical-forecasts.md index 6c3f1030f..00e4053c1 100644 --- a/docs/uncertainty/boil-canonical-forecasts.md +++ b/docs/uncertainty/boil-canonical-forecasts.md @@ -84,3 +84,7 @@ Forecast/verifier jobs `22690858_0` and `22690863_1` wait for successful complet Each requests four CPUs, 20 GB and at most two hours in `mit_preemptable`. The paired report `22690864` waits for both forecast/verifier jobs to succeed. Both fits were still running at submission, so there is no completed-population forecast result yet. + +The separate [incumbent control](boil-incumbent-control.md) supplies the same literal program through a verified subclass conversion and fits the identical recorded prefix with the existing full fitter. +Its selected-point forecast will be compared with the two posterior means after independent replay verification. +The control keeps its existing initial-state and objective assumptions, so this comparison alone does not isolate the estimator from the explicit posterior discrepancy extension. diff --git a/docs/uncertainty/boil-incumbent-control.md b/docs/uncertainty/boil-incumbent-control.md new file mode 100644 index 000000000..ec911b966 --- /dev/null +++ b/docs/uncertainty/boil-incumbent-control.md @@ -0,0 +1,72 @@ +# Boil incumbent fitter control + +September 13, 2026. +This adds the incumbent-fitter comparison needed alongside the [canonical Boil posterior forecasts](boil-canonical-forecasts.md). +The work remains in Stage B, with production behavior unchanged. + +## Fixed model and comparison scope + +The original noisy-sweep Boil artifacts declare no learnable parameters. +They remain incomplete-model controls, but cannot compare parameter estimators. +The canonical posterior experiment instead freezes an earlier filling/heating program with eight parameters, as described in the [historical model audit](historical-model-controls.md). +This control uses that same literal program, source recording and first 132 actions with 133 observations. +The remaining 132 actions are reserved for prediction assessment. + +The archived program uses recurrent rules, while the current MB fitting path treats a simulator subclass's declared parameters as physical parameters. +Calling the old rule dispatcher would therefore change which fitter and parameter-selection policy were exercised. +The experiment represents the selected historical program through the current subclass interface before invoking the incumbent fitter. +This is an offline conversion of this particular fixed program, not a new production simulator format. + +The subclass calls the original two rule functions without rewriting their dynamics. +Its declared model state retains the original latent dictionary and observable feature updates. +Only water volume, bubbling and spilled volume may be updated; physics commands are forbidden. +The original rules do not read their history argument, which the converter checks explicitly. +The generated feature values are exposed through the state readout without changing native body properties or adding physical forces. + +## Native parity check + +Job `22691098` completed in 31 seconds on a compute node. +It compares four complete 264-action trajectories against the archived literal-rule control: historical defaults, their independent repeat, and the lower and upper fill-rate bounds. +All observed predictions and every carried model-memory value agree exactly across the conversion. +The check executes 1,056 native actions. +It establishes parity for these complete test trajectories; a separate reader also checks the final fitted values before the control's predictions can enter the comparison. + +## Incumbent fit + +Job `22691175` runs the frozen `b09217bb3` incumbent fitting implementation through the same full rollout orchestrator used by subclass models. +It receives the model's eight declared parameter specifications and registry anchors, with the original noisy-sweep fit configuration. +Interval belief, fit-side noise handling and fit evidence are enabled. +Because the subclass declares model memory, the existing trajectory-preparation path retains the complete 132-action prefix. +The experiment verifies that preparation neither drops actions nor changes the supplied observation frames. +It retains the incumbent's residual scaling, robust objective, explainability trimming, optimization, identifiability reporting and parameter selection. +Parameters not selected by that policy retain their declared or registry values; a fit with no survivors is reported explicitly. + +The fit requests four CPUs, 20 GB and at most eight hours on `mit_preemptable`. +Per-process completed-world telemetry records native steps, including work performed in child processes. +The main report refreshes those totals at phase boundaries, so an in-progress report's counter is not a current total. +Allocation time and unfinished work remain separate costs. + +After fitting, the selected parameter values generate two fresh full-episode predictions from the public initial observation. +Those predictions must agree exactly. +Neither fitting nor generation reads the reserved future observations. + +## Verification and comparison + +Reader `22691200` waits for the fit to complete successfully. +It replays the selected parameters using the original native no-op simulator and literal post-step rules, independently of the converted subclass. +Every predicted observation and carried memory value must match the control's saved full trajectory. +Only after that check does it read the assessment-only future observations and calculate six feature errors and five native goal-predicate scores. + +Comparison `22691239` waits for this reader and the two-posterior report `22690864`. +It requires identical fitting-data and literal-program identities, the same held-out assessment artifact, and successful independent verification of every input. +It compares the incumbent's selected-point forecast with each joint posterior's predictive mean, retaining per-feature errors and goal-relevant Brier scores. +The incumbent's interval or ensemble planning policy is not represented by that selected-point row. + +This comparison does not isolate the numerical estimator alone. +The posterior arm also changes initial-state treatment and introduces explicit transition/output discrepancy; the incumbent retains its existing initialization and objective. +Those differences remain named experimental changes, not hidden adjustments to sensor noise. +The comparison does not establish calibration, planning equivalence or live-agent non-regression. +The matched ablations and later planning experiments remain requirements of the full simplification plan. + +Artifacts and frozen scripts are in `logs/uncertainty_boil_legacy_subclass_control_20260913/`. +At submission, the incumbent fit and both canonical posterior fits were running; no final comparison result was available. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 0ee053a88..1dc4e901a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -19,6 +19,12 @@ The two Boil fits remain running; their reserved-132-action forecasts and indepe Forecast generation cannot read future observations, and the original fitted weights remain unchanged. These are mechanical and offline comparison gates, not evidence of improved agent performance or permission to advance to live posterior use. +The [Boil incumbent control](boil-incumbent-control.md) now represents the same historical filling/heating program through the current subclass interface. +Four complete 264-action trajectories and all model memory match the archived literal-rule program exactly. +The incumbent full fitter is running as `22691175` on the same 132-action prefix, with interval belief, fit-side noise handling and fit evidence enabled. +Its independent literal-program verifier `22691200` and comparison against the two posterior forecasts `22691239` are queued behind their inputs. +This compares selected-point and posterior-mean predictions while retaining the explicit differences in initial-state treatment and discrepancy model; it is not an estimator-only ablation or an agent performance result. + The [combined Domino assessment](domino-comparison-summary.md) now has all six completed 64-particle populations, their reserved-action forecasts and verified complete weighted histories. Matched local-only proposals fail all three initial stability screens; mixed local/full-range proposals reduce disagreement and pass both toppling screens, but still fail the position-mean screen. The fixed-initial-state approximation passes all three screens while showing worse position error and better toppling error than legacy on this recording. From 3c4f3e968d05360e197af2a18021367448393907 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 15:50:36 -0400 Subject: [PATCH 54/94] Document verified prefix-only Balloons inference pilots --- .../balloons-composed-inference.md | 3 + docs/uncertainty/balloons-prefix-inference.md | 82 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 7 ++ 3 files changed, 92 insertions(+) create mode 100644 docs/uncertainty/balloons-prefix-inference.md diff --git a/docs/uncertainty/balloons-composed-inference.md b/docs/uncertainty/balloons-composed-inference.md index 9ccdc4a1d..a796a874c 100644 --- a/docs/uncertainty/balloons-composed-inference.md +++ b/docs/uncertainty/balloons-composed-inference.md @@ -5,6 +5,9 @@ This joins the previously tested [conditional velocity transition](transition-di It is an explicit stochastic model extension beside the deterministic reference and the production agent. It changes neither the sensor variances nor the exact tie, pop and clip observations. +The later [prefix-only inference experiment](balloons-prefix-inference.md) fits 64 actions and reserves 171 for prediction assessment without reusing the full-recording guide center. +Its results remain separate from the complete-recording support and inference results below. + ## Accounting for each observation After each native action, the existing transition diagnostic conditions Gaussian joint-position corrections on the nine exact current-joint readings. diff --git a/docs/uncertainty/balloons-prefix-inference.md b/docs/uncertainty/balloons-prefix-inference.md new file mode 100644 index 000000000..58352abfb --- /dev/null +++ b/docs/uncertainty/balloons-prefix-inference.md @@ -0,0 +1,82 @@ +# Balloons prefix-only inference + +September 13, 2026. +This is a separate Stage B experiment under the [uncertainty simplification plan](simplification-proposal.md). +It does not change the production agent or reinterpret the earlier [full-recording Balloons pilots](balloons-composed-inference.md). + +## Fitting boundary + +The earlier numerical seeds 300 and 301 fit all 235 actions in the first training episode. +Their guide center was selected using the complete recording. +The existing 64-action-prefix future-generation fixture uses that previously selected witness, so it validates mechanics rather than a prefix-fitted forecast. + +The new experiment fits only the first 64 actions and 65 observations, reserving the following 171 actions for later prediction assessment. +It removes future observations immediately after the trusted recording loader validates the complete file, before constructing conditioning factors, trajectories or a guide. +The target's observation lookup contains only steps 0 through 64. +It does not load the full-recording guide center or fitted populations. + +The simulator program remains the same frozen development artifact used by the earlier experiment. +That program itself was historically selected using training experience, so this is a conditional forecast comparison for a fixed development program, not an independent test of program synthesis or calibration on unseen tasks. + +## Support and guide construction + +The first guide uses original parameter-prior medians and a root centered on the analytically conditioned initial observation. +It retains the existing broad/local mixture, uniform motion-case selectors, inactive auxiliaries, conditional velocity-direction coordinates and density-corrected table-clearance proposal. +No future observation chooses the guide center. + +Job `22691470` completed in 1:04 on four compute CPUs. +It tests 32 candidates and repeats each fresh native prefix. +Six candidates fail initial support checks; 26 reach complete native prefix evaluation. +Fifteen have finite complete prefix factors, while eleven retain exact-event/output contradictions. +All repeated evaluations agree exactly, including rejected cases. +The check executes 3,328 native actions. +Finite support alone does not establish accurate predictions or adequate numerical exploration. + +The fitting guide is centered on candidate 5, selected by the largest corrected joint prefix target among these 32 candidates. +That selection uses only the fitting prefix, and the new guide retains its complete probability-density correction. +The original parameter and scene priors are unchanged. + +## Factorization and independent verification + +The fixed representation has 206 joint coordinates: ten parameters, 68 root coordinates and 128 conditional velocity-direction coordinates for the 64 actions. +It retains unused case auxiliaries; this is not a claim of 206 active continuous dimensions in every motion case. +The mixture selector adds one proposal coordinate. + +The base factor contains initial analytic conditioning constants and the inverse guide density, together with geometry and exact-event support. +The tempered factor contains the complete remaining output likelihood and all speed and joint-transition density factors. +The earlier full-recording implementation placed those transition density factors in the base. +Moving finite continuous density factors into the annealed term changes the numerical path while preserving the final target at temperature one. +Exact joint and speed observations remain represented through the same conditional coordinates at every temperature; no equality is replaced by a tolerance band. +Every exact event still has an indicator likelihood. + +Independent verifier `22691577` completed in 2:01. +It changes every future observation and action and verifies that the resulting fitting data are identical. +Access beyond the target's step-64 observation boundary fails. +It reconstructs all 32 audited candidates under the numerical-runtime overlay and matches every saved result exactly. +It then evaluates sixteen candidates from the new guide, seven of which have finite prefix factors. +The final target agrees exactly under the old and new factorization on these checked points, with maximum observed log-factor difference zero. +Serial and isolated parallel target evaluations also agree exactly. + +The initial analytic constant is 3.522918693126006, retaining the location, conditioned-joint, clip-yaw and initial box-rest factors. +The common unknown geometric normalizer is not used to claim absolute model evidence. +These checks validate implementation and support on the selected cases, not posterior adequacy. + +## Bounded numerical pilots + +Array `22691680` launches numerical seeds 620 and 621 after the verified prefix checks. +Each uses 64 particles, 32 cubic-spaced temperatures, eight moves per temperature and a cap of 16,448 target evaluations. +The blocked proposal covers all 207 proposal coordinates, including the mixture selector and four blocks of conditional velocity directions. +Every target evaluation reconstructs its complete native prefix in a fresh candidate world. +Four isolated workers evaluate each population without sharing simulator instances. + +Each pilot requests four CPUs, 20 GB and at most eight hours on `mit_preemptable`, pinned to the previously audited Balloons CPU type. +Complete-stage checkpoints preserve the original prior, numerical configuration, random state, particle weights, coordinates and cumulative evaluation count. +Returned native work and worker time are reported per allocation; interrupted work and previous allocations remain additional cost. +The launcher checks the independent certificate, source hashes and fitting boundary before sampling. + +These are numerical inference replicas on one development recording, not agent seeds. +They remain separate from seeds 300/301, hatch experiments and the MB/MF performance sweep. +The 171-action causal forecast path, prediction stability, inference cost and comparison against the incumbent remain unfinished. +No posterior is approved for planning merely because a pilot completes. + +Artifacts are in `logs/uncertainty_balloons_prefix64_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 1dc4e901a..6199b3d97 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -25,6 +25,13 @@ The incumbent full fitter is running as `22691175` on the same 132-action prefix Its independent literal-program verifier `22691200` and comparison against the two posterior forecasts `22691239` are queued behind their inputs. This compares selected-point and posterior-mean predictions while retaining the explicit differences in initial-state treatment and discrepancy model; it is not an estimator-only ablation or an agent performance result. +The separate [Balloons prefix experiment](balloons-prefix-inference.md) now fits only 64 actions and reserves 171 for later forecasts. +Its guide does not reuse a center or population selected from the complete recording. +Thirty-two candidate prefixes repeat exactly, with fifteen finite targets; independent verification confirms the future-data boundary, complete saved replays and exact serial/parallel targets. +Continuous transition normalizers are annealed with the output likelihood while preserving the same final conditional target and exact-observation representation. +The gated numerical pilots `22691680_0` and `_1` use seeds 620/621 and remain separate from the full-recording 300/301 pair. +Their posterior adequacy and reserved-action forecasts remain unassessed. + The [combined Domino assessment](domino-comparison-summary.md) now has all six completed 64-particle populations, their reserved-action forecasts and verified complete weighted histories. Matched local-only proposals fail all three initial stability screens; mixed local/full-range proposals reduce disagreement and pass both toppling screens, but still fail the position-mean screen. The fixed-initial-state approximation passes all three screens while showing worse position error and better toppling error than legacy on this recording. From 1515f09c2c4d5053f931b87b3a999fe04afa8ecf Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 15:52:19 -0400 Subject: [PATCH 55/94] Record verified Boil incumbent held-out predictions --- docs/uncertainty/boil-incumbent-control.md | 27 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 +++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/boil-incumbent-control.md b/docs/uncertainty/boil-incumbent-control.md index ec911b966..6de0268b2 100644 --- a/docs/uncertainty/boil-incumbent-control.md +++ b/docs/uncertainty/boil-incumbent-control.md @@ -70,3 +70,30 @@ The matched ablations and later planning experiments remain requirements of the Artifacts and frozen scripts are in `logs/uncertainty_boil_legacy_subclass_control_20260913/`. At submission, the incumbent fit and both canonical posterior fits were running; no final comparison result was available. + +## Completed incumbent control + +Fit `22691175` completed in 8:49 of allocation time. +The driver reports 515.05 seconds and 44,352 native actions including its two full prediction repeats. +The single complete fitting segment survives trimming. +Only the selected fill rate moves, from 0.0151 to 0.014247993659283687; the other selected parameters retain their declared values. +The fitting objective changes from 0.850290 to 0.814498 on the same surviving segment. + +Independent reader `22691200` completed in 19 seconds of allocation time, using another 264 native actions. +The original literal-rule replay matches every selected-point prediction and model-memory value exactly. +The saved source and verifier hashes were checked after completion. + +| Reserved-future reading | RMSE against noisy readings | RMSE against clean public readings | +| --- | ---: | ---: | +| Jug x, metres | 0.011978 | 0.001698 | +| Jug y, metres | 0.012859 | 0.005936 | +| Jug z, metres | 0.014010 | 0.006976 | +| Water volume | 0.071350 | 0.005384 | +| Bubbling level | 0.065171 | 0.019151 | +| Spilled level | 0.071813 | 0 | + +The filled, no-spill, burner-off and task-goal predicate predictions match all 132 reserved frames. +The boiled predicate differs on one frame, giving a Brier score of 1/132 for that deterministic prediction. +All five final predicates are predicted true and match the recorded final state. +These are forecast results, not a new solved agent seed. +The posterior fits and combined comparison remain unfinished, so no improvement claim follows from this control alone. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 6199b3d97..063752288 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -21,8 +21,10 @@ These are mechanical and offline comparison gates, not evidence of improved agen The [Boil incumbent control](boil-incumbent-control.md) now represents the same historical filling/heating program through the current subclass interface. Four complete 264-action trajectories and all model memory match the archived literal-rule program exactly. -The incumbent full fitter is running as `22691175` on the same 132-action prefix, with interval belief, fit-side noise handling and fit evidence enabled. -Its independent literal-program verifier `22691200` and comparison against the two posterior forecasts `22691239` are queued behind their inputs. +The incumbent full fitter ran as `22691175` on the same 132-action prefix, with interval belief, fit-side noise handling and fit evidence enabled. +The fit completed in 8:49, and independent literal-program verifier `22691200` completed in 19 seconds with every predicted frame and model-memory value matching exactly. +The selected-point forecast matches the task-goal predicate on all 132 reserved frames, while the boiled predicate differs on one frame. +The combined comparison `22691239` remains queued behind the posterior forecasts. This compares selected-point and posterior-mean predictions while retaining the explicit differences in initial-state treatment and discrepancy model; it is not an estimator-only ablation or an agent performance result. The separate [Balloons prefix experiment](balloons-prefix-inference.md) now fits only 64 actions and reserves 171 for later forecasts. From 3025b3b6196627e47c404269c3e3e0369f1e2b96 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 16:13:44 -0400 Subject: [PATCH 56/94] Document verified prefix-only Balloons future forecasts --- docs/uncertainty/balloons-prefix-forecasts.md | 93 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 7 ++ 2 files changed, 100 insertions(+) create mode 100644 docs/uncertainty/balloons-prefix-forecasts.md diff --git a/docs/uncertainty/balloons-prefix-forecasts.md b/docs/uncertainty/balloons-prefix-forecasts.md new file mode 100644 index 000000000..4dbdb4a64 --- /dev/null +++ b/docs/uncertainty/balloons-prefix-forecasts.md @@ -0,0 +1,93 @@ +# Balloons: forecasts after a prefix-only fit + +September 13, 2026. +This continues the [64-action fitting experiment](balloons-prefix-inference.md) toward the offline comparison required by the [simplification proposal](simplification-proposal.md). +The incumbent agent remains unchanged. + +## Data boundary and scope + +The fitted prefix contains 64 actions and 65 observations; the reserved suffix contains 171 actions. +The new fixture uses the candidate selected by the prefix-only support audit, rather than a center selected from the complete recording. +Its simulator program was historically selected using development training data, so this is a conditional forecast check for that fixed program, not an independent test of program synthesis. +Neither the fixture nor its numerical random seeds count as agent solve-rate runs. + +The implementation is frozen under `logs/uncertainty_balloons_prefix_forecast_20260913/`. +Its manifest identifies the fitting worker, prefix guide and verification reports, action recording, and observation/transition modules. +The forecast snapshot adds the already implemented future-generation and output-likelihood methods without changing the live fitting snapshots. +Exact reproduction of the saved prefix candidate and its complete likelihood is an execution gate for the fixture. + +## Two different computations + +### Generate predictions + +A fresh simulator reconstructs the candidate initial state and executes all 235 actions continuously. +The first 64 actions reproduce the fitted conditional history, including the sampled velocity directions and exact joint/speed observations. +For the remaining actions, controlled joint positions receive draws from the declared Gaussian transition discrepancy, and box velocity follows the original rest/Gaussian mixture. +The generator has no lookup entries for future observations. +Native attachment changes, balloon bursts and simulator memory continue through the whole history. +It does not restore an arbitrary observation-boundary snapshot. + +The output model conditions its correlated error process on prefix observations, then draws all 58 future observation fields. +The original sensor variances remain fixed. +Initial-observation evidence and the prefix speed factors are not counted a second time as output evidence. +The artifact records native band membership, rest, burst, the `InBand` predicate, and the evaluator's no-burst goal condition separately. +These are predictions along the supplied action sequence, not an executed agent outcome. + +### Evaluate future density + +A separate computation receives the observed suffix explicitly. +It conditions each future transition on the observed controlled joint positions and box speed, retains their Gaussian and radial density factors, and samples the unobserved velocity directions from the conditional direction law. +The complete output likelihood retains the other exact observations, including discrete events, while integrating the declared correlated output error. +Each such history supplies an importance weight for integration over future velocity directions. + +The fixture averages density contributions over eight complete conditional-history draws, including zero-support histories in the denominator. +It also reports two four-draw estimates, importance effective sample size and maximum normalized weight. +This is a finite Monte Carlo estimate, not an automatically adequate marginal likelihood. +A single path's score is never reported as the integrated future likelihood. +Zero contributions do not justify declaring the model's full conditional support empty. +Density-evaluation histories never enter generated forecast means or change the prefix fitting weights. + +## Validation and acceptance + +The fixture checks four generated histories and eight conditional-density histories, with selected full-history repetitions. +An independent reader checks the serialized transitions, random draws, exact readbacks, native goal semantics, output composition, and the density denominator. +It also compares radial densities against a separate noncentral-chi calculation and deliberately corrupts saved fields to check rejection. +The reader repeats one complete native history of each kind in a separate process. +A separate initialization audit holds the physical candidate fixed while changing robot numeric readouts that should be overwritten during initialization. + +These checks establish forecast mechanics only. +The completed prefix populations must still be evaluated without changing their weights, assessed for numerical stability and compared with the incumbent fitter's predictions. +The physical-support and predictive-adequacy gates remain open until their own evidence is available. + +## Completed native fixture + +Job `22692309` completed on node1390 in 53 seconds, executing 3,824 native actions including repeated histories and the prefix reference. +All twelve histories reproduce the saved 64-action prefix exactly under the forecast snapshot. +The four generated suffixes each contain 171 observations with all 58 fields. + +Independent verifier `22692420` completed in 36 seconds. +It checked 25,380 joint factors or unconditional draws, 2,136 radial-density evaluations against a separate noncentral-chi calculation, all recorded goal fields, and exact replay of one complete generated and one complete conditional-density history. +It rejected eight corrupted artifacts and verified that zero-density paths stay in the integration denominator. +The source report is `fixture-22692309.json`; the independent certificate is `verify-22692420.json` in the experiment bundle. + +All eight conditional-density draws for this preliminary prefix candidate have zero likelihood on the actual reserved suffix. +Each predicts `balloon0.popped = 1` while the recorded value remains zero, with the first event disagreement between actions 97 and 114. +This is an observed failure of these eight candidate histories, not proof that no compatible history exists and not a result for either running prefix posterior population. +Small raw robot-pose differences in the diagnostic mismatch list are handled by the declared output discrepancy; they should not be confused with the uncompromised burst-event contradictions. + +The first initialization audit, job `22692482`, stopped when its deliberately zeroed robot orientation violated the existing reconstruction guard. +That was a diagnostic setup failure, not an agent failure. +The revised audit `22692596` completed in 27 seconds and reports rejected initialization separately from changed physical trajectories. +All five small perturbations of robot position, orientation or finger readout leave the complete 64-action trajectory exactly unchanged, including repeated reconstruction. +The deliberately inconsistent zero orientation is rejected reproducibly. +This rules out dependence on those five tested readout perturbations for this candidate; it does not establish unrestricted initialization invariance. + +Two additional finite-density checks completed as job `22692681`. +They retain the recorded future joint/speed inputs and conditional velocity draws while replacing the other future outputs with explicit synthetic draws on those same physical histories. +The physical trajectories and all transition factors remain exactly unchanged, and the independent reader verifies the finite full-history-minus-prefix output factors and their composition with transition densities. +This adds 4,230 joint-factor checks and 470 independent radial-density checks. +These synthetic cases test the finite-density calculation; they are not held-out evidence or unconditional physical forecasts. +The first attempt, `22692619`, failed during imports because the fitting directory shadowed the forecast verifier's module name; the corrected attempt loads the identified verifier by its explicit file path. + +The fixture, independent reader, initialization audit and finite-density certificate are indexed by `verification-manifest.json` in the experiment bundle. +The full plan remains at Stage B, with Stage A physical-support and numerical gates still open. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 063752288..e9cbca490 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,13 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Balloons prefix forecast fixture](balloons-prefix-forecasts.md) now covers all 171 reserved actions using a candidate selected only from the 64-action prefix. +Twelve native histories reproduce that prefix exactly, and the independent reader verifies 25,380 joint factors or draws, 2,136 radial-density evaluations, full native replay and eight corruption rejections. +Generation has no future-observation lookup; density evaluation separately averages conditional-history contributions while retaining zero-support draws. +All eight density histories for this preliminary candidate contradict a recorded burst event, so their likelihood contributions are zero; this does not assess the running fitted populations or prove that conditional support is empty. +Five small initialization-readout perturbations preserve the complete prefix exactly, while a deliberately inconsistent orientation is rejected by the reconstruction guard. +The next step is to apply the verified forecast mechanics to completed prefix populations with their original weights and assess numerical stability and incumbent prediction differences. + The [canonical Boil forecast path](boil-canonical-forecasts.md) now passes its native fixture, independent full-history verifier and completed-population adapter checks. The verifier checks twelve histories and 28,512 joint-transition factors, and rejects eight corrupted records. The adapter independently checks weighted moments, zero future support and exact checkpoint recovery, rejects nine additional corruptions, and repeats generated and density-evaluation paths exactly. From d6e1e184f4bac3cb069960eac65ac5cd2d31d986 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 16:50:00 -0400 Subject: [PATCH 57/94] Add verified Balloons population and incumbent forecast protocols --- .../uncertainty/balloons-incumbent-control.md | 54 +++++++++++++++++++ docs/uncertainty/balloons-prefix-forecasts.md | 29 ++++++++++ docs/uncertainty/implementation-progress.md | 10 +++- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/balloons-incumbent-control.md diff --git a/docs/uncertainty/balloons-incumbent-control.md b/docs/uncertainty/balloons-incumbent-control.md new file mode 100644 index 000000000..867e7c6e3 --- /dev/null +++ b/docs/uncertainty/balloons-incumbent-control.md @@ -0,0 +1,54 @@ +# Balloons: incumbent fitter control on the reserved suffix + +September 13, 2026. +This supplies the existing-fitter comparison for the [prefix posterior forecasts](balloons-prefix-forecasts.md). +The experiment is frozen under `logs/uncertainty_balloons_incumbent_prefix_20260913/`. +It does not change the production agent. + +## Shared inputs and retained differences + +Both arms receive the same first 64 actions and 65 observations from the original noisy Balloons development recording, with 171 actions reserved for future prediction. +Both use the same selected simulator subclass, its ten declared physical parameters and the identified historical runtime. +The program was previously selected using training data, so the comparison is conditional on that fixed development program, not an independent program-synthesis evaluation. +No local parameter-override file is present in either isolated runtime. + +The incumbent uses its full existing fitting pipeline, with interval belief, fit-side noise handling and fit evidence enabled. +Its optimizer, trimming, parameter-selection rules and observed-state initialization remain intact. +The posterior arm uses the separately declared joint initial-state and transition/output model. +This is a comparison of the complete configurations, not an estimator-only ablation. + +The incumbent preparation produces three overlapping action windows, using zero-based half-open indices: + +| Action window | Actions | Initial observation treatment | +| --- | --- | --- | +| `[0, 21)` | 21 | Original initial frame. | +| `[15, 64)` | 49 | Average noisy features over observation frames 8 through 15. | +| `[63, 64)` | 1 | Average noisy features over observation frames 56 through 63. | + +The eight-frame averages are arithmetic for ordinary noisy features and circular for angular features. +Exact features retain their original values. +The driver verifies every following frame and action against its original recording window, and independently reconstructs each averaged initial feature from the identified source observations. +The posterior likelihood uses the complete 64-action history once; the incumbent's overlapping windows and averaging are preserved for comparison. + +## Native validation and current execution + +Preflight `22693164` completed in 30 seconds, executing 1,410 native actions. +It compares three complete 235-action trajectories at the declared default, low-gold-lift and high-gold-lift parameter settings. +The normal rollout helper and an independent manual reset/pin/zero-velocity/step lifecycle agree exactly in every case, and the lift changes alter predictions. +Future observations are not used in these trajectories. + +The initial fitting attempt `22693556` stopped before fitting because the new comparison driver incorrectly required averaged segment starts to equal raw observations. +Its dependent verifier and comparison jobs were cancelled by the scheduler. +This was a comparison-harness failure, not an agent outcome. +The corrected driver verifies raw successor frames and actions separately from the intended rest-window averages. +Those checks pass, and fitting job `22693806` is running on a compute node. + +| Job | Purpose | +| --- | --- | +| `22693806` | Full incumbent fit, followed by two repeated causal forecasts. | +| `22693822` | After fitting: independently reproduce the selected forecast and score the reserved 171 actions. | +| `22693842` | After incumbent verification and the posterior pair: compare the verified predictions. | + +The future metrics match the posterior reports: thirteen position/speed features and fourteen event indicators. +The control reports selected-point predictions, not the incumbent's complete interval or planning ensemble. +Final comparisons must retain numerical-stability warnings and cannot be treated as agent solve-rate or sample-efficiency results. diff --git a/docs/uncertainty/balloons-prefix-forecasts.md b/docs/uncertainty/balloons-prefix-forecasts.md index 4dbdb4a64..a19eeaceb 100644 --- a/docs/uncertainty/balloons-prefix-forecasts.md +++ b/docs/uncertainty/balloons-prefix-forecasts.md @@ -91,3 +91,32 @@ The first attempt, `22692619`, failed during imports because the fitting directo The fixture, independent reader, initialization audit and finite-density certificate are indexed by `verification-manifest.json` in the experiment bundle. The full plan remains at Stage B, with Stage A physical-support and numerical gates still open. + +## Complete-population forecast adapter + +The adapter now recovers a completed fitting checkpoint without evaluating the target again, and requires its entire returned result to match the saved fit. +It preserves each original particle weight and maps the 207 proposal coordinates back to the 206-dimensional joint chart, checking the stored physical parameters and proposal correction. +Every forecast must reproduce the particle's saved 64-action likelihood before its future predictions are used. +The independent population reader additionally reconstructs a native prefix for each positive-weight particle and verifies all saved histories and summaries. + +Each positive-weight particle receives two independent banks of four generated suffixes and eight separate conditional-density draws. +The generated histories determine predictive means, variances, event probabilities and first-occurrence curves. +The density histories only estimate future likelihood, retaining all zero contributions and the original particle weights. +The report covers box position and speed, each balloon's position, the five global event predicates, individual attachment/burst events and clip states. +It reports within-population bank disagreement and a separate comparison between the two numerical fitting seeds. +These finite budgets do not automatically establish numerical adequacy. + +Adapter preflight `22692938` completed in 34 seconds, with 1,004 native actions. +It independently checks the nonuniform weighted mean and variance, event probabilities and first-occurrence curves, zero-density and mixed-density denominators, and completed-checkpoint recovery on an explicitly synthetic target. +It also repeats native generation and conditional-density adapter paths against the previously verified fixture. +Twelve malformed cases are rejected, including missing or duplicated histories, changed weights, invalid densities, incorrect event curves and inconsistent checkpoints. +The certificate is `adapter-22692938.json`; `posterior-submitted.json` identifies the submitted scripts and inputs. + +| Job | Dependency and purpose | +| --- | --- | +| `22693026` | After fit `22691680_0`: forecast seed620's completed population, then independently verify it. | +| `22693027` | After fit `22691680_1`: forecast seed621's completed population, then independently verify it. | +| `22693044` | After both forecast/verifier jobs: compare the two numerical populations. | + +The [incumbent control](balloons-incumbent-control.md) runs the existing fitter on the same supplied prefix and compares its selected-point forecast with these population forecasts. +That comparison retains the differences in initial-state treatment, segmentation and discrepancy assumptions explicitly. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index e9cbca490..8264955a8 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,12 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. +Forecast jobs `22693026` and `22693027` are gated behind the two live prefix fits; each is followed by independent artifact verification, then paired report `22693044`. +Every positive-weight particle keeps its original weight, receives eight generated suffixes and eight separate conditional-density draws, and must reproduce its saved prefix likelihood. +The [Balloons incumbent control](balloons-incumbent-control.md) passes three complete native lifecycle comparisons and is fitting as `22693806`, with independent verification and comparison queued. +Its overlapping windows and rest-averaged starts remain intact; the comparison driver now verifies those averages after correcting its initial overly strict raw-frame check. + The [Balloons prefix forecast fixture](balloons-prefix-forecasts.md) now covers all 171 reserved actions using a candidate selected only from the 64-action prefix. Twelve native histories reproduce that prefix exactly, and the independent reader verifies 25,380 joint factors or draws, 2,136 radial-density evaluations, full native replay and eight corruption rejections. Generation has no future-observation lookup; density evaluation separately averages conditional-history contributions while retaining zero-support draws. @@ -22,7 +28,9 @@ The next step is to apply the verified forecast mechanics to completed prefix po The [canonical Boil forecast path](boil-canonical-forecasts.md) now passes its native fixture, independent full-history verifier and completed-population adapter checks. The verifier checks twelve histories and 28,512 joint-transition factors, and rejects eight corrupted records. The adapter independently checks weighted moments, zero future support and exact checkpoint recovery, rejects nine additional corruptions, and repeats generated and density-evaluation paths exactly. -The two Boil fits remain running; their reserved-132-action forecasts and independent verifiers are queued as `22690858_0` and `22690863_1`, followed by paired report `22690864`. +Both Boil fits completed all 32 stages: numerical seeds410/411 used 7,455/7,317 target evaluations, respectively. +Their reserved-132-action forecast/verifier jobs `22690858_0` and `22690863_1` are running, followed by queued paired report `22690864`. +Both completed populations descend from one initial ancestor, so completion alone does not establish adequate exploration. Forecast generation cannot read future observations, and the original fitted weights remain unchanged. These are mechanical and offline comparison gates, not evidence of improved agent performance or permission to advance to live posterior use. From 1bc7a11228bac7b01314b2026b82049df40650f9 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:06:41 -0400 Subject: [PATCH 58/94] Diagnose structural glue-law and face-selection errors in Bridge --- docs/uncertainty/bridge-glue-attribution.md | 78 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 5 ++ 2 files changed, 83 insertions(+) create mode 100644 docs/uncertainty/bridge-glue-attribution.md diff --git a/docs/uncertainty/bridge-glue-attribution.md b/docs/uncertainty/bridge-glue-attribution.md new file mode 100644 index 000000000..fa5295da6 --- /dev/null +++ b/docs/uncertainty/bridge-glue-attribution.md @@ -0,0 +1,78 @@ +# Bridge glue model: structural and geometry errors + +This Stage B development diagnostic separates a wrong glue-update law from accumulated native trajectory error. +It extends the [historical model audit](historical-model-controls.md), retaining both the original no-op and transferred-program controls. +It changes neither the production agent nor the running inference experiments. + +## A parameter change cannot reproduce the recorded glue sequence + +Four faces in the frozen Bridge training recording show consecutive readings `0 → 0.2 → 0.4 → 1`. +Glue is an exact observed channel in this recording. +The transferred program implements deposition as `min(1, previous_level + glue_rate)`. +Its other glue mutations drain a partial level to zero, consume glue when a bond forms, or retain an existing level. +The rule memory persists between actions and is not reset from later observations. + +Starting from zero, the first positive reading forces `glue_rate = 0.2`. +That rate reproduces the second reading but gives `0.6`, not `1`, at the third deposition. +Drainage, retention and bond consumption cannot supply the missing increase. +This contradiction holds even if geometry is allowed to choose deposition or the other transitions arbitrarily on each step. +It therefore rules out fitting this exact sequence by changing any of the transferred program's parameters or initial physical states while preserving its update law and memory contract. +It does not establish that every possible learned Bridge program is inconsistent. + +| Face | Four consecutive observation steps | +| --- | --- | +| `span0.glue_end_a` | 86, 87, 88, 89 | +| `span1.glue_end_b` | 121, 122, 123, 124 | +| `span0.glue_end_b` | 155, 156, 157, 158 | +| `span2.glue_end_a` | 199, 200, 201, 202 | + +An independent check executes the actual historical deposition function with continuously eligible geometry. +Its default rate produces `0, 0.35, 0.7, 1`; rate `0.2` produces `0, 0.2, 0.4, 0.6000000000000001`. +The floating-point result has the same contradiction as the exact arithmetic argument. +The source's own earlier decision record describes the observed `0.2, 0.4, 1` progression, so the implemented constant-increment law also differs from that recorded modeling intent. + +## Correcting saturation alone is insufficient + +A diagnostic one-line revision uses rate `0.2` and latches to `1` when accumulated progress reaches `0.6`. +That revision reproduces the isolated progression. +It is chosen after inspecting development evidence, is not a fitted posterior model, and has not been accepted as a replacement. + +To separate geometry drift from glue-law errors, each program is also evaluated on all 1,186 recorded post-action public geometries. +The six cases use either the noisy public geometry or its clean public projection, with the historical defaults, rate `0.2` alone, or the saturation revision. +Only the initial glue readings enter the model memory. +Subsequent glue readings are replaced by the model's own carried levels before invoking its rules; dwell counters and bonds also persist. +Attachment commands are counted but not executed, because geometry is supplied from the recording. +No private simulator state enters these rule evaluations. + +| Rule variant | Mismatched glue readings with noisy geometry | Mismatched glue readings with clean geometry | +| --- | ---: | ---: | +| Historical defaults | 4,444 | 3,816 | +| Historical law, rate `0.2` | 3,382 | 4,442 | +| Rate `0.2`, latch at `0.6` | 4,442 | 3,814 | + +Each column compares 17,790 face readings, including unchanged readings between events. +These counts are descriptive errors, not independent statistical trials. +They are also not causal forecast scores: later recorded geometry is explicitly supplied. +Lower counts from an incorrect law can reflect accidental compensation between errors and do not justify selecting that law. + +The clean-geometry cases still show incorrect face selection and event timing. +At steps 87-89, the program deposits on `span0.glue_top`, while the recorded change is on `span0.glue_end_a`. +At steps 122-124, it similarly maintains glue on `span1.glue_top` instead of the observed `span1.glue_end_b`. +Its first deposition on `span0.glue_end_a` starts at step 57, one step before the recorded change, even with clean geometry. +Consequently, neither denoising the geometry nor fixing the saturation progression is sufficient for this transferred program. +The diagnosed face-selection and timing errors do not, by themselves, prove that all geometric parameter settings fail. + +## Verification and next work + +The frozen bundle is `logs/uncertainty_bridge_glue_attribution_20260913/`. +Job `22695084` completed the six cases in a 35-second allocation on `node1412` under `mit_preemptable`. +Independent verifier `22695164` completed in 25 seconds. +It reconstructs public feature views without a native world, calls the two rules directly instead of using the rule-dispatch helper, and reproduces every glue prediction and attachment-command count across 7,116 rule steps. +It independently checks error totals, all four recorded witnesses, and the actual deposition-function traces. +These jobs perform no recorded native actions and are neither agent seeds nor parameter-inference runs. +The plan and verification manifest pin source programs, recordings, scripts and completed artifacts. + +The next supported positive Bridge case requires a revised glue model, including deposition geometry and observation timing, before fitting can become meaningful. +Validate any revision on causal native trajectories and freeze it for both estimator arms; the geometry-conditioned diagnostic cannot establish attachment physics or held-out prediction quality. +Keep this historical program as an explicit inconsistent-model case rather than concealing its exact-output errors with additional sensor noise or a larger sampler. +Stage A/B acceptance remains open, and production continues to use the incumbent estimator. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 8264955a8..24eba4bce 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,11 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Bridge glue attribution](bridge-glue-attribution.md) now proves a structural contradiction in the transferred parameterized program, beyond the original no-op control. +Four exact `0, 0.2, 0.4, 1` progressions cannot arise from its constant-increment deposition law for any parameter setting under the preserved memory contract. +Six geometry-conditioned development cases and an independent 7,116-step direct-rule verifier complete; clean recorded geometry still exposes wrong face selection and timing, and a saturation correction alone is insufficient. +This identifies model revision as the next Bridge action; these conditional diagnostics are not causal forecasts or an accepted posterior. + The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. Forecast jobs `22693026` and `22693027` are gated behind the two live prefix fits; each is followed by independent artifact verification, then paired report `22693044`. Every positive-weight particle keeps its original weight, receives eight generated suffixes and eight separate conditional-density draws, and must reproduce its saved prefix likelihood. From 5243d2e47068d9f825062e83f2ec216c5793f3ca Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:13:16 -0400 Subject: [PATCH 59/94] Evaluate prefix-fitted Bridge deposition geometry hypotheses --- docs/uncertainty/bridge-glue-attribution.md | 35 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 2 ++ 2 files changed, 37 insertions(+) diff --git a/docs/uncertainty/bridge-glue-attribution.md b/docs/uncertainty/bridge-glue-attribution.md index fa5295da6..0e8a7f141 100644 --- a/docs/uncertainty/bridge-glue-attribution.md +++ b/docs/uncertainty/bridge-glue-attribution.md @@ -62,6 +62,41 @@ Its first deposition on `span0.glue_end_a` starts at step 57, one step before th Consequently, neither denoising the geometry nor fixing the saturation progression is sufficient for this transferred program. The diagnosed face-selection and timing errors do not, by themselves, prove that all geometric parameter settings fail. +## Separating face choice from deposition eligibility + +A follow-up tests choosing the face whose center is nearest to the bottle tip, followed by a single distance threshold for deposition. +It uses the historical bottle-tip offset and block-face geometry, without reading private domain mechanics. +The threshold is chosen by exhaustive search over prediction change points within 10 cm, using only actions 1-124 and minimizing incorrect deposition labels. +Already saturated faces are excluded from that label loss because another deposition would be invisible in their glue reading. +No later glue levels enter threshold selection. +Each geometry source is evaluated separately using either its post-action or pre-action frame. + +On clean post-action geometry, the nearest face matches the observed face at all thirteen positive glue transitions across the recording. +This resolves the face-choice ambiguity at those positive examples, but does not establish a correct deposition trigger. +The one-distance trigger cannot reproduce the fitting-prefix labels in any of the four cases. + +| Supplied geometry | Frame phase | Selected radius | Prefix TP / FP / FN | Suffix TP / FP / FN | +| --- | --- | ---: | --- | --- | +| Noisy | Post-action | 0.040166 m | 7 / 6 / 0 | 3 / 1 / 3 | +| Noisy | Pre-action | 0 m | 0 / 0 / 7 | 0 / 0 / 6 | +| Clean | Post-action | 0 m | 0 / 0 / 7 | 0 / 0 / 6 | +| Clean | Pre-action | 0 m | 0 / 0 / 7 | 0 / 0 / 6 | + +TP counts correctly predicted positive transitions, FP counts spurious deposition labels, and FN counts missed positive transitions. +The prefix has 1,825 informative face readings and seven positive transitions; the suffix has 13,702 informative readings and six positive transitions. +The zero-radius solutions predict no deposition and miss every positive transition; they are failures, not acceptable low-error models. +All later geometry remains supplied from the recording, and the suffix had already been inspected during prior development diagnostics, so this is neither a causal prediction test nor an untouched validation set. +The failures do not prove that uncertain latent geometry or a richer physical deposition model has no support. + +Compute job `22695397` completed in 16 allocation seconds, followed by independent verifier `22695437` in 17 seconds, both on `node1412` under `mit_preemptable`. +The verifier uses an independent rotation implementation to check 16,320 finite face distances, agreeing within `2.23e-16` m. +It independently scores all 148 tested thresholds, verifies that only prefix labels select them, and checks all four candidates' predicted labels and confusion counts. +The frozen bundle is `logs/uncertainty_bridge_deposition_geometry_20260913/`. + +The next geometry hypothesis should preserve the distinction between choosing a face and determining whether a drip reaches it, including height and side-of-plane eligibility. +A scalar distance alone is insufficient on these supplied trajectories. +Any revised program still requires causal native replay and a separately frozen estimator comparison. + ## Verification and next work The frozen bundle is `logs/uncertainty_bridge_glue_attribution_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 24eba4bce..67c912253 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -15,6 +15,8 @@ The full plan remains incomplete, and the incumbent estimator remains the produc The [Bridge glue attribution](bridge-glue-attribution.md) now proves a structural contradiction in the transferred parameterized program, beyond the original no-op control. Four exact `0, 0.2, 0.4, 1` progressions cannot arise from its constant-increment deposition law for any parameter setting under the preserved memory contract. Six geometry-conditioned development cases and an independent 7,116-step direct-rule verifier complete; clean recorded geometry still exposes wrong face selection and timing, and a saturation correction alone is insufficient. +The follow-up nearest-face hypothesis identifies the observed face at all thirteen positive clean-geometry transitions, but a single proximity threshold fails the fitting-prefix labels in all four geometry/phase cases. +An independent rotation and scalar-loss reader verifies 16,320 distances and 148 tested thresholds; face choice and deposition eligibility need separate treatment before causal replay. This identifies model revision as the next Bridge action; these conditional diagnostics are not causal forecasts or an accepted posterior. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. From cadc4832cbecec66d4adc706a44c61fb9fa28a81 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:24:01 -0400 Subject: [PATCH 60/94] Validate revised Bridge glue dynamics in causal native replay --- docs/uncertainty/bridge-glue-attribution.md | 66 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 5 ++ 2 files changed, 71 insertions(+) diff --git a/docs/uncertainty/bridge-glue-attribution.md b/docs/uncertainty/bridge-glue-attribution.md index 0e8a7f141..c07c9adbd 100644 --- a/docs/uncertainty/bridge-glue-attribution.md +++ b/docs/uncertainty/bridge-glue-attribution.md @@ -97,6 +97,72 @@ The next geometry hypothesis should preserve the distinction between choosing a A scalar distance alone is insufficient on these supplied trajectories. Any revised program still requires causal native replay and a separately frozen estimator comparison. +## Nearest eligible face with the historical drip conditions + +The next diagnostic retains the historical height, outward-plane and footprint conditions, then compares its original score against distance to the eligible face's center. +Both scoring rules receive the same 713 combinations of drip radius and maximum height, in 2.5 mm increments within the historical parameter bounds. +Only actions 1-124 select each parameter pair; clean/noisy geometry and pre/post-action frames remain separate cases. +This is a finite grid comparison, not exhaustive continuous optimization or a parameter posterior. + +| Geometry and phase | Face score | Radius / height (m) | Prefix TP / FP / FN | Suffix TP / FP / FN | +| --- | --- | --- | --- | --- | +| Noisy, post-action | Historical | 0.005 / 0.005 | 0 / 2 / 7 | 0 / 3 / 6 | +| Noisy, post-action | Nearest eligible center | 0.0175 / 0.0125 | 6 / 2 / 1 | 3 / 3 / 3 | +| Noisy, pre-action | Historical | 0.005 / 0.005 | 0 / 2 / 7 | 0 / 3 / 6 | +| Noisy, pre-action | Nearest eligible center | 0.0175 / 0.0125 | 4 / 3 / 3 | 3 / 3 / 3 | +| Clean, post-action | Historical | 0.0125 / 0.005 | 1 / 4 / 6 | 0 / 0 / 6 | +| Clean, post-action | Nearest eligible center | 0.005 / 0.0225 | 6 / 0 / 1 | 6 / 0 / 0 | +| Clean, pre-action | Historical | 0.005 / 0.005 | 0 / 4 / 7 | 0 / 0 / 6 | +| Clean, pre-action | Nearest eligible center | 0.005 / 0.0275 | 5 / 0 / 2 | 6 / 0 / 0 | + +With clean post-action geometry, nearest-eligible-center scoring reproduces all six later positive transitions without false positives, but misses the isolated partial deposition at prefix step 58. +With noisy post-action geometry, three positive transitions violate the historical fixed outward-plane condition regardless of the two fitted thresholds: steps 89, 158 and 200. +The negative outward offsets there are approximately 8.74, 5.02 and 15.06 mm, exceeding the fixed 5 mm tolerance. +Those conflicts on noisy point observations do not prove that a joint latent-geometry model has no support. +They do show why fitting geometric thresholds against one noisy pose sequence is not a complete uncertainty treatment. + +Job `22695592` completed in 19 allocation seconds. +Verifier `22695648` completed in 21 seconds, independently reconstructing the grid search and invoking the actual historical deposition function, with only its score expression changed for the nearest-center cases. +All 5,704 grid scores and 9,488 selected-case rule steps agree. +The frozen bundle is `logs/uncertainty_bridge_deposition_eligibility_20260913/`. +These remain geometry-conditioned labels, with the same previously inspected development suffix and no attachment-physics validation. + +A separate candidate now combines nearest-eligible-center scoring with the progress latch from the structural-law diagnostic. +Native replay uses the same public noisy initial state and recorded actions as the historical control, with no later observation correction. +It compares the original no-op, the noisy-prefix parameter pair twice, a clean-prefix-derived parameter pair, and the historical geometry parameter pair. +The clean-derived pair is an explicitly diagnostic parameter choice; the native initial state is still noisy. +Bond geometry and dwell rules remain those of the historical program. +The native job is `22695718`, followed by artifact verifier `22695733`, under `logs/uncertainty_bridge_revised_native_20260913/`. +This is an explicitly revised development program, not the production agent or an accepted estimator comparison. + +Both jobs have completed: native replay used 2:17 of allocation time and 5,930 recorded native actions across five complete trajectories. +The original no-op matches its archived trajectory exactly, and the revised default's two runs match every prediction, command count and saved model-memory value. +The independent reader verifies all five histories, parameter pairs, error calculations and retained exact-output contradictions. + +| Causal native replay | Mismatched glue readings | Steps issuing attachment commands | +| --- | ---: | ---: | +| Original no-op | 2,272 | 0 | +| Earlier transferred program, historical defaults | 3,695 | 347 | +| Revised program, noisy-prefix geometry parameters | 23 | 599 | +| Revised program, clean-prefix geometry parameters | 21 | 599 | +| Revised program, historical geometry parameters | 32 | 599 | + +The earlier transferred-program row comes from the independently verified historical control on the same recording and native runtime. +All rows use the same public noisy initial state, so the large reduction does not result from supplying later observed geometry during native rollout. +No estimator has been fitted in this comparison; this improvement concerns a changed program on a development recording. +It is not evidence of better solve rate or better posterior inference. + +The revised default's 23 glue errors have a simple decomposition. +Three occur at steps 87-89, when one deposition starts one action late. +The remaining twenty are two pairs of mating glue readings consumed five actions late at each bond event: predicted consumption at 588 and 840 versus recorded consumption at 583 and 835. +The unchanged default bond dwell was not informed by the 124-action fitting prefix, which contains no bond event. +Do not interpret its later point-prediction error as a demonstrated failure of posterior inference. +The revised trajectories still contradict 22 exact observed channels overall, including robot-related channels, so reduced glue error does not establish a supported complete sensor-only likelihood. + +The result supplies a substantially more useful candidate for the next Bridge comparison, while keeping the failed original program as an explicit model-inconsistency control. +Next preserve this candidate through the current subclass interface with complete native parity, then define its joint state/parameter target and causal fitting split before comparing estimators. +In particular, the exact partial glue readings constrain the continuous rate parameter; drawing rates from an ordinary continuous proposal and rejecting unequal readings is not a valid conditional construction. + ## Verification and next work The frozen bundle is `logs/uncertainty_bridge_glue_attribution_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 67c912253..9e5bd67da 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -18,6 +18,11 @@ Six geometry-conditioned development cases and an independent 7,116-step direct- The follow-up nearest-face hypothesis identifies the observed face at all thirteen positive clean-geometry transitions, but a single proximity threshold fails the fitting-prefix labels in all four geometry/phase cases. An independent rotation and scalar-loss reader verifies 16,320 distances and 148 tested thresholds; face choice and deposition eligibility need separate treatment before causal replay. This identifies model revision as the next Bridge action; these conditional diagnostics are not causal forecasts or an accepted posterior. +The next [Bridge eligibility and native revision](bridge-glue-attribution.md#nearest-eligible-face-with-the-historical-drip-conditions) now completes eight conditional grid cases and five full causal native trajectories. +Nearest-eligible-face scoring plus a progress latch reduces glue mismatches from 3,695 in the transferred historical program to 23 with noisy-prefix geometry parameters, on the same 1,186-action recording and public noisy initial state. +Independent readers verify 5,704 grid scores, 9,488 rule steps, all five native artifacts, archived no-op parity and exact repetition of the revised default. +The remaining default glue errors are three delayed deposition readings and two bond-consumption events delayed five actions; 22 exact channels still disagree overall. +This is a development program revision, not an estimator comparison or agent result; subclass parity and the conditional probability target remain next. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. Forecast jobs `22693026` and `22693027` are gated behind the two live prefix fits; each is followed by independent artifact verification, then paired report `22693044`. From aa9e5c2feeecec912c6ff354d31abc3dbaff17e1 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:26:42 -0400 Subject: [PATCH 61/94] Record completed Balloons prefix fits and verified incumbent forecast --- docs/uncertainty/balloons-incumbent-control.md | 12 +++++++++++- docs/uncertainty/balloons-prefix-inference.md | 4 ++++ docs/uncertainty/implementation-progress.md | 7 +++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/uncertainty/balloons-incumbent-control.md b/docs/uncertainty/balloons-incumbent-control.md index 867e7c6e3..fca4c29c6 100644 --- a/docs/uncertainty/balloons-incumbent-control.md +++ b/docs/uncertainty/balloons-incumbent-control.md @@ -41,7 +41,11 @@ The initial fitting attempt `22693556` stopped before fitting because the new co Its dependent verifier and comparison jobs were cancelled by the scheduler. This was a comparison-harness failure, not an agent outcome. The corrected driver verifies raw successor frames and actions separately from the intended rest-window averages. -Those checks pass, and fitting job `22693806` is running on a compute node. +Those checks pass, and fitting job `22693806` completed in 39:11 of allocation time. +The driver recorded 157,664 native actions, including fitting and repeated predictions. +All three prepared segments survived. +The full incumbent's publication rules retained the original applied parameters: several were insensitive or anchored, and the fitted drag value at its upper bound was not applied. +This is the verified outcome of the complete incumbent fitter, not a shortcut that skipped fitting. | Job | Purpose | | --- | --- | @@ -52,3 +56,9 @@ Those checks pass, and fitting job `22693806` is running on a compute node. The future metrics match the posterior reports: thirteen position/speed features and fourteen event indicators. The control reports selected-point predictions, not the incumbent's complete interval or planning ensemble. Final comparisons must retain numerical-stability warnings and cannot be treated as agent solve-rate or sample-efficiency results. + +Independent verifier `22693822` completed in 17 allocation seconds and reproduced every frame of the complete 235-action selected forecast using the manual native lifecycle. +The selected-point forecast reproduces all reserved tie, burst and clip indicators, but misses the final `InBand`/evaluator-win event. +Each of those two indicators has one error across the 171 reserved frames; the rest indicator differs on 67 frames. +These are prediction errors on recorded actions, not a failed agent seed. +Both posterior fits have completed and their population forecasts are now running; comparison `22693842` remains dependent on their independently verified paired report. diff --git a/docs/uncertainty/balloons-prefix-inference.md b/docs/uncertainty/balloons-prefix-inference.md index 58352abfb..31ba04eb5 100644 --- a/docs/uncertainty/balloons-prefix-inference.md +++ b/docs/uncertainty/balloons-prefix-inference.md @@ -76,6 +76,10 @@ The launcher checks the independent certificate, source hashes and fitting bound These are numerical inference replicas on one development recording, not agent seeds. They remain separate from seeds 300/301, hatch experiments and the MB/MF performance sweep. +Both pilots have completed all 32 temperatures: seed 620 used 12,487 evaluations in a 1:30:55 allocation, and seed 621 used 13,439 in 1:36:43. +Their complete reports match the checksummed checkpoints' joint populations and weights exactly. +Both retain only one initial ancestor, with 19 and 15 resampling events respectively, so completion does not establish adequate exploration. +Full-population forecast jobs `22693026` and `22693027` are running, followed by independent verification and the paired comparison. The 171-action causal forecast path, prediction stability, inference cost and comparison against the incumbent remain unfinished. No posterior is approved for planning merely because a pilot completes. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 9e5bd67da..6435c841b 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -25,9 +25,11 @@ The remaining default glue errors are three delayed deposition readings and two This is a development program revision, not an estimator comparison or agent result; subclass parity and the conditional probability target remain next. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. -Forecast jobs `22693026` and `22693027` are gated behind the two live prefix fits; each is followed by independent artifact verification, then paired report `22693044`. +Both prefix fits have completed, and forecast jobs `22693026` and `22693027` are running; each is followed by independent artifact verification, then paired report `22693044`. Every positive-weight particle keeps its original weight, receives eight generated suffixes and eight separate conditional-density draws, and must reproduce its saved prefix likelihood. -The [Balloons incumbent control](balloons-incumbent-control.md) passes three complete native lifecycle comparisons and is fitting as `22693806`, with independent verification and comparison queued. +The [Balloons incumbent control](balloons-incumbent-control.md) completed its full fit as `22693806` in 39:11 and its independent complete-trajectory verification as `22693822` in 17 seconds; the comparison remains queued. +All three segments survived, but the incumbent's publication rules retained the original applied parameters. +Its selected-point forecast reproduces reserved tie/burst/clip indicators but misses the final goal event; this is a recorded-action prediction result, not an agent failure. Its overlapping windows and rest-averaged starts remain intact; the comparison driver now verifies those averages after correcting its initial overly strict raw-frame check. The [Balloons prefix forecast fixture](balloons-prefix-forecasts.md) now covers all 171 reserved actions using a candidate selected only from the 64-action prefix. @@ -59,6 +61,7 @@ Its guide does not reuse a center or population selected from the complete recor Thirty-two candidate prefixes repeat exactly, with fifteen finite targets; independent verification confirms the future-data boundary, complete saved replays and exact serial/parallel targets. Continuous transition normalizers are annealed with the output likelihood while preserving the same final conditional target and exact-observation representation. The gated numerical pilots `22691680_0` and `_1` use seeds 620/621 and remain separate from the full-recording 300/301 pair. +Both completed all 32 temperatures with 12,487/13,439 evaluations; checksummed checkpoints match the complete reports exactly, but each population retains only one initial ancestor. Their posterior adequacy and reserved-action forecasts remain unassessed. The [combined Domino assessment](domino-comparison-summary.md) now has all six completed 64-particle populations, their reserved-action forecasts and verified complete weighted histories. From de2a3224c2e706da27b3a013098a4e38048745cf Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:35:34 -0400 Subject: [PATCH 62/94] Verify Bridge subclass parity and exact glue-rate conditioning --- docs/uncertainty/bridge-subclass-inference.md | 63 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 + 2 files changed, 66 insertions(+) create mode 100644 docs/uncertainty/bridge-subclass-inference.md diff --git a/docs/uncertainty/bridge-subclass-inference.md b/docs/uncertainty/bridge-subclass-inference.md new file mode 100644 index 000000000..bb5c5a28e --- /dev/null +++ b/docs/uncertainty/bridge-subclass-inference.md @@ -0,0 +1,63 @@ +# Revised Bridge subclass and exact-rate conditioning + +This follows the [Bridge program-revision diagnostics](bridge-glue-attribution.md). +It preserves the useful revised candidate through the current simulator interface and verifies one required probability-model component. +Neither result establishes a complete Bridge posterior or changes the acting agent. + +## Complete subclass parity + +The revised candidate now uses `AGENT_PARAM_SPECS`, observation-driven `MODEL_STATE_INIT`/`update_model_state`, and the native `_domain_specific_step` hook. +The observation callback carries glue levels, bond dwell counters and bonds, computes feature outputs, and records the attachment commands to apply. +The native hook queues those commands for the next action; the state getter exposes the predicted glue features. +The converted functions do not read trajectory history, which is checked against their source before conversion. +The historical rules remain a frozen reference for this conversion, rather than a second new production interface. + +Job `22696083` completed four full 1,186-action native trajectories in 1:53 of allocation time on `node1412` under `mit_preemptable`. +The four cases are the revised default, its repetition, the clean-prefix-derived geometry parameter pair, and the historical geometry parameter pair. +Every observed frame, encoded learned-memory value and attachment-command count/type matches the independently verified literal reference exactly in all four cases. +The report contains all 4,744 native steps, and the completed artifacts were also compared directly against the frozen reference after the job finished. +This establishes conversion parity for these cases, not arbitrary simulator restoration or unchanged stochastic agent conversations. + +The frozen bundle is `logs/uncertainty_bridge_revised_subclass_20260913/`. +It contains the subclass factory, complete native parity driver, source/input identities and verified artifacts. +The candidate remains an offline development model; its remaining exact-output disagreements are preserved. + +## Exact glue readings constrain the rate directly + +The revised program adds a rate `r` to its partial glue level and publishes `1` once accumulated progress reaches a latch threshold `L`. +Starting from an exactly observed zero level, a positive partial reading `y < 1` after one deposition implies `r = y`. +Drainage and bond consumption cannot create a positive partial reading. +The geometric event that permits deposition is still a separate constraint; solving for the rate does not guarantee it occurs. + +The new component reference uses the already implemented affine-conditioning chart for `y = r`, retaining the original rate-prior density and unit Jacobian. +It does not draw continuous rates and reject unequal observations, introduce sensor noise on the exact reading, or count later deterministic repetitions as independent rate measurements. + +Under the explicitly declared component priors `r ~ Uniform(0.05, 1)` and `L ~ Uniform(0.4, 1)`, the progression `0, 0.2, 0.4, 1` gives: + +- `r = 0.2` from the partial reading, with density factor `1 / 0.95`. +- `L > 0.4` from the second partial reading. +- `L <= 0.6000000000000001` from the third addition in the actual floating-point program. + +Thus the remaining latch distribution is uniform on that interval for this fixed deposition schedule. +Its mean is approximately `0.5`, and integrating the retained rate density and latch constraint gives approximately `0.3508771929824563`. +This is a density/mass factor for the stated component representation, not a probability of the full recording or a Bayes factor between arbitrary programs. +The uniform boxes are declared development assumptions inherited from the candidate's bounds, not a claim that those bounds were chosen before all historical learning data. + +Job `22696234` completed the component reference in 15 allocation seconds on the compute node. +It evaluates 600 latch candidates through the actual revised deposition function and compares their exact supported/unsupported traces with an independent interval calculation. +The retained weights reproduce the analytic integral and conditional mean. +Additional checks preserve an uninformed height coordinate under the fixed eligible schedule, reproduce the same result when reusing the original prior, and verify that changing the original rate-prior width changes the evidence factor. +The check performs 1,812 rule steps and no recorded native actions. +Artifacts are frozen in `logs/uncertainty_bridge_rate_conditioning_20260913/`. + +## What remains before a Bridge inference comparison + +The component check supplies a valid treatment of the rate equality under its stated schedule; it does not condition the complete native trajectory. +A complete target must still retain geometric eligibility, partial-level drainage, latch and bond events, and every remaining exact and noisy observation. +It also needs the physical initial-state prior and a supported representation for exact joint/readout constraints, with their probability factors retained. +The discrete bond dwell and the other continuous parameters remain free unless the chosen fitting observations constrain them. + +The current 124-action diagnostic prefix contains no bond event, so its later bond predictions should retain uncertainty about dwell rather than treating the default dwell as learned. +Choose and freeze the fitting split and complete prior before launching the estimator comparison, and distinguish any changed split from the existing geometry diagnostics. +Use the same revised subclass for both estimator arms and retain the old inconsistent programs as negative controls. +Stage A/B acceptance remains open; production continues with the incumbent uncertainty handling. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 6435c841b..2b95af4fe 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -23,6 +23,9 @@ Nearest-eligible-face scoring plus a progress latch reduces glue mismatches from Independent readers verify 5,704 grid scores, 9,488 rule steps, all five native artifacts, archived no-op parity and exact repetition of the revised default. The remaining default glue errors are three delayed deposition readings and two bond-consumption events delayed five actions; 22 exact channels still disagree overall. This is a development program revision, not an estimator comparison or agent result; subclass parity and the conditional probability target remain next. +The [revised Bridge subclass](bridge-subclass-inference.md) now matches all observations, learned memory and attachment-command counts/types over four complete native trajectories, totaling 4,744 actions. +The exact-rate component also passes: 600 actual-rule latch cases reproduce the analytic conditional support and retained rate-prior density, with 1,812 rule steps and no native rollout. +That component assumes an eligible deposition schedule; full latent geometry, remaining exact-output constraints and a complete Bridge inference target are still required. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. Both prefix fits have completed, and forecast jobs `22693026` and `22693027` are running; each is followed by independent artifact verification, then paired report `22693044`. From 8acf3f58c0ba6c0a88d14f86bb4d610367b37784 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:46:39 -0400 Subject: [PATCH 63/94] Freeze and queue the revised Bridge incumbent comparison --- docs/uncertainty/bridge-incumbent-control.md | 54 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 57 insertions(+) create mode 100644 docs/uncertainty/bridge-incumbent-control.md diff --git a/docs/uncertainty/bridge-incumbent-control.md b/docs/uncertainty/bridge-incumbent-control.md new file mode 100644 index 000000000..c28db333e --- /dev/null +++ b/docs/uncertainty/bridge-incumbent-control.md @@ -0,0 +1,54 @@ +# Revised Bridge incumbent comparison control + +This control uses the [verified revised subclass](bridge-subclass-inference.md) to prepare the incumbent arm of the Bridge estimator comparison. +It keeps the full incumbent fitter, including its objective, parameter-selection rules, interval handling and fit-evidence configuration. +It does not substitute the exact-rate component for the incumbent's fitting behavior. +The new probability-model arm remains unfinished and is not launched by this control. + +## Frozen development split + +The fitting data are actions 1-600 and observations 0-600 from the same 1,186-action Bridge training recording used in the program diagnostics. +The remaining 586 actions are reserved for causal forecast assessment. +This extends the earlier 124-action geometry diagnostic prefix so that fitting includes the first recorded bond at action 583, while the second bond at action 835 lies in the later segment. +The changed split is explicit; results from the two prefixes must not be treated as interchangeable. +The recording and candidate program have already been inspected during development, so the later segment is not an untouched final evaluation set. + +The source program, subclass implementation, recording files, prefix length and preflight driver are frozen in the control plan. +The fitting data identity hashes the serialized prefix episode separately from the full recording identity. +The model's memory requires the incumbent preparation to retain the full 600-action prefix as one trajectory. +The driver checks every prepared observation and action against that prefix. +No rest-point reset of glue levels, dwell counters or bonds is introduced. + +Both complete forecast repetitions start from the same original public noisy initial state and run all 1,186 recorded actions. +They receive no later observation corrections. +Assessment independently reconstructs the selected forecast through the manual native reset/pin/zero-velocity/step lifecycle, then computes feature errors and exact glue-reading mismatches only on actions 601-1186. +Angular feature errors use circular differences; these metrics are distinct from a probability-model likelihood. +Selected-point predictions are not the incumbent's full interval or planning ensemble. + +## Compute pipeline + +| Job | Purpose | Submission state | +| --- | --- | --- | +| `22696941` | Check full-prefix preparation and helper/manual native parity at default dwell, dwell 20 and dwell 40. | Queued for resources. | +| `22697046` | Run the complete incumbent fitter, then two complete causal forecasts. | Depends on successful preflight. | +| `22697048` | Independently reproduce the selected forecast and assess the 586-action suffix. | Depends on successful fitting. | + +The preflight compares three pairs of full trajectories and checks that changing bond dwell changes predictions. +The fitting driver requires that certificate, preserves all six declared parameters, and records fitted versus actually applied values separately. +If the incumbent rejects every segment, it retains its declared default parameter behavior rather than treating the fit as successful evidence. +Infrastructure failures remain separate from model prediction failures. + +All jobs use `mit_preemptable` on the previously audited `node1412`. +The incumbent fitting implementation is explicitly serial, so its allocation requests one CPU, 20 GB and an eight-hour limit. +The preflight was confirmed pending because all 64 CPUs on that node were allocated; dependent jobs have not begun fitting. +These scheduler states are dated submission observations, not permanent status claims. +The frozen bundle is `logs/uncertainty_bridge_incumbent_prefix600_20260913/`. + +## Remaining comparison requirements + +The revised subclass's geometry, attachments and other exact observations still need a complete joint probability target before the replacement arm can be fitted. +The exact-rate reference supplies only one component of that target. +The physical initial-state inventory, supported geometry and remaining continuous exact-output representations must stay explicit. +Use the same fixed program and 600-action prefix for both estimator arms, and separately identify any new transition-discrepancy assumptions or state-inference approximations. +There is no estimator advantage, posterior adequacy or agent-performance result to report from a queued incumbent control. +The incumbent remains the production default while Stage A/B validation continues. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2b95af4fe..f78ff43a5 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -26,6 +26,9 @@ This is a development program revision, not an estimator comparison or agent res The [revised Bridge subclass](bridge-subclass-inference.md) now matches all observations, learned memory and attachment-command counts/types over four complete native trajectories, totaling 4,744 actions. The exact-rate component also passes: 600 actual-rule latch cases reproduce the analytic conditional support and retained rate-prior density, with 1,812 rule steps and no native rollout. That component assumes an eligible deposition schedule; full latent geometry, remaining exact-output constraints and a complete Bridge inference target are still required. +The [revised Bridge incumbent control](bridge-incumbent-control.md) now freezes a 600-action fitting prefix containing the first bond and reserves the remaining 586 actions, including the second bond. +Preflight `22696941` is queued for resources on the audited node, followed by the full serial incumbent fit `22697046` and independent forecast verification `22697048`. +The new Bridge probability-model arm remains unfinished; this pipeline does not substitute a point fit for the planned posterior comparison. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. Both prefix fits have completed, and forecast jobs `22693026` and `22693027` are running; each is followed by independent artifact verification, then paired report `22693044`. From 9326292316b3ab7bc13efb7982cf452ba4cf0f60 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 17:49:57 -0400 Subject: [PATCH 64/94] Record completed larger Domino fits and active forecast checks --- docs/uncertainty/domino-joint-transition.md | 7 +++++++ docs/uncertainty/implementation-progress.md | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md index a3a54bd2d..4a08050aa 100644 --- a/docs/uncertainty/domino-joint-transition.md +++ b/docs/uncertainty/domino-joint-transition.md @@ -166,4 +166,11 @@ The runtime identities legitimately differ because they include the budget-speci Follow-up reader check `22689407` passes, and both live 128-particle reports match their independently reconstructed expected runtime identity. Reader test `22689104` passes identical-summary, injected-regression and misaligned-coordinate controls and correctly reports the current two-population comparison as incomplete. The follow-up forecast/report bundle is `logs/uncertainty_domino_transition_128_forecast_20260913/`. + +Both larger fits have now completed all 32 temperatures. +Seed 100 used 31,921 evaluations in a 3:39:43 allocation and retained 15 initial ancestors; seed 101 used 31,959 evaluations in 3:40:32 and retained 10. +Each recorded five resampling events. +Their complete sample populations and weights match the checksummed checkpoints exactly. +The dependent native forecast/verifier jobs `22689058_0` and `22689059_1` are running, with the budget comparison `22689095` still queued behind them. +Completion and ancestor counts do not establish that the previously observed toppling-prediction disagreement has been resolved. All results remain a fixed-initial-state ablation and do not close Stage B. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index f78ff43a5..b229ef2b3 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -27,7 +27,7 @@ The [revised Bridge subclass](bridge-subclass-inference.md) now matches all obse The exact-rate component also passes: 600 actual-rule latch cases reproduce the analytic conditional support and retained rate-prior density, with 1,812 rule steps and no native rollout. That component assumes an eligible deposition schedule; full latent geometry, remaining exact-output constraints and a complete Bridge inference target are still required. The [revised Bridge incumbent control](bridge-incumbent-control.md) now freezes a 600-action fitting prefix containing the first bond and reserves the remaining 586 actions, including the second bond. -Preflight `22696941` is queued for resources on the audited node, followed by the full serial incumbent fit `22697046` and independent forecast verification `22697048`. +Preflight `22696941` has started on the audited node, followed by the queued full serial incumbent fit `22697046` and independent forecast verification `22697048`. The new Bridge probability-model arm remains unfinished; this pipeline does not substitute a point fit for the planned posterior comparison. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. @@ -92,7 +92,8 @@ Both 64-particle point-start fits and their independently verified full-suffix f The new pair agrees to 0.215 mm in position means but differs by 0.166830 in final-toppling probability, failing the declared 0.15 limit. Future-bank variability is much smaller than fitted-replica disagreement, and the new model has worse toppling scores than the matched older model on this recording despite slightly better position error. The complete artifacts retain original weights, separate generation from conditioned density evaluation and reject six deliberately corrupted inputs. -Matched 128-particle follow-ups `22688869_0` and `_1` are running, with dependent forecast/verifier jobs `22689058_0` and `22689059_1` and a six-pair budget report `22689095`. +Matched 128-particle follow-ups `22688869_0` and `_1` completed all 32 temperatures with 31,921/31,959 evaluations, retaining 15/10 initial ancestors. +Complete populations and weights match their checksummed checkpoints; forecast/verifier jobs `22689058_0` and `22689059_1` are running, with six-pair budget report `22689095` still dependent. These remain offline diagnostics; neither the short fixture nor a completed sampler is an approved posterior. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. From b519262aeb997d71a20abcdb27a5cf8fa9a431a1 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 18:27:39 -0400 Subject: [PATCH 65/94] Document verified Bridge reset-scene prior and long replay audits --- docs/uncertainty/bridge-incumbent-control.md | 12 +- docs/uncertainty/bridge-initial-scene.md | 127 +++++++++++++++++++ docs/uncertainty/initial-state-inventory.md | 3 + 3 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 docs/uncertainty/bridge-initial-scene.md diff --git a/docs/uncertainty/bridge-incumbent-control.md b/docs/uncertainty/bridge-incumbent-control.md index c28db333e..a6bacfddf 100644 --- a/docs/uncertainty/bridge-incumbent-control.md +++ b/docs/uncertainty/bridge-incumbent-control.md @@ -27,10 +27,10 @@ Selected-point predictions are not the incumbent's full interval or planning ens ## Compute pipeline -| Job | Purpose | Submission state | +| Job | Purpose | Verified state on September 13 | | --- | --- | --- | -| `22696941` | Check full-prefix preparation and helper/manual native parity at default dwell, dwell 20 and dwell 40. | Queued for resources. | -| `22697046` | Run the complete incumbent fitter, then two complete causal forecasts. | Depends on successful preflight. | +| `22696941` | Check full-prefix preparation and helper/manual native parity at default dwell, dwell 20 and dwell 40. | Completed; all three pairs exact, 7,116 native actions. | +| `22697046` | Run the complete incumbent fitter, then two complete causal forecasts. | Running. | | `22697048` | Independently reproduce the selected forecast and assess the 586-action suffix. | Depends on successful fitting. | The preflight compares three pairs of full trajectories and checks that changing bond dwell changes predictions. @@ -40,13 +40,15 @@ Infrastructure failures remain separate from model prediction failures. All jobs use `mit_preemptable` on the previously audited `node1412`. The incumbent fitting implementation is explicitly serial, so its allocation requests one CPU, 20 GB and an eight-hour limit. -The preflight was confirmed pending because all 64 CPUs on that node were allocated; dependent jobs have not begun fitting. -These scheduler states are dated submission observations, not permanent status claims. +The preflight initially waited because all 64 CPUs on that node were allocated; it subsequently completed and released the dependent fit. +These scheduler states are dated observations, not permanent status claims. The frozen bundle is `logs/uncertainty_bridge_incumbent_prefix600_20260913/`. ## Remaining comparison requirements The revised subclass's geometry, attachments and other exact observations still need a complete joint probability target before the replacement arm can be fitted. +The [new initial-scene construction](bridge-initial-scene.md) supplies a declared reset law and verified 600-action native continuations, including moving bodies and robot joints. +Those trajectories repeat exactly but still contradict exact recorded outputs, so replay support does not establish a usable conditional target. The exact-rate reference supplies only one component of that target. The physical initial-state inventory, supported geometry and remaining continuous exact-output representations must stay explicit. Use the same fixed program and 600-action prefix for both estimator arms, and separately identify any new transition-discrepancy assumptions or state-inference approximations. diff --git a/docs/uncertainty/bridge-initial-scene.md b/docs/uncertainty/bridge-initial-scene.md new file mode 100644 index 000000000..dd764c055 --- /dev/null +++ b/docs/uncertainty/bridge-initial-scene.md @@ -0,0 +1,127 @@ +# Revised Bridge initial-state model + +September 13, 2026. +This extends the [revised subclass comparison](bridge-subclass-inference.md) with an explicit physical root law. +It is an offline development model; the incumbent remains the production estimator. +The 600-action fitting split in [the incumbent control](bridge-incumbent-control.md) remains the intended comparison boundary. +Short root audits do not replace that comparison. + +## Geometry established by the native model + +The visible scene has five movable blocks, a movable bottle and two static site markers. +All eight bodies have box collision shapes and no articulated joints. +The bottle's collision shape is a box, so its omitted roll and pitch cannot be discarded using rotational symmetry. +The sites also have collision shapes and remain in whole-scene collision checks. + +| Body | Full collision extents, metres | Mass, kilograms | +| --- | --- | --- | +| Each block | 0.10, 0.05, 0.05 | 0.10 | +| Bottle | 0.024, 0.024, 0.060 | 0.05 | +| Each site | 0.090, 0.090, 0.0002 | 0 | + +Native inventory job `22697529` completed 96 support probes in two discarded worlds with exact fresh-instance agreement. +The tilted block's native support depth differs from the ideal sharp-box depth by approximately 0.482 mm in the tested orientation. +Follow-up `22697773` checked 240 probes, adding twelve deterministic random orientations to the eight original cases for each movable body. +For these bodies, the reported collision margin is smaller than every half-extent, and the native support depth agrees within 1e-9 m with `sum(abs(R[2]) * (half_extents - margin)) + margin`. +Each inferred contact height is checked through a second native signed-distance query. +The margin audit repeats exactly across two fresh worlds. +These probes inspect candidate geometry only and execute no recorded actions. + +The thin sites do not meet the formula's half-extent condition: their half-thickness is 0.0001 m while their reported margin is 0.001 m. +The first root audit, `22698166`, placed them at the ideal sharp-box contact height and found native table penetration in the tested upright cases. +All sixteen complete candidates failed geometry support, with zero recorded native actions executed. +The failed driver and probability-model assumptions are retained under `ideal-site-support-22698166/`. +The corrected law gives fixed sites uncertain mounting xyz positions; upright versus arbitrary orientation remains a separate mixture. +It does not change the native shapes or suppress site/table collision witnesses. + +## Declared reset law and information boundary + +This construction covers the recorded unheld, initially dry reset case of the revised program. +Every initial `is_held` and glue reading is checked against that scope. +The subclass declares empty glue/bond/dwell memory and an empty command queue at reset; the initializer verifies that declaration and the absence of native constraints in its new candidate world. +These are assumptions of the candidate program's reset model, not facts inferred from omitted evaluator metadata. +Continued episodes or an initially bonded model require a different initial-memory and assembly construction. +Later bonds, dwell counts, command queues and constraint frames must arise from the candidate's uninterrupted action history. + +The initializer first performs a fixed numeric template reset. +It preserves only immutable public sizes/colors and the conditioned fixed-base pose in that template. +Recorded noisy object coordinates and measured joint values do not determine intermediate template geometry. +It then applies the entire sampled root and exact initial controlled-joint conditioning before stepping. +No private velocities, curing counters or recorded weld metadata enter the root law. + +| Quantity and source | What the interface establishes | What remains unknown | Prior and feasible representation | Conditioning or elimination | Remaining continuous dimensions and discrete cases | +| --- | --- | --- | --- | --- | --- | +| Each fixed site, native shape and public xyz | Fixed mass-zero collision box, noisy xyz | Mounting position and omitted orientation | Uniform workspace xyz; probability 0.8 for upright orientation with uniform yaw, otherwise Haar-uniform rotation; whole-scene collision conditioning | Static base motion is absent under this fixed-fixture model; xyz readings remain likelihood factors | 4 or 6 per site; upright/free-rotation cases | +| Each block and bottle, native shape and public features | Fixed box dimensions; noisy xyz and available angles | True pose, omitted bottle tilt, motion and table support | Probability 0.8 for table-supported rest, divided equally among six local faces with uniform xy/yaw; otherwise uniform xyz, Haar rotation and uniform linear/angular velocity | Supported z follows the verified shape map; all noisy pose readings remain in the likelihood | 3 or 12 per body; six rest-face cases plus free moving | +| Robot, native joint schema and public joint positions | Nine exact controlled positions, four other movable joints and eleven fixed joints | Unobserved positions and all movable-joint velocities | Gaussian movable-joint positions; probability 0.8 for zero velocities, otherwise independent bounded velocities | Exact controlled positions conditioned with their original density retained; fixed joints and fixed base are declared inputs | 4 or 17; rest/moving cases | +| Block sizes/colors, public descriptors | Exact immutable inputs to this program | No sampled descriptor uncertainty under this comparison | Fixed known geometry and labels | Condition on descriptors and retain native shape checks | 0 | +| Holding, public flags | Every initial body is observed unheld | Other reset cases are outside this construction | Explicit unheld reset scope | Check initial flags, with no initial grasp constraint | 0 in this scoped case | +| Glue, dwell, bonds and pending commands, subclass reset declaration | Empty declared model memory; observed initial glue values are all zero | Continued-episode memory is unsupported by this root law | Revised program's declared empty memory and constraint-free reset case; future memory from uninterrupted history | Check the dry initial scope and the reset declaration; do not import private curing or weld metadata | 0 in this scoped model | + +The workspace bounds are x in [0.2, 1.3], y in [0.8, 1.9] and z in [0.4, 0.95] metres. +Moving bodies have componentwise linear-velocity bounds of +/-0.1 m/s and angular-velocity bounds of +/-0.2 rad/s. +Robot velocity bounds are +/-0.1 in the native joint units. +Gaussian robot position priors use standard deviation pi for revolute joints and 0.1 m for prismatic joints, as in the earlier reset-compatible reference. +Resting bodies derive their z position from the verified native box support formula; a sampled yaw does not imply a fixed world orientation for standing blocks. +The physical root has 30 through 101 continuous coordinates across its declared cases, before adding model parameters or trajectory-discrepancy variables. +The implementation uses 140 augmented unit coordinates to include mixture selectors, proposal selectors and inactive-case coordinates; 140 is not the physical dimension. + +## Proposals and normalization + +Noisy initial xyz and available yaw readings guide defensive Gaussian/uniform proposals inside the fixed original coordinate bounds. +Every guided point retains its complete original-prior/proposal density ratio. +The likelihood must still score all those observations once; using them for proposal guidance does not condition them away. +Orientation cases are not selected by plugging noisy Euler angles into an exact physical state. +Unguided continuous rotations use a Haar-uniform quaternion map. + +The geometry predicate checks the robot, every movable body and both sites against candidate collision geometry. +It retains the fixed wheel/ground contacts only at the previously audited -0.011075 m distance, within a 1e-8 m numerical geometry allowance. +Other penetrating pairs reject the entire candidate. +This allowance is a geometric roundoff policy, not an observation-likelihood tolerance. + +For this constraint-free reset law, initial geometry and its feasibility probability are independent of all six fitted glue parameters. +Those parameters enter later model updates rather than initial collision dimensions, poses or attachments. +Thus the original parameter marginal is preserved when normalizing scene feasibility conditionally on parameters, and the same unknown scene-normalization constant cancels within this fixed target. +The audit also compares complete initialized roots under default and lower-bound parameter vectors. +That finite check supplements the construction's independence argument; it does not justify omitting normalization for a future parameter-dependent attachment prior or for cross-model evidence. + +## Validation and remaining work + +Corrected compute job `22698304` tests sixteen stratified roots covering the six rest faces, mixed/free body motion, site orientation cases and both robot motion cases. +These deliberately stratified probes are not claimed as independent prior samples or as a Monte Carlo estimate of feasible prior mass. +Every feasible root receives two independent fresh-world 64-action continuations, comparing all public predictions and learned memory exactly. +Every root also receives a separate initialization under the alternate parameter vector. +The job completed on `mit_preemptable` with one CPU and 12 GB in 44 allocation seconds. +One root was feasible and repeated all 64 actions exactly, for 128 native actions total; the other fifteen retained their geometry rejections. +Every root initialized identically under the two parameter vectors. +Independent reader `22698424` completed 182 density-ratio checks over 384 guided coordinates, 54 supported-body charts, 42 free-body cases, 384 stored joint states and 47 negative contact witnesses. +The maximum independently recomputed density-ratio difference is 1.776e-15. +The earlier corrected-mounting attempt `22698233` reached its first feasible 64-action continuation but failed report serialization because a shallow copy of initial model memory shared later mutations. +Its replacement deep-copies that diagnostic memory; it changes neither dynamics nor the probability law, and the failed attempt is not a model or agent failure. + +Follow-up `22698483` changes only moving-body proposal guidance: it centers the proposed z coordinate at least 0.02 m above the orientation-dependent native support height. +This is a proposal center, not a new prior bound or collision allowance; the prior component retains full original support and every candidate retains its full conditional proposal-density correction. +The motivation is the retained moving-root table penetrations, rather than an assumption that the physical initial state is exactly at its noisy height. +The original root law, initial-observation likelihood requirement and whole-scene feasibility predicate remain unchanged. +The follow-up native audit completed in 53 allocation seconds with five feasible roots and 640 native actions. +Four supported roots include three moving bodies each, and two also have moving robot joints; the original all-rest positive root is retained. +Every complete predicted frame and learned-memory state repeats exactly, and all sixteen roots retain exact parameter-independent initialization. +The remaining eleven complete-scene rejections remain in the report. +Dependent density reader `22698502` passed, preserving the same independent density and support-chart checks. +This improves proposal support on the stratified audit; it is not a feasible-prior-mass estimate or a posterior-efficiency result. + +The same frozen law and proposal now feed long-prefix job `22698641`, followed by a dependent independent density reader. +It extends the five supported roots to two complete 600-action continuations each and records exact-output mismatch attribution against the fitting prefix. +Future observations after action 600 are excluded from that attribution. +The extension completed in 2:32, with all five feasible roots repeating all 600 actions exactly, for 6,000 native actions. +Independent reader `22698656` completed in six seconds and verified the saved densities, support charts and original weights. +All five roots nevertheless contradict exact recorded outputs, with their first discrepancies at actions 34, 32, 1, 34 and 1 respectively. +The all-rest root has sixteen glue mismatches across two channels; the moving-root cases have much larger glue mismatches. +These are selected candidate failures, not proof that the full conditional support is empty. +The next target must represent the remaining continuous exact trajectory constraints and retain the geometry-dependent discrete glue likelihood. +Neither a completed replay nor a mismatch count is a continuous exact-conditioning method. + +Full 600-action exact-output support, independent density verification, the complete conditional inference target and posterior adequacy remain required. +In particular, the exact-rate component assumes an eligible deposition schedule and cannot simply replace the full geometry-dependent glue likelihood. +This root construction alone supplies neither a usable posterior nor evidence of improved agent performance. +Frozen inputs, scripts and reports are in `logs/uncertainty_bridge_initial_scene_20260913/` and `logs/uncertainty_bridge_scene_inventory_20260913/`. diff --git a/docs/uncertainty/initial-state-inventory.md b/docs/uncertainty/initial-state-inventory.md index df82c8d1a..33f5a5aa1 100644 --- a/docs/uncertainty/initial-state-inventory.md +++ b/docs/uncertainty/initial-state-inventory.md @@ -71,6 +71,9 @@ Similarly, `rollout_states` starting a simulated trial at rest is a legacy fitti ## Bridge The recording contains five blocks, one bottle, two site markers, and the robot. +The rows below retain the frozen no-op program's original inventory. +The later [revised Bridge root inventory](bridge-initial-scene.md#declared-reset-law-and-information-boundary) separately specifies the parameterized subclass's body/site/robot priors and declared reset-memory scope. +Its short native root audit is complete, but full-prefix probability support and inference remain unvalidated. The [environment source](../../predicators/envs/pybullet_bridge.py) separates partially observed block features from hidden curing counters and attached partners. Its hidden task generator and evaluator restoration code are diagnostic sources, not an agent prior. From 6a6db14bd998110669a0c1e8bba034cf8078f5cd Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 18:27:40 -0400 Subject: [PATCH 66/94] Record completed uncertainty forecasts and unresolved prediction regressions --- docs/uncertainty/balloons-prefix-forecasts.md | 18 +++++++++++++ docs/uncertainty/boil-canonical-forecasts.md | 23 ++++++++++++++++ docs/uncertainty/domino-joint-transition.md | 15 +++++++++-- docs/uncertainty/implementation-progress.md | 26 +++++++++++++++---- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/docs/uncertainty/balloons-prefix-forecasts.md b/docs/uncertainty/balloons-prefix-forecasts.md index a19eeaceb..f42095956 100644 --- a/docs/uncertainty/balloons-prefix-forecasts.md +++ b/docs/uncertainty/balloons-prefix-forecasts.md @@ -120,3 +120,21 @@ The certificate is `adapter-22692938.json`; `posterior-submitted.json` identifie The [incumbent control](balloons-incumbent-control.md) runs the existing fitter on the same supplied prefix and compares its selected-point forecast with these population forecasts. That comparison retains the differences in initial-state treatment, segmentation and discrepancy assumptions explicitly. + +## Completed paired forecasts + +Jobs `22693026` and `22693027` completed with their independent verifiers in 54:41 and 53:29 allocation time. +Each population forecast uses 241,110 native actions. +Paired report `22693044` completed, and both forecast files match their independent verification and frozen identity records. + +The replicas disagree by 0.21909 m RMS in predicted box height and 0.58361 m/s RMS in predicted speed. +Their maximum band-event probability gap is 0.51132, and the maximum individual burst-probability gap is 0.59913. +Box-height RMSE against the clean public future is 0.16170 m for numerical seed 620 and 0.09725 m for 621. +Both assign zero empirical probability to the final evaluator-win event among their generated futures. +That finite-sample result is not a proof that the model's true win probability is zero. + +Future-density integration is also unstable: seed 620's two density banks have log means approximately -290,981 and -434,259, while seed 621's are approximately -50,634 and -52,053. +Retain those bank discrepancies rather than treating a finite mixture density as a converged likelihood estimate. +Both fitting populations descend from one initial ancestor, so neither the fitting nor the future integration is established as adequate. +The scheduled incumbent comparison `22693842` is still waiting for resources at this update. +These outcomes require diagnosis of state/transition assumptions and numerical exploration, not promotion to live planning. diff --git a/docs/uncertainty/boil-canonical-forecasts.md b/docs/uncertainty/boil-canonical-forecasts.md index 00e4053c1..9e60cc950 100644 --- a/docs/uncertainty/boil-canonical-forecasts.md +++ b/docs/uncertainty/boil-canonical-forecasts.md @@ -88,3 +88,26 @@ Both fits were still running at submission, so there is no completed-population The separate [incumbent control](boil-incumbent-control.md) supplies the same literal program through a verified subclass conversion and fits the identical recorded prefix with the existing full fitter. Its selected-point forecast will be compared with the two posterior means after independent replay verification. The control keeps its existing initial-state and objective assumptions, so this comparison alone does not isolate the estimator from the explicit posterior discrepancy extension. + +## Completed forecasts and incumbent comparison + +Forecast/verifier jobs `22690858_0` and `22690863_1` completed in 1:34:28 and 1:31:38 allocation time. +Each forecast uses 93,456 native actions; each independent verifier checks 288 histories, 684,288 joint factors and 4,224 additional native reference actions. +Paired report `22690864` and incumbent comparison `22691239` completed, with all source and verification hashes checked. + +| Forecast | Jug x RMSE against clean public future (m) | Jug y RMSE (m) | Bubbling RMSE | Boiled Brier score | Final task-goal prediction | +| --- | ---: | ---: | ---: | ---: | ---: | +| Incumbent selected point | 0.00170 | 0.00594 | 0.01915 | 0.00758 | True | +| New joint model, numerical seed 410 | 0.12922 | 0.08599 | 0.48328 | 0.25314 | 0.22319 | +| New joint model, numerical seed 411 | 0.12632 | 0.08671 | 0.45393 | 0.22527 | 0.30614 | + +The new forecasts are substantially worse on these motion and heating metrics on this development recording. +The two new populations agree more closely with each other than with the recorded jug path; small replica disagreement would therefore not establish predictive adequacy. +Each fit also retains only one initial ancestor, leaving numerical exploration unresolved. +The final goal actually holds in the reference, but the averaged goal Brier scores dilute that final error across 132 frames; retain both temporal and final-event metrics. +The current estimator remains preferable on this comparison, although the changed initial-state treatment and explicit joint/output discrepancy mean this is not a clean estimator-only attribution. + +Before larger-budget fits, separate the effects of uncertain initial state, per-step stochastic joint transitions and posterior exploration on these same frozen inputs. +In particular, a finite likelihood under joint conditioning does not establish that unconditioned future motor perturbations preserve contact and grasp behavior. +Matched physical forecasts and a separately declared output-discrepancy control can test that mechanism without changing sensor noise or hiding the failed results. +No live replacement or retirement gate has passed. diff --git a/docs/uncertainty/domino-joint-transition.md b/docs/uncertainty/domino-joint-transition.md index 4a08050aa..52977a2ef 100644 --- a/docs/uncertainty/domino-joint-transition.md +++ b/docs/uncertainty/domino-joint-transition.md @@ -171,6 +171,17 @@ Both larger fits have now completed all 32 temperatures. Seed 100 used 31,921 evaluations in a 3:39:43 allocation and retained 15 initial ancestors; seed 101 used 31,959 evaluations in 3:40:32 and retained 10. Each recorded five resampling events. Their complete sample populations and weights match the checksummed checkpoints exactly. -The dependent native forecast/verifier jobs `22689058_0` and `22689059_1` are running, with the budget comparison `22689095` still queued behind them. -Completion and ancestor counts do not establish that the previously observed toppling-prediction disagreement has been resolved. +The dependent native forecast/verifier jobs `22689058_0` and `22689059_1` completed in 22:35 and 22:47, and budget comparison `22689095` completed in three seconds. +Both forecast source hashes match their independent verification reports. +Each report checks 1,152 histories and 1,669,248 joint factors, preserves original weights and rejects all six corruption classes. +The 128-particle pair passes its three replication screens: position-mean RMS disagreement is 0.200 mm and maximum toppling-curve/final-toppling disagreement is 0.127143. +That is an improvement in same-budget agreement, but one cross-budget pair still differs by 0.192756 in final toppling probability and fails the 0.15 screen. +Four of the six retained comparisons pass all three screens; the failed 64-particle pair and failed cross-budget pair remain part of the assessment. + +| Numerical seed, 128 particles | Position RMSE (m) | Toppling Brier score | Final-toppling Brier score | Forecast native actions | +| --- | ---: | ---: | ---: | ---: | +| 100 | 0.0113284 | 0.000045175 | 0.004373531 | 185,794 | +| 101 | 0.0113363 | 0.000162052 | 0.013933829 | 185,794 | + +The larger budget does not uniformly improve event predictions, and same-budget agreement does not resolve the remaining sensitivity to numerical budget. All results remain a fixed-initial-state ablation and do not close Stage B. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index b229ef2b3..b7edeee5e 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -27,13 +27,24 @@ The [revised Bridge subclass](bridge-subclass-inference.md) now matches all obse The exact-rate component also passes: 600 actual-rule latch cases reproduce the analytic conditional support and retained rate-prior density, with 1,812 rule steps and no native rollout. That component assumes an eligible deposition schedule; full latent geometry, remaining exact-output constraints and a complete Bridge inference target are still required. The [revised Bridge incumbent control](bridge-incumbent-control.md) now freezes a 600-action fitting prefix containing the first bond and reserves the remaining 586 actions, including the second bond. -Preflight `22696941` has started on the audited node, followed by the queued full serial incumbent fit `22697046` and independent forecast verification `22697048`. +Preflight `22696941` completed all 7,116 native actions with exact helper/manual agreement; the full serial incumbent fit `22697046` is running, with independent forecast verification `22697048` dependent on completion. The new Bridge probability-model arm remains unfinished; this pipeline does not substitute a point fit for the planned posterior comparison. +The [Bridge initial-scene construction](bridge-initial-scene.md) now declares the six-face rest/moving body cases, uncertain fixed-site mounting, all robot joints and the revised program's reset-memory boundary. +Its 140 augmented proposal coordinates represent 30 through 101 continuous physical root coordinates across cases, before parameters and trajectory discrepancies. +Native geometry audits verify the rounded-box support formula for blocks and the bottle, while retaining the failed ideal-contact assumption for thin site markers. +Corrected root audit `22698304` completes sixteen cases: one is feasible and repeats all 64 actions exactly; fifteen retain collision rejections. +Independent reader `22698424` verifies 182 proposal-density ratios, 384 guided coordinates, all root support charts and stored rejection witnesses. +Geometry-aware moving-height guidance `22698483` and independent reader `22698502` have completed, with the original prior and full density corrections retained. +Five of sixteen stratified roots are now feasible, including moving-body and moving-robot cases; all five repeat exactly over 64 actions, totaling 640 native actions. +Long-prefix job `22698641` and reader `22698656` also completed: all five roots repeat 600 actions exactly, totaling 6,000 native actions. +All five still contradict exact recorded outputs, starting at actions 1-34 depending on the root; this leaves full-prefix conditional support and the Bridge posterior target open. The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. -Both prefix fits have completed, and forecast jobs `22693026` and `22693027` are running; each is followed by independent artifact verification, then paired report `22693044`. +Both prefix fits, forecast/verifier jobs `22693026` and `22693027`, and paired report `22693044` have completed. Every positive-weight particle keeps its original weight, receives eight generated suffixes and eight separate conditional-density draws, and must reproduce its saved prefix likelihood. The [Balloons incumbent control](balloons-incumbent-control.md) completed its full fit as `22693806` in 39:11 and its independent complete-trajectory verification as `22693822` in 17 seconds; the comparison remains queued. +The new populations disagree by 0.21909 m RMS in box height and up to 0.59913 in an individual burst probability, with large density-bank disagreements as well. +Both assign zero empirical probability to the final win among generated futures; finite forecast completion does not establish reliable uncertainty. All three segments survived, but the incumbent's publication rules retained the original applied parameters. Its selected-point forecast reproduces reserved tie/burst/clip indicators but misses the final goal event; this is a recorded-action prediction result, not an agent failure. Its overlapping windows and rest-averaged starts remain intact; the comparison driver now verifies those averages after correcting its initial overly strict raw-frame check. @@ -49,7 +60,8 @@ The [canonical Boil forecast path](boil-canonical-forecasts.md) now passes its n The verifier checks twelve histories and 28,512 joint-transition factors, and rejects eight corrupted records. The adapter independently checks weighted moments, zero future support and exact checkpoint recovery, rejects nine additional corruptions, and repeats generated and density-evaluation paths exactly. Both Boil fits completed all 32 stages: numerical seeds410/411 used 7,455/7,317 target evaluations, respectively. -Their reserved-132-action forecast/verifier jobs `22690858_0` and `22690863_1` are running, followed by queued paired report `22690864`. +Reserved-132-action forecast/verifier `22690863_1` has completed, including independent checks of 288 histories, 684,288 joint factors and 4,224 native reference actions. +The other forecast/verifier, `22690858_0`, and paired report `22690864` have also completed. Both completed populations descend from one initial ancestor, so completion alone does not establish adequate exploration. Forecast generation cannot read future observations, and the original fitted weights remain unchanged. These are mechanical and offline comparison gates, not evidence of improved agent performance or permission to advance to live posterior use. @@ -59,7 +71,9 @@ Four complete 264-action trajectories and all model memory match the archived li The incumbent full fitter ran as `22691175` on the same 132-action prefix, with interval belief, fit-side noise handling and fit evidence enabled. The fit completed in 8:49, and independent literal-program verifier `22691200` completed in 19 seconds with every predicted frame and model-memory value matching exactly. The selected-point forecast matches the task-goal predicate on all 132 reserved frames, while the boiled predicate differs on one frame. -The combined comparison `22691239` remains queued behind the posterior forecasts. +The combined comparison `22691239` has completed and shows substantially worse motion and heating predictions from the new model on this recording. +Its two jug-x RMSEs are 0.12922/0.12632 m versus 0.00170 m for the incumbent; boiled Brier scores are 0.25314/0.22527 versus 0.00758. +Both new populations also retain one initial ancestor, so state/discrepancy assumptions and numerical exploration must be isolated before larger-budget fits or live use. This compares selected-point and posterior-mean predictions while retaining the explicit differences in initial-state treatment and discrepancy model; it is not an estimator-only ablation or an agent performance result. The separate [Balloons prefix experiment](balloons-prefix-inference.md) now fits only 64 actions and reserves 171 for later forecasts. @@ -93,7 +107,9 @@ The new pair agrees to 0.215 mm in position means but differs by 0.166830 in fin Future-bank variability is much smaller than fitted-replica disagreement, and the new model has worse toppling scores than the matched older model on this recording despite slightly better position error. The complete artifacts retain original weights, separate generation from conditioned density evaluation and reject six deliberately corrupted inputs. Matched 128-particle follow-ups `22688869_0` and `_1` completed all 32 temperatures with 31,921/31,959 evaluations, retaining 15/10 initial ancestors. -Complete populations and weights match their checksummed checkpoints; forecast/verifier jobs `22689058_0` and `22689059_1` are running, with six-pair budget report `22689095` still dependent. +Complete populations and weights match their checksummed checkpoints; forecast/verifier jobs `22689058_0` and `22689059_1` and six-pair budget report `22689095` have completed. +The 128-particle pair passes all three screens, with 0.200 mm position disagreement and a 0.127143 final-toppling gap, but one cross-budget pair still fails at a 0.192756 final-toppling gap. +Four of six comparisons pass all screens; this does not establish budget-stable inference or close the uncertain-initial-state requirement. These remain offline diagnostics; neither the short fixture nor a completed sampler is an approved posterior. The [Fan prefix comparison](fan-prefix-comparison.md) has two completed 64-particle fits and causal forecasts on 68 reserved actions. From f55e7faadf717793ea74336e3873160f45ba99cf Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 18:58:49 -0400 Subject: [PATCH 67/94] Add shared variance inference after isolating Boil forecast perturbations --- .../boil-joint-noise-attribution.md | 66 ++++++++ .../shared-variance-discrepancy.md | 88 +++++++++++ .../code_sim_learning/inference_variance.py | 149 ++++++++++++++++++ .../test_inference_variance.py | 108 +++++++++++++ 4 files changed, 411 insertions(+) create mode 100644 docs/uncertainty/boil-joint-noise-attribution.md create mode 100644 docs/uncertainty/shared-variance-discrepancy.md create mode 100644 predicators/code_sim_learning/inference_variance.py create mode 100644 tests/code_sim_learning/test_inference_variance.py diff --git a/docs/uncertainty/boil-joint-noise-attribution.md b/docs/uncertainty/boil-joint-noise-attribution.md new file mode 100644 index 000000000..d8911918a --- /dev/null +++ b/docs/uncertainty/boil-joint-noise-attribution.md @@ -0,0 +1,66 @@ +# Boil forecast regression: joint-perturbation attribution + +September 13, 2026. +The [completed Boil comparison](boil-canonical-forecasts.md#completed-forecasts-and-incumbent-comparison) showed much worse jug motion and heating forecasts under the experimental joint-transition model than under the incumbent selected point. +This diagnostic isolates future joint perturbations while keeping fitted candidates and their conditioning histories fixed. +It does not fit a replacement posterior or run an agent. + +## Matched intervention + +The plan selects particle indices 0, 8, 16 and 24 from each completed numerical population, 410 and 411, before inspecting their intervention outcomes. +Each uses the original bank-0/draw-0 random seed. +These eight points are diagnostic cases, not a representative posterior sample or a posterior-weighted forecast estimate. +Their original weights are retained as provenance, without renormalizing or averaging the selected subset. + +Every case receives four complete continuations from its original root: + +- All nine future joint perturbations, reproducing the archived forecast exactly. +- No future joint perturbations. +- Perturbations on the seven revolute arm joints only. +- Perturbations on the two prismatic gripper joints only. + +The entire 132-action fitting prefix, candidate parameters, initial state, physical predictions, model memory and conditioning factors remain identical. +The intervention starts only in the reserved future. +All variants consume the same Gaussian random stream, including discarded draws, so the masks do not shift later random numbers. +The future generator receives no observed future states. +The no-perturbation variant is also repeated independently in a fresh world. + +Suppressed-perturbation paths are explicitly marked as counterfactuals. +They do not retain the original Gaussian future-density fields as if those were the normalized densities of the intervened model. +A new probability model and fresh fitting are required before any variant can be assessed as a replacement posterior. + +## Verified result + +Compute job `22699007` completed in 3:34, covering all 32 intervention histories plus the eight no-perturbation repeats. +Independent reader `22699056` completed in 34 seconds. +It verifies every mask and Gaussian draw, unchanged fitting prefixes, literal model updates, native goal predicates, output moments and future-error calculations. +It also rejects deliberately altered suppressed-finger coordinates and altered prefix memory. + +| Numerical population / particle | Jug x RMSE, all perturbations (m) | None | Arm only | Fingers only | +| --- | ---: | ---: | ---: | ---: | +| 410 / 0 | 0.1552 | 0.0073 | 0.0127 | 0.0111 | +| 410 / 8 | 0.1444 | 0.0073 | 0.2300 | 0.2262 | +| 410 / 16 | 0.1508 | 0.0073 | 0.0077 | 0.1458 | +| 410 / 24 | 0.1996 | 0.0073 | 0.0077 | 0.1978 | +| 411 / 0 | 0.3092 | 0.0036 | 0.0019 | 0.0040 | +| 411 / 8 | 0.2674 | 0.0067 | 0.0054 | 0.1883 | +| 411 / 16 | 0.2742 | 0.0034 | 0.2131 | 0.2646 | +| 411 / 24 | 0.0018 | 0.0013 | 0.0019 | 0.1575 | + +Suppressing all future perturbations consistently brings these jug-x errors into the millimetre range. +The no-perturbation paths have twelve or thirteen held future frames, whereas most original variants have only one to seven. +Finger perturbations are frequently harmful, but arm-only perturbations also produce large errors in two cases. +The effect is nonlinear: the combined perturbations need not be worse than each individual mask on every candidate. +This identifies stochastic joint perturbations as a major contributor to the motion regression in these selected cases. +It does not justify removing their density factors from fitting or claiming that one mask is a validated inference model. + +Heating remains a separate problem. +Even without future joint perturbations, bubbling errors range from approximately 0.107 to 0.592 on these candidates, and some final task-goal predictions still fail. +The new law's state and parameter uncertainty, numerical exploration and heating information in the fitting prefix still need separate assessment. + +The saved fitting residuals also show that most per-joint root-mean-square deviations are below the fixed 0.001 scale, particularly for the fingers. +The next [shared-variance model](shared-variance-discrepancy.md) therefore estimates a declared variance from those residuals, preserving its uncertainty and complete likelihood, rather than choosing a smaller fixed scale from future performance. +Its posterior must be refitted under the changed law. +The production estimator remains unchanged. + +The frozen plan, source hashes, full trajectories and verification report are in `logs/uncertainty_boil_joint_noise_attribution_20260913/`. diff --git a/docs/uncertainty/shared-variance-discrepancy.md b/docs/uncertainty/shared-variance-discrepancy.md new file mode 100644 index 000000000..71dc72489 --- /dev/null +++ b/docs/uncertainty/shared-variance-discrepancy.md @@ -0,0 +1,88 @@ +# Learning a shared discrepancy variance + +September 13, 2026. +This is an offline alternative to a fixed joint-transition discrepancy scale, motivated by the [Boil intervention audit](boil-joint-noise-attribution.md). +It does not change sensor noise or production agent behavior. +Learning the scale is a model change that still requires new fitting and predictive validation. + +## Probability model + +For one selected channel, declare a single variance `v` shared across its residual history: + +``` +v ~ InverseGamma(alpha, beta) +r_t | v ~ Normal(0, v) +``` + +The inverse-gamma density is `beta**alpha / Gamma(alpha) * v**(-alpha-1) * exp(-beta/v)` on positive variance. +The scale `beta` has squared channel units. +The residuals may describe an explicitly declared transition or output discrepancy; that choice and the sharing boundary belong in the caller's probability-model identity. +This component does not establish that independent zero-mean residuals are adequate for a particular domain. + +After `n` residuals with squared sum `S`, the conditional variance has shape `a = alpha + n/2` and scale `b = beta + S/2`. +The complete normalized residual-history log density is: + +``` +log Gamma(a) - log Gamma(alpha) ++ alpha * log(beta) - a * log(b) +- n/2 * log(2*pi) +``` + +The next residual has a Student-t predictive density with `2*a` degrees of freedom and scale `sqrt(b/a)`. +Sequentially conditioning and multiplying those predictive factors gives the same complete marginal as integrating the shared variance once. +Independent Student-t draws with a fixed scale at every step would instead lose the dependence induced by the shared variance. + +For a physical future, draw one variance from the conditional inverse-gamma law and retain that draw throughout the entire continuation. +The simulator then draws its per-step Gaussian residuals conditionally on that shared value. +When evaluating the density of an observed future, update the variance sufficient statistics causally and retain each normalized Student-t factor. +Generating a future and evaluating its marginal density are different operations, even though they use the same declared model. + +The original prior is fixed before the fitting history. +Refitting the same complete data uses that original prior again; it does not treat the previous posterior as a new prior. +The component has no sensor-noise parameter and never writes a simulator state. +It raises explicit errors for invalid inputs and unrepresentable arithmetic rather than substituting a tiny variance or clipping observations. + +## Implementation and checks + +[inference_variance.py](../../predicators/code_sim_learning/inference_variance.py) supplies immutable original-prior and conditional sufficient-statistic objects, complete-history density, causal predictive factors and shared-variance draws. +The implementation is not routed into the production fitter or execution estimator. + +Compute check `22699354` passed thirteen functional tests, focused two-file type and lint checks, and the pinned formatters. +The tests independently integrate the original variance density by numerical quadrature, check Student-t predictions against a separate implementation, verify density changes under physical-unit conversion, and check dependence between squared future residuals under a common variance. +They also check repeated batch fitting and explicit rejection of invalid inputs and arithmetic overflow. + +## Boil integration + +The first physical comparison retains the previous Boil program, original scene and parameter priors, observations, 132-action prefix, proposal map, sampler settings and numerical seeds 410/411. +It replaces only the fixed 0.001 joint-transition scale with one independent shared variance per controlled joint for this episode. +Each original variance prior has `alpha=2` and `beta=1e-6`, giving the same prior mean variance as the former fixed value squared. +These are native squared units: radians squared for the seven arm joints and metres squared for the two fingers. +The prior specification is recorded explicitly; data-derived scale estimates are not substituted for the prior. + +During fitting, the nine variances are analytically integrated out, so they add no sampler proposal coordinates. +Every exact observed joint correction retains its normalized predictive density, and the output model does not add a second joint-error density for the same correction. +The candidate physical histories are unchanged under the existing exact-joint conditioning protocol; their statistical weights change. +The initial-state and model-parameter priors remain fixed. + +Native preflight `22699445` completed eight candidate checks and their independent repeats, totaling 2,112 native actions. +All native observations and literal model outputs match the archived fixed-scale prefixes exactly. +The new complete likelihood agrees with an independent gamma-integral calculation after replacing the old joint factors, with maximum difference 3.638e-12. +Repeated fresh replay and complete target evaluation are exact. +The resulting conditional root-mean-square finger scales range from approximately 0.127 to 0.288 mm on these selected candidates, below the former fixed 1 mm. +Those candidates were selected before this diagnostic; they are not samples from a newly fitted posterior. + +Fresh matched 32-particle fits `22699492_0` and `_1` are running on compute nodes. +Their complete model identity includes the numerical variance prior as well as the transition law. +Both retain the original fitting-data and physical-prior definitions, while recording the changed discrepancy model separately. +They remain unassessed until independent replicas, held-out predictions and computational cost are checked. + +Future fixture `22699602` tests repeated generated continuations, one variance draw per joint per history, and density round trips using generated observations on two fixed candidates. +It completed in 2:04 with 1,848 native actions. +Both generated histories repeat exactly and reproduce their full physical trajectories and marginal density when their generated future observations are supplied to the density evaluator. +Independent Student-t calculations differ from the recorded per-step factors by at most 2.354e-13, and complete future joint densities agree with direct gamma-integral calculations. +Separate artifact reader `22699757` completed in 20 seconds, checking four saved histories and 9,504 joint factors, future joint-output alignment, native predicates, scalar moments and literal memory. +It rejects altered shared variances, future joint readings and density factors. +Full-population forecasts still need the completed replacement fits and the corresponding verified adapter. +Reducing perturbation scale alone does not resolve the remaining heating errors or establish agent non-regression. + +Frozen component checks are in `logs/uncertainty_shared_variance_20260913/`, fitting/preflight artifacts in `logs/uncertainty_boil_shared_variance_20260913/`, and future checks in `logs/uncertainty_boil_shared_variance_forecast_20260913/`. diff --git a/predicators/code_sim_learning/inference_variance.py b/predicators/code_sim_learning/inference_variance.py new file mode 100644 index 000000000..b9eb857a1 --- /dev/null +++ b/predicators/code_sim_learning/inference_variance.py @@ -0,0 +1,149 @@ +"""Collapsed Gaussian discrepancy variance for offline trajectory inference. + +One inverse-gamma variance is shared across a channel's residual +history. This is a separately declared dynamics or output discrepancy, +not a change to sensor noise. Callers must identify the channel, +primitive time step, units, original prior and episode-sharing policy in +their model identity. No production estimator or physical state is +modified by this component. +""" +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from typing import Sequence + +import numpy as np + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_data import content_digest + + +def _square(value: float) -> float: + if not math.isfinite(value): + raise ValueError("Residuals must be finite") + result = value * value + if not math.isfinite(result): + raise ConditioningNumericalError("Residual square overflow") + return result + + +@dataclass(frozen=True) +class GaussianVariancePrior: + """Variance v has density beta**alpha/Gamma(alpha) v**(-alpha-1) + exp(-beta/v); residuals given v are independent Normal(0, v). + + beta has squared channel units. The prior is fixed before seeing the + fitted residual history. It is not replaced by a previous posterior + when fitting that same history again. + """ + alpha: float + beta: float + + def __post_init__(self) -> None: + if any(not math.isfinite(v) or v <= 0 + for v in (self.alpha, self.beta)): + raise ValueError("Positive finite shape and scale required") + + @property + def digest(self) -> str: + """Identify the normalized shared-variance law and original prior.""" + return content_digest( + json.dumps( + { + "schema": 1, + "family": + "zero_mean_gaussian_shared_inverse_gamma_variance", + "alpha": float(self.alpha), + "beta": float(self.beta) + }, + sort_keys=True).encode("utf-8")) + + def condition(self, + residuals: Sequence[float]) -> GaussianVariancePosterior: + """Condition once on a complete sequence, retaining its density.""" + try: + squared = math.fsum(_square(r) for r in residuals) + except OverflowError as exc: + raise ConditioningNumericalError("Residual sum overflow") from exc + return GaussianVariancePosterior(self, len(residuals), squared) + + +@dataclass(frozen=True) +class GaussianVariancePosterior: + """Sufficient statistics for one originally declared variance. + + Sequential predictive densities integrate the same variance, rather + than independently mixing a new variance at every step. For a future + simulation, draw one variance and retain it for that entire history. + """ + prior: GaussianVariancePrior + count: int + sum_squares: float + + def __post_init__(self) -> None: + if isinstance(self.count, bool) or not isinstance(self.count, int) or \ + self.count < 0: + raise ValueError("Residual count must be a nonnegative integer") + if not math.isfinite(self.sum_squares) or self.sum_squares < 0: + raise ValueError("Finite nonnegative residual sum required") + if self.count == 0 and self.sum_squares != 0: + raise ValueError("Empty history cannot have residual energy") + + @property + def alpha(self) -> float: + """Conditional inverse-gamma shape.""" + return self.prior.alpha + self.count / 2 + + @property + def beta(self) -> float: + """Conditional inverse-gamma scale in squared channel units.""" + result = self.prior.beta + self.sum_squares / 2 + if not math.isfinite(result): + raise ConditioningNumericalError("Variance scale overflow") + return result + + @property + def log_evidence(self) -> float: + """Normalized complete residual-history density under the prior.""" + if not self.count: + return 0. + result = (math.lgamma(self.alpha) - math.lgamma(self.prior.alpha) + + self.prior.alpha * math.log(self.prior.beta) - + self.alpha * math.log(self.beta) - + self.count / 2 * math.log(2 * math.pi)) + if not math.isfinite(result): + raise ConditioningNumericalError("Variance evidence overflow") + return result + + def log_predictive(self, residual: float) -> float: + """Student-t density for one additional exactly observed residual.""" + energy = _square(residual) + # Separate logarithms to avoid overflow in 2*pi*beta. + ratio = energy / self.beta / 2 + result = (math.lgamma(self.alpha + .5) - math.lgamma(self.alpha) - .5 * + (math.log(2 * math.pi) + math.log(self.beta)) - + (self.alpha + .5) * math.log1p(ratio)) + if not math.isfinite(result): + raise ConditioningNumericalError("Variance predictive overflow") + return result + + def advance(self, residual: float) -> GaussianVariancePosterior: + """Append new evidence while preserving the original prior.""" + try: + energy = math.fsum((self.sum_squares, _square(residual))) + except OverflowError as exc: + raise ConditioningNumericalError("Residual sum overflow") from exc + return GaussianVariancePosterior(self.prior, self.count + 1, energy) + + def draw_variance(self, rng: np.random.Generator) -> float: + """Draw one shared variance for a complete future continuation.""" + gamma = float(rng.gamma(self.alpha)) + if not math.isfinite(gamma) or gamma <= 0: + raise ConditioningNumericalError("Unrepresentable gamma draw") + result = self.beta / gamma + if not math.isfinite(result) or result <= 0: + raise ConditioningNumericalError("Unrepresentable variance draw") + return result diff --git a/tests/code_sim_learning/test_inference_variance.py b/tests/code_sim_learning/test_inference_variance.py new file mode 100644 index 000000000..26e384b4c --- /dev/null +++ b/tests/code_sim_learning/test_inference_variance.py @@ -0,0 +1,108 @@ +"""Independent integration and predictive checks for shared variances.""" +import math +from typing import Tuple + +import numpy as np +import pytest +from scipy.integrate import quad +from scipy.stats import invgamma, norm, t + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_variance import \ + GaussianVariancePosterior, GaussianVariancePrior + + +@pytest.mark.parametrize("residuals", [(), (.12, ), (.12, -.3, .05)]) +def test_normalized_history_matches_variance_quadrature( + residuals: Tuple[float, ...]) -> None: + """Integrate the original variance density, with its log-coordinate + Jacobian, independently of the conjugate implementation.""" + prior = GaussianVariancePrior(2.5, .08) + posterior = prior.condition(residuals) + + def integrand(log_variance: float) -> float: + variance = math.exp(log_variance) + density = invgamma.logpdf(variance, prior.alpha, scale=prior.beta) + density += sum( + norm.logpdf(r, scale=math.sqrt(variance)) for r in residuals) + return math.exp(density + log_variance) + + actual, error = quad(integrand, -25, 25, epsabs=1e-11) + assert error < 1e-8 + assert posterior.log_evidence == pytest.approx(math.log(actual), abs=1e-10) + + +def test_causal_updates_equal_batch_and_student_prediction() -> None: + """Every causal factor is normalized; their product equals one batch + marginal, with no repeated-prior update.""" + prior = GaussianVariancePrior(2., 1e-6) + residuals = (.0001, -.002, .0003, .0005) + state = prior.condition(()) + density = 0. + for residual in residuals: + expected = t.logpdf(residual, + df=2 * state.alpha, + scale=math.sqrt(state.beta / state.alpha)) + assert state.log_predictive(residual) == pytest.approx(expected) + density += state.log_predictive(residual) + state = state.advance(residual) + batch = prior.condition(residuals) + assert density == pytest.approx(batch.log_evidence, abs=1e-12) + assert state.alpha == batch.alpha + assert state.beta == pytest.approx(batch.beta, abs=1e-20) + assert prior.condition(residuals) == batch + assert prior.digest == GaussianVariancePrior(2., 1e-6).digest + assert prior.digest != GaussianVariancePrior(2., 1e-5).digest + + +@pytest.mark.parametrize("multiplier", [.001, 1000.]) +def test_change_of_physical_units_retains_density(multiplier: float) -> None: + """Scaling channel units transforms the prior and all density factors.""" + residuals = (.1, -.25, .05) + original = GaussianVariancePrior(3., .01).condition(residuals) + transformed = GaussianVariancePrior(3., .01 * multiplier**2).condition( + tuple(r * multiplier for r in residuals)) + assert transformed.log_evidence == pytest.approx( + original.log_evidence - len(residuals) * math.log(multiplier), + abs=1e-11) + assert transformed.log_predictive(.2 * multiplier) == pytest.approx( + original.log_predictive(.2) - math.log(multiplier), abs=1e-11) + + +def test_future_draws_share_the_conditioned_variance() -> None: + """A common variance induces dependence in squared future residuals; + independent Student draws would lose that history dependence.""" + state = GaussianVariancePrior(4., .03).condition((.1, -.2)) + rng = np.random.default_rng(501) + variance = np.asarray([state.draw_variance(rng) for _ in range(40000)]) + pair = rng.normal(size=(len(variance), 2)) * np.sqrt(variance[:, None]) + expected_mean = state.beta / (state.alpha - 1) + expected_second = state.beta**2 / ((state.alpha - 1) * (state.alpha - 2)) + assert variance.mean() == pytest.approx(expected_mean, rel=.02) + assert np.mean(pair[:, 0]**2 * pair[:, 1]**2) == pytest.approx( + expected_second, rel=.08) + assert np.mean(pair[:, 0]**2 * pair[:, 1]**2) > expected_mean**2 * 1.15 + + +@pytest.mark.parametrize("alpha,beta", [(0., 1.), (-1., 1.), (1., 0.), + (math.inf, 1.), (1., math.nan)]) +def test_invalid_prior_rejected(alpha: float, beta: float) -> None: + """A degenerate variance prior is not silently repaired.""" + with pytest.raises(ValueError): + GaussianVariancePrior(alpha, beta) + + +def test_invalid_history_and_numerical_overflow_rejected() -> None: + """Setup errors and unrepresentable arithmetic remain explicit.""" + prior = GaussianVariancePrior(2., .01) + for count, energy in [(-1, 0.), (True, 0.), (0, .1), (1, -.1), + (1, math.inf)]: + with pytest.raises(ValueError): + GaussianVariancePosterior(prior, count, energy) + with pytest.raises(ValueError): + prior.condition((math.nan, )) + with pytest.raises(ConditioningNumericalError): + prior.condition((1e300, )) + with pytest.raises(ConditioningNumericalError): + prior.condition(()).log_predictive(1e300) From 383e8cc7d401292786debf0ae2c3ee1f18dd16a0 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 18:58:50 -0400 Subject: [PATCH 68/94] Update uncertainty validation status and completed Balloons comparison --- docs/uncertainty/balloons-prefix-forecasts.md | 5 ++++- docs/uncertainty/implementation-progress.md | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/balloons-prefix-forecasts.md b/docs/uncertainty/balloons-prefix-forecasts.md index f42095956..8c402aa35 100644 --- a/docs/uncertainty/balloons-prefix-forecasts.md +++ b/docs/uncertainty/balloons-prefix-forecasts.md @@ -136,5 +136,8 @@ That finite-sample result is not a proof that the model's true win probability i Future-density integration is also unstable: seed 620's two density banks have log means approximately -290,981 and -434,259, while seed 621's are approximately -50,634 and -52,053. Retain those bank discrepancies rather than treating a finite mixture density as a converged likelihood estimate. Both fitting populations descend from one initial ancestor, so neither the fitting nor the future integration is established as adequate. -The scheduled incumbent comparison `22693842` is still waiting for resources at this update. +The scheduled incumbent comparison `22693842` has now completed, with matching source and verification identities. +The incumbent's box-height RMSE is approximately 0.01000 m and speed RMSE 0.04805 m/s, versus 0.16170/0.09725 m and 0.60338/0.34613 m/s for the two new populations. +The incumbent also predicts the final win incorrectly; the new populations do not recover it in their generated banks and additionally predict spurious bursts. +Some individual balloon-coordinate errors improve, so retain the complete feature table rather than describing every metric as worse. These outcomes require diagnosis of state/transition assumptions and numerical exploration, not promotion to live planning. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index b7edeee5e..59efbc8e5 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -42,7 +42,8 @@ All five still contradict exact recorded outputs, starting at actions 1-34 depen The [Balloons population forecast adapter](balloons-prefix-forecasts.md#complete-population-forecast-adapter) now passes native replay, nonuniform weighted-summary, zero-density and no-refit checkpoint checks, including twelve malformed-result rejections. Both prefix fits, forecast/verifier jobs `22693026` and `22693027`, and paired report `22693044` have completed. Every positive-weight particle keeps its original weight, receives eight generated suffixes and eight separate conditional-density draws, and must reproduce its saved prefix likelihood. -The [Balloons incumbent control](balloons-incumbent-control.md) completed its full fit as `22693806` in 39:11 and its independent complete-trajectory verification as `22693822` in 17 seconds; the comparison remains queued. +The [Balloons incumbent control](balloons-incumbent-control.md) completed its full fit as `22693806` in 39:11 and its independent complete-trajectory verification as `22693822` in 17 seconds; comparison `22693842` has also completed. +The incumbent has lower box-height and speed errors than both new populations on this recording, although it also misses the final goal. The new populations disagree by 0.21909 m RMS in box height and up to 0.59913 in an individual burst probability, with large density-bank disagreements as well. Both assign zero empirical probability to the final win among generated futures; finite forecast completion does not establish reliable uncertainty. All three segments survived, but the incumbent's publication rules retained the original applied parameters. @@ -74,6 +75,14 @@ The selected-point forecast matches the task-goal predicate on all 132 reserved The combined comparison `22691239` has completed and shows substantially worse motion and heating predictions from the new model on this recording. Its two jug-x RMSEs are 0.12922/0.12632 m versus 0.00170 m for the incumbent; boiled Brier scores are 0.25314/0.22527 versus 0.00758. Both new populations also retain one initial ancestor, so state/discrepancy assumptions and numerical exploration must be isolated before larger-budget fits or live use. +The [Boil future-joint intervention](boil-joint-noise-attribution.md) now isolates one major motion failure: suppressing future perturbations reduces jug-x errors from tens of centimetres to millimetres on eight preselected fixed candidates, while holding their entire fitting prefixes and parameters constant. +Both finger and arm perturbations can be harmful, and substantial heating errors remain after removing them. +The native audit and independent reader completed all 32 intervention histories, eight additional repeats, mask/RNG checks and deliberate corruption rejections. +The [shared-variance discrepancy](shared-variance-discrepancy.md) therefore replaces a fixed transition scale in a separate offline model with a normalized inverse-gamma variance per joint, integrated during fitting and shared across each generated future. +Thirteen component tests, focused type/lint/format checks, eight native-prefix comparisons and 2,112 native actions pass; complete target factors agree with independent integration within 3.638e-12. +Matched 32-particle Boil refits `22699492_0` and `_1` are running with the original physical priors, program, data, sampler configuration and seeds retained. +The corresponding future fixture completed 1,848 native actions with exact generation/density round trips; independent reader `22699757` also passed, verifying four histories and 9,504 joint factors and rejecting three corruption classes. +This is a model alternative under evaluation, not a validated posterior or a deployed replacement. This compares selected-point and posterior-mean predictions while retaining the explicit differences in initial-state treatment and discrepancy model; it is not an estimator-only ablation or an agent performance result. The separate [Balloons prefix experiment](balloons-prefix-inference.md) now fits only 64 actions and reserves 171 for later forecasts. From 72e6cecc234999647fd8b8af41cd5b8447e9c31c Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 19:23:24 -0400 Subject: [PATCH 69/94] Record shared-variance forecast validation and Bridge conditioning audit --- docs/uncertainty/bridge-incumbent-control.md | 16 +++++++++++--- docs/uncertainty/bridge-initial-scene.md | 22 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 10 +++++++++ .../shared-variance-discrepancy.md | 19 +++++++++++++++- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/uncertainty/bridge-incumbent-control.md b/docs/uncertainty/bridge-incumbent-control.md index a6bacfddf..f0ba9c7f8 100644 --- a/docs/uncertainty/bridge-incumbent-control.md +++ b/docs/uncertainty/bridge-incumbent-control.md @@ -30,8 +30,8 @@ Selected-point predictions are not the incumbent's full interval or planning ens | Job | Purpose | Verified state on September 13 | | --- | --- | --- | | `22696941` | Check full-prefix preparation and helper/manual native parity at default dwell, dwell 20 and dwell 40. | Completed; all three pairs exact, 7,116 native actions. | -| `22697046` | Run the complete incumbent fitter, then two complete causal forecasts. | Running. | -| `22697048` | Independently reproduce the selected forecast and assess the 586-action suffix. | Depends on successful fitting. | +| `22697046` | Run the complete incumbent fitter, then two complete causal forecasts. | Completed in 1:11:39; 304,172 native actions including fitting and forecast repetitions. | +| `22697048` | Independently reproduce the selected forecast and assess the 586-action suffix. | Completed in 44 seconds; all 1,186 native actions reproduce the selected forecast exactly. | The preflight compares three pairs of full trajectories and checks that changing bond dwell changes predictions. The fitting driver requires that certificate, preserves all six declared parameters, and records fitted versus actually applied values separately. @@ -44,6 +44,16 @@ The preflight initially waited because all 64 CPUs on that node were allocated; These scheduler states are dated observations, not permanent status claims. The frozen bundle is `logs/uncertainty_bridge_incumbent_prefix600_20260913/`. +## Completed selected-point control + +The complete 600-action segment survives the incumbent preparation and fitting pipeline. +The parameters actually used for prediction retain `glue_latch=0.6`, `drip_r=0.0175`, `drip_h=0.0125`, `glue_rate=0.2` and `bond_dist=0.022`, while `bond_dwell` is approximately 25.9896. +The report preserves the optimizer notes and publication decisions separately from those applied values. +Both complete causal predictions repeat exactly, and the independent manual native reconstruction reproduces every saved predicted observation. +The 586-action suffix contains two mismatched glue readings against both the clean and noisy reference channels. +Other exact observations still disagree, including joint positions and some holding and orientation fields; this control does not satisfy the new probability model's exact-output constraints. +These are offline prediction measurements on the revised fixed program, not an agent solve-rate result or a comparison with a completed replacement posterior. + ## Remaining comparison requirements The revised subclass's geometry, attachments and other exact observations still need a complete joint probability target before the replacement arm can be fitted. @@ -52,5 +62,5 @@ Those trajectories repeat exactly but still contradict exact recorded outputs, s The exact-rate reference supplies only one component of that target. The physical initial-state inventory, supported geometry and remaining continuous exact-output representations must stay explicit. Use the same fixed program and 600-action prefix for both estimator arms, and separately identify any new transition-discrepancy assumptions or state-inference approximations. -There is no estimator advantage, posterior adequacy or agent-performance result to report from a queued incumbent control. +The completed incumbent control supplies one comparison arm; it does not establish estimator advantage, posterior adequacy or agent performance. The incumbent remains the production default while Stage A/B validation continues. diff --git a/docs/uncertainty/bridge-initial-scene.md b/docs/uncertainty/bridge-initial-scene.md index dd764c055..7cf1ea390 100644 --- a/docs/uncertainty/bridge-initial-scene.md +++ b/docs/uncertainty/bridge-initial-scene.md @@ -121,6 +121,28 @@ These are selected candidate failures, not proof that the full conditional suppo The next target must represent the remaining continuous exact trajectory constraints and retain the geometry-dependent discrete glue likelihood. Neither a completed replay nor a mismatch count is a continuous exact-conditioning method. +## Conditional robot-joint trajectory audit + +The next isolated audit preserves all five feasible initial roots and compares their native continuations with a declared conditional joint-transition model on the same 600-action prefix. +After each native base step and before the learned model-memory update, it conditions the nine controlled joint positions on their exact observations, retaining native joint velocities and existing physical constraints. +It never restores a later recorded scene or imports private model memory. +One independent inverse-gamma variance per controlled joint is shared across the entire prefix, using the [tested variance component](shared-variance-discrepancy.md) with `alpha=2` and `beta=1e-6` in native squared units. +Every conditional correction retains its normalized Student-t factor; these factors alone are not the complete likelihood of the remaining outputs. + +Compute job `22700282` completed ten trajectories and their fresh-world repetitions in 3:48, totaling 12,000 native actions. +All five native controls reproduce the previously saved failures exactly, including every prediction and learned-memory value. +Both modes repeat exactly for every root. +The conditional paths remove all controlled-joint mismatches and the exact finger-readout mismatches, while retaining other contradictions. +The all-rest root drops from eighteen mismatching channels to eight, with the same sixteen glue-reading mismatches. +Its remaining channels are the six robot pose/orientation fields and two glue fields. +The four moving roots each retain twelve mismatching channels, including holding and glue discrepancies. +Thus exact joint conditioning supplies one valid trajectory component but does not resolve the Bridge support problem. +Independent reader `22700337` completed in fifteen allocation seconds, checking all 27,000 causal Student-t factors against separate gamma-function calculations, the full shared-variance integral, posterior sufficient statistics and alignment of exact joints with the observed and predicted histories. +The maximum per-factor discrepancy is 6.8301e-13. +It also reconstructed every remaining mismatch count and rejected corrupted joint values, factors, variance statistics and missing transitions. +The remaining work is an explicit representation for the other continuous exact robot outputs and supported geometry-dependent glue/holding histories, while preserving this same fitting split and root prior. +Artifacts are in `logs/uncertainty_bridge_joint_conditioning_20260913/`. + Full 600-action exact-output support, independent density verification, the complete conditional inference target and posterior adequacy remain required. In particular, the exact-rate component assumes an eligible deposition schedule and cannot simply replace the full geometry-dependent glue likelihood. This root construction alone supplies neither a usable posterior nor evidence of improved agent performance. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 59efbc8e5..fbfad4a4d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,16 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The shared-variance Boil forecast adapter has now completed its compute validation as `22699944`, with 2,376 native actions, exact checkpoint recovery, independent weighted summaries and twelve corruption rejections. +Full forecast/reader jobs `22700066_0` and `22700067_1` are queued behind the two live replacement fits; comparison `22700111` depends on both forecasts succeeding. +These preserve the previous fitting and forecast seeds and compare the shared-variance law with the fixed-variance law and incumbent selected point. +The [Bridge incumbent control](bridge-incumbent-control.md) has also completed its 600-action fit and independently verified 586-action forecast assessment. +It records two suffix glue-reading mismatches, while other exact channels still disagree and the complete replacement target remains unfinished. +The follow-up [Bridge joint-conditioning audit](bridge-initial-scene.md#conditional-robot-joint-trajectory-audit) reproduced all five original root failures and compared exact-joint conditional paths with fresh repeats, totaling 12,000 native actions. +Independent verification passed 27,000 normalized joint factors and four corruption controls. +Joint and finger-readout mismatches disappear, but all sixteen glue mismatches remain on the all-rest root, alongside six robot pose/orientation channels. +The next Bridge work must resolve those remaining conditional-output and geometric constraints; more replay repetitions alone cannot supply a supported posterior. + The [Bridge glue attribution](bridge-glue-attribution.md) now proves a structural contradiction in the transferred parameterized program, beyond the original no-op control. Four exact `0, 0.2, 0.4, 1` progressions cannot arise from its constant-increment deposition law for any parameter setting under the preserved memory contract. Six geometry-conditioned development cases and an independent 7,116-step direct-rule verifier complete; clean recorded geometry still exposes wrong face selection and timing, and a saturation correction alone is insufficient. diff --git a/docs/uncertainty/shared-variance-discrepancy.md b/docs/uncertainty/shared-variance-discrepancy.md index 71dc72489..0449f1502 100644 --- a/docs/uncertainty/shared-variance-discrepancy.md +++ b/docs/uncertainty/shared-variance-discrepancy.md @@ -82,7 +82,24 @@ Both generated histories repeat exactly and reproduce their full physical trajec Independent Student-t calculations differ from the recorded per-step factors by at most 2.354e-13, and complete future joint densities agree with direct gamma-integral calculations. Separate artifact reader `22699757` completed in 20 seconds, checking four saved histories and 9,504 joint factors, future joint-output alignment, native predicates, scalar moments and literal memory. It rejects altered shared variances, future joint readings and density factors. -Full-population forecasts still need the completed replacement fits and the corresponding verified adapter. +Completed-population adapter check `22699944` passed in 3:54, including 2,376 native actions. +It preserves the entire completed checkpoint and original particle weights without reevaluating the fitting target during checkpoint recovery. +Independent mixture calculations reproduce weighted observation means, variances, native means and event probabilities, including retained zero-density mass. +Native generated and observed-future density trajectories repeat exactly through the adapter. +Twelve deliberate corruption cases are rejected: missing and duplicate histories, changed weights, incorrect roles and banks, negative variances, inconsistent shared-variance draws, changed future joints and joint factors, unfinished checkpoints, changed checkpoint weights and changed identities. + +The complete forecast plan reconstructs the changed statistical and runtime identities from frozen inputs and checks the unchanged data, program, physical prior, scene map and sampler settings against the previous fits. +It records expected final report and checkpoint locations without hashing a mutable running report as though it were complete. +Forecast jobs `22700066_0` and `22700067_1` are queued behind their respective fit jobs. +Each retains every positive-weight particle and generates two banks of four 132-action futures, plus a separate conditional-density trajectory for the recorded future. +The same numerical and forecast seeds are retained from the fixed-variance comparison. +An independent reader reconstructs all weighted summaries from complete saved histories before the forecast job can succeed. + +Dependent comparison `22700111` will assess differences between replicas, the previous fixed-variance populations and the incumbent selected-point forecast. +Independent headline metric reconstruction already matches both previous verified forecast reports. +The comparison retains numerical collapse and simulation cost alongside prediction errors; a completed forecast alone is not numerical acceptance. +The fixed/shared-variance comparison changes the discrepancy law, while the incumbent comparison additionally changes initial-state treatment and output discrepancy. +Neither comparison is a new live-agent seed or establishes solve-rate non-regression. Reducing perturbation scale alone does not resolve the remaining heating errors or establish agent non-regression. Frozen component checks are in `logs/uncertainty_shared_variance_20260913/`, fitting/preflight artifacts in `logs/uncertainty_boil_shared_variance_20260913/`, and future checks in `logs/uncertainty_boil_shared_variance_forecast_20260913/`. From d47bc3adeb10e6acfdfd8804881e890cf3c956c5 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 20:33:42 -0400 Subject: [PATCH 70/94] Add prior-preserving categorical guidance and document Bridge support checks --- docs/uncertainty/bridge-glue-support.md | 130 ++++++++++++++++++ docs/uncertainty/implementation-progress.md | 13 +- .../inference_categorical.py | 98 +++++++++++++ .../test_inference_categorical.py | 85 ++++++++++++ 4 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/bridge-glue-support.md create mode 100644 predicators/code_sim_learning/inference_categorical.py create mode 100644 tests/code_sim_learning/test_inference_categorical.py diff --git a/docs/uncertainty/bridge-glue-support.md b/docs/uncertainty/bridge-glue-support.md new file mode 100644 index 000000000..5aa072c67 --- /dev/null +++ b/docs/uncertainty/bridge-glue-support.md @@ -0,0 +1,130 @@ +# Bridge glue support after joint conditioning + +September 13, 2026. +These are offline development diagnostics for the [uncertainty simplification plan](simplification-proposal.md). +They retain the revised subclass, original scene law and 600-action fitting prefix. +They do not change the production agent or establish posterior adequacy. + +## Radius and height alone do not fix the saved path + +The [joint-conditioning audit](bridge-initial-scene.md#conditional-robot-joint-trajectory-audit) removed joint and finger-readout contradictions but retained sixteen glue-reading mismatches on its all-rest root. +The next diagnostic partitions the original `drip_r` and `drip_h` boxes at every eligibility change point on the saved pre-rule geometry. +Within each resulting cell, the nearest eligible face is constant at every step. +It tests an interior representative of every positive-area cell and checks the selected/default choices through the literal deposition function. +The screen compares deposition labels where the observed previous glue level is not already latched; full causal updates and bond consumption require separate native validation. + +Compute job `22701184` completed the five-root screen, and independent reader `22701286` verified 2,419 threshold cells, the saved geometry arrays and three corruption controls. +Independent rotation calculations differ from the saved geometry by at most 3.3307e-16 m. +The all-rest root improves from two deposition-label errors to one, but no tested cell has zero errors. +The four moving-root cases retain thirteen errors because their paths do not supply the required held-bottle deposition geometry. +The first screen attempt, `22701022`, completed its calculations but failed to serialize integer array indices; the corrected attempt preserves its computation and selection rule. +That failure is a diagnostic reporting error, not an agent or model failure. + +The all-rest path supplies a concrete incompatibility: + +| Recorded event | Necessary condition on the saved geometry | +| --- | --- | +| Deposition at action 87 on `span0.glue_end_a` | `drip_h >= 0.02439433018` m. | +| Deposition at action 200 on `span2.glue_end_a` | `drip_r >= 0.01606379755` m. | +| No deposition at action 57 on `span0.glue_end_a` | The candidate instead has horizontal distance 0.00483451437 m and height 0.02003017178 m, so the two necessary bounds force deposition. | + +No competing face can become eligible at action 57 anywhere in the declared radius/height boxes. +This is a contradiction for the frozen geometry and deposition-label screen, not proof that every initial scene or every causal parameter trajectory is unsupported. + +## Nearby initial scenes + +The next native audit perturbs eight XY proposal coordinates for the three spans and the bottle around the all-rest root. +It uses predetermined unit-coordinate changes of +/-0.10 and +/-0.25 where those remain inside the original coordinate domain. +The root map, original prior, proposal-density corrections, default model parameters and conditional joint law remain unchanged. +These thirty candidates are deterministic support probes, not posterior samples or estimates of prior mass. + +Job `22701392` completed in 3:13 with 29 feasible scenes, one retained geometry rejection and 34,800 native actions. +Every feasible 600-action path repeated exactly, including the original baseline failure. +Four candidates reduced the complete glue-reading mismatches from sixteen to thirteen without holding errors; other changes produced much larger errors. +Independent reader `22701745` checked all 156,600 conditional joint factors and 420 original-prior/proposal density terms. +Maximum discrepancies were 6.8479e-13 for joint factors and 8.8818e-16 for density ratios. +It also rejected a deliberately altered original-prior/proposal factor. + +The radius/height screen was then applied to all 29 verified feasible paths. +The one rejected geometry remains identified in the upstream report and is not silently promoted to a successful path. +Job `22701914` completed in 44 seconds, and independent reader `22701927` checked all 73,082 threshold cells in 26 seconds. +Every path still has at least one deposition-label error at its best threshold cell. +Thus these local scene changes and threshold adjustments do not supply exact glue support. +They do not justify claiming that the full original scene law has empty support. + +## Explicit stochastic reach trial + +A separate discrepancy model adds an unobserved independent reach perturbation before each learned glue update: + +``` +eta_t ~ Normal(0, 0.005**2) # metres +effective_drip_h_t = drip_h + eta_t +``` + +The other eligibility conditions, face ranking, glue accumulation, drainage and bond rules remain the same. +The recorded glue readings remain exact; they are not assigned sensor noise or a likelihood floor. +This is a changed stochastic simulator model, with its own identified assumptions, rather than a claim that the previous deterministic model already explained the recording. +The reach scale is a declared development assumption and still requires predictive assessment. + +For a fixed pre-rule scene, the eligible face changes only at finitely many reach thresholds. +The conditional driver enumerates all intervals between those thresholds and evaluates the unchanged model update at one interior point per interval. +It sums the normalized Gaussian masses of intervals that produce exactly the observed glue outputs. +It eliminates the unobserved perturbation only after checking that every compatible interval produces identical complete next model memory and commands. +The retained interval mass is part of the conditional likelihood; choosing a compatible representative does not make that mass one. +No compatible interval gives zero likelihood and terminates that candidate as unsupported. +A future generator for this model must draw fresh reach perturbations without consulting future glue readings. + +The initial native trial uses the same all-rest root, the screened radius/height pair and bond dwell 25, derived from the recorded first-bond step and the candidate dwell counter. +The original dwell 30 is retained as a negative control. +Job `22702160` completed both fresh repetitions of each case in 1:01, totaling 2,366 native actions. +The dwell-25 case matches all glue readings over the complete 600-action prefix and retains an event log-probability of approximately -6.38423. +The dwell-30 case matches the first 582 actions and correctly has no compatible transition at the recorded bond event, action 583. +The positive case still has six exact robot pose/orientation channels to represent in the complete output model. +Independent verifier `22702255` completed in ten seconds. +It checked all 1,471 reach intervals using separate Gaussian integration, reproduced every literal rule and memory update, and verified the retained joint factors. +The maximum interval log-mass discrepancy was 3.4107e-13 and the maximum joint-factor discrepancy was 6.8301e-13. +Compatible intervals also agree on their command sequences, so marginalizing the perturbation does not discard a hidden branch of future model behavior in these cases. +Changed probabilities, interval boundaries, compatibility labels and model memory are rejected. +The two earlier setup attempts failed at the public-feature projection boundary; the corrected driver checks the callback's public features against the candidate world and records public joint metadata separately. +Neither setup failure is an agent result. + +## Complete output composition at the supported point + +Job `22702471` composes the supported path with the existing continuous-output components: correlated scalar position errors, a coupled robot-orientation error and the checked exact finger readout. +The other recorded noisy fields retain their original sensor likelihood, and exact fields still reject contradictions. +The composition includes the original root-density correction, conditional joint factors, normalized reach-event masses and the exact-rate conditioning density `1 / 0.95`, retained once at the first partial glue increment. +Initial and remaining output factors sum to the complete output likelihood, and repeated scoring is exact. +Independent reader `22702562` completed in 34 seconds with zero output-accounting discrepancy. +It checks scalar innovations and sensor factors separately, verifies all 1,202 observed/predicted finger readouts and uses an independent quaternion conversion with the previously checked orientation-density kernel. + +The resulting finite composed log weight is approximately -11,013,135, so support alone is not a useful inference starting point. +Three noisy orientation channels dominate the poor fit: `leg1.pitch`, `span0.roll` and `span1.roll` contribute approximately -7.414 million, -1.883 million and -1.826 million respectively. +Their initial-observation penalties are already large; the sampled resting-face choices must be investigated before launching a posterior from this point. +The original six-face prior remains unchanged, and a diagnostic face selection must not be misreported as a posterior sample or a normalized importance proposal. +The dwell-30 control retains zero complete likelihood at its exact glue contradiction. + +These results do not establish a complete normalized Bridge target, a reliable posterior, better predictions or unchanged agent performance. +The remaining requirements include a complete parameter/scene inference map with full prior and data identities, adequate exploration, and causal future prediction tests. + +## Initial orientation guidance with the original prior retained + +Native scan `22702774` evaluates all six resting faces of each body using only its initial public noisy orientation and height readings. +The preferred choices correct `leg1` from face 3 to 2, `span0` from face 4 to 0, `span1` from face 5 to 0 and the bottle from face 1 to 0. +The other two blocks retain their original faces. +Independent checks reproduce the Gaussian factors, represented block rotations and exact native height readout for all 36 cases, and reject four corrupted reports. +These maximum-score selections are diagnostic points, not posterior draws. + +The shared `CategoricalProposal` component converts that guidance into a normalized sampling proposal while retaining all six original prior cases. +For original masses `p_i` and guide likelihoods `L_i`, it samples from `q_i = 0.25 p_i + 0.75 p_i L_i / sum_j(p_j L_j)` and returns `log(p_i / q_i)`. +The caller must retain the likelihood used to construct the guide; it does not become the physical prior. +An impossible guide case still has positive defensive prior mass, and a numerically unrepresentable sampling interval is rejected explicitly. +Sixteen functional tests, focused type/lint checks and the pinned formatters pass in compute job `22702995`. +The tests recover prior moments and an independently specified posterior by exact finite enumeration, including a strongly concentrated guide and impossible guide cases. + +Bridge adapter check `22703098` recovers all seven diagnostic roots exactly through the unchanged native initializer. +It checks all 36 per-body density corrections and sums all 46,656 joint face combinations back to unit original-prior mass. +Moving-body coordinates remain unchanged and acquire no spurious resting-face factor. +This validates a proposal boundary for the planned inference map; it does not supply a posterior population or satisfy the prediction gate. + +Artifacts are in `logs/uncertainty_bridge_joint_glue_screen_20260913/`, `logs/uncertainty_bridge_local_scene_20260913/`, `logs/uncertainty_bridge_local_glue_screen_20260913/` and `logs/uncertainty_bridge_reach_discrepancy_20260913/`. +Orientation diagnostics and the face proposal are in `logs/uncertainty_bridge_orientation_support_20260913/` and `logs/uncertainty_bridge_face_proposal_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index fbfad4a4d..406f94305 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -13,7 +13,8 @@ Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. The shared-variance Boil forecast adapter has now completed its compute validation as `22699944`, with 2,376 native actions, exact checkpoint recovery, independent weighted summaries and twelve corruption rejections. -Full forecast/reader jobs `22700066_0` and `22700067_1` are queued behind the two live replacement fits; comparison `22700111` depends on both forecasts succeeding. +Both replacement Boil fits completed all 32 stages with 7,357 and 7,293 evaluations, but each retains one initial ancestor. +Full forecast/reader jobs `22700066_0` and `22700067_1` are now running; comparison `22700111` depends on both forecasts succeeding. These preserve the previous fitting and forecast seeds and compare the shared-variance law with the fixed-variance law and incumbent selected point. The [Bridge incumbent control](bridge-incumbent-control.md) has also completed its 600-action fit and independently verified 586-action forecast assessment. It records two suffix glue-reading mismatches, while other exact channels still disagree and the complete replacement target remains unfinished. @@ -21,6 +22,16 @@ The follow-up [Bridge joint-conditioning audit](bridge-initial-scene.md#conditio Independent verification passed 27,000 normalized joint factors and four corruption controls. Joint and finger-readout mismatches disappear, but all sixteen glue mismatches remain on the all-rest root, alongside six robot pose/orientation channels. The next Bridge work must resolve those remaining conditional-output and geometric constraints; more replay repetitions alone cannot supply a supported posterior. +The [Bridge glue-support follow-up](bridge-glue-support.md) independently checked 73,082 radius/height cells across 29 repeated nearby scene trajectories, but none supplied exact deposition support. +A separate stochastic-reach trial retains exact glue observations and their normalized transition probabilities rather than assigning them sensor noise. +Its first native case supports all 600 glue updates with bond dwell 25; the dwell-30 control correctly rejects action 583. +Independent verification passed all 1,471 reach intervals, literal model-memory updates and four corruption controls. +The continuous-output composition and retained exact-rate factor now pass reader `22702562`, but the finite supported point has very poor noisy block-orientation likelihood. +Three initial resting-face choices dominate that error, motivating a native orientation-support diagnostic before fitting. +The initial-face scan and independent reader now cover all 36 body/face cases. +The new prior-preserving categorical proposal passes sixteen functional tests and focused static/format checks; its Bridge adapter recovers seven native roots and verifies all 46,656 joint face weights. +Complete conditional-trajectory and output checks for these face changes are running as `22703020` after correcting a narrow-interval probability calculation exposed by the independent reader. +The complete inference map, adequate exploration and future-prediction assessment are still required before a usable Bridge posterior can be claimed. The [Bridge glue attribution](bridge-glue-attribution.md) now proves a structural contradiction in the transferred parameterized program, beyond the original no-op control. Four exact `0, 0.2, 0.4, 1` progressions cannot arise from its constant-increment deposition law for any parameter setting under the preserved memory contract. diff --git a/predicators/code_sim_learning/inference_categorical.py b/predicators/code_sim_learning/inference_categorical.py new file mode 100644 index 000000000..f485022a5 --- /dev/null +++ b/predicators/code_sim_learning/inference_categorical.py @@ -0,0 +1,98 @@ +"""Defensive proposals for discrete cases of an unchanged physical prior. + +Observation guidance changes sampling frequencies only. Retain the +returned prior/proposal correction and every observation factor used by +the guide. +""" +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass +from typing import Tuple + +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError +from predicators.code_sim_learning.inference_data import content_digest + + +@dataclass(frozen=True) +class CategoricalPoint: + """A selected case and its original-prior/proposal log ratio.""" + index: int + log_weight: float + + +@dataclass(frozen=True) +class CategoricalProposal: + """Mix a positive categorical prior with a likelihood-guided proposal. + + The guide is proportional to prior mass times the supplied guide + likelihood. Impossible guide cases still receive defensive prior + mass. This class does not condition the physical prior on the guide. + """ + prior: Tuple[float, ...] + log_guide: Tuple[float, ...] + prior_mass: float = .25 + + def __post_init__(self) -> None: + prior = tuple(float(p) for p in self.prior) + guide = tuple(float(v) for v in self.log_guide) + if not prior or len(prior) != len(guide) or any( + not math.isfinite(p) or p <= 0 or p > 1 for p in prior): + raise ValueError("Positive normalized prior masses are required") + total = math.fsum(prior) + if not math.isclose(total, 1., rel_tol=0., abs_tol=1e-12): + raise ValueError("Prior masses must sum to one") + if any(math.isnan(v) or v == math.inf for v in guide) or \ + all(v == -math.inf for v in guide): + raise ValueError("Guide needs a finite case and no NaN or +inf") + if not 0 < self.prior_mass < 1: + raise ValueError("Defensive prior mass must lie strictly in (0,1)") + object.__setattr__(self, "prior", tuple(p / total for p in prior)) + object.__setattr__(self, "log_guide", guide) + # A positive mathematical mass may have no representable interval + # after cumulative rounding. Do not silently omit such a case. + probabilities = self.probabilities + previous = 0. + for i in range(len(prior)): + boundary = math.fsum(probabilities[:i + 1]) + if boundary <= previous or (i < len(prior) - 1 and boundary >= 1): + raise ConditioningNumericalError( + "Categorical probability interval is not representable") + previous = boundary + + @property + def probabilities(self) -> Tuple[float, ...]: + """Normalized proposal probabilities, including defensive mass.""" + largest = max(self.log_guide) + scaled = tuple(p * math.exp(v - largest) + for p, v in zip(self.prior, self.log_guide)) + total = math.fsum(scaled) + masses = tuple(self.prior_mass * p + (1 - self.prior_mass) * w / total + for p, w in zip(self.prior, scaled)) + normalizer = math.fsum(masses) + return tuple(p / normalizer for p in masses) + + @property + def digest(self) -> str: + """Identify prior, guide and proposal separately from data factors.""" + return content_digest( + json.dumps({ + "schema": 1, + "proposal": asdict(self) + }, sort_keys=True).encode("utf-8")) + + def transform(self, unit: float) -> CategoricalPoint: + """Map one unit uniform to a case, retaining its density correction.""" + if not math.isfinite(unit) or not 0 <= unit <= 1: + raise ValueError("Expected a finite unit coordinate") + probabilities = self.probabilities + index = len(probabilities) - 1 + for i in range(len(probabilities) - 1): + if unit < math.fsum(probabilities[:i + 1]): + index = i + break + return CategoricalPoint( + index, + math.log(self.prior[index]) - math.log(probabilities[index])) diff --git a/tests/code_sim_learning/test_inference_categorical.py b/tests/code_sim_learning/test_inference_categorical.py new file mode 100644 index 000000000..8456eb7fe --- /dev/null +++ b/tests/code_sim_learning/test_inference_categorical.py @@ -0,0 +1,85 @@ +"""Exact finite-case references for prior-preserving discrete guidance.""" +import math + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_categorical import \ + CategoricalProposal +from predicators.code_sim_learning.inference_conditioning import \ + ConditioningNumericalError + + +@pytest.mark.parametrize("strength", [0., 1., 10000.]) +def test_guidance_preserves_prior_and_likelihood(strength: float) -> None: + """Exact enumeration recovers prior moments and a separate posterior.""" + prior = (.1, .2, .3, .4) + guide = CategoricalProposal(prior, + (0., -strength, -2 * strength, -math.inf), .25) + likelihood = (.8, .1, .5, .2) + evidence, moment, first, second = 0., 0., 0., 0. + previous = 0. + for i, probability in enumerate(guide.probabilities): + point = guide.transform(previous + probability / 2) + assert point.index == i + mass = probability * math.exp(point.log_weight) + first += mass * i + second += mass * i**2 + evidence += mass * likelihood[i] + moment += mass * likelihood[i] * i + assert probability >= .25 * prior[i] - 1e-15 + previous = math.fsum(guide.probabilities[:i + 1]) + assert first == pytest.approx(sum(i * p for i, p in enumerate(prior))) + assert second == pytest.approx(sum(i**2 * p for i, p in enumerate(prior))) + expected = sum(p * y for p, y in zip(prior, likelihood)) + assert evidence == pytest.approx(expected) + assert moment / evidence == pytest.approx( + sum(i * p * y + for i, (p, y) in enumerate(zip(prior, likelihood))) / expected) + + +def test_boundaries_reference_probabilities_and_identity() -> None: + """Intervals agree with a directly normalized categorical Bayes guide.""" + prior, likelihood = (.2, .3, .5), (.8, .4, .1) + guide = CategoricalProposal(prior, tuple(math.log(x) for x in likelihood)) + mass = sum(p * y for p, y in zip(prior, likelihood)) + expected = tuple(.25 * p + .75 * p * y / mass + for p, y in zip(prior, likelihood)) + assert guide.probabilities == pytest.approx(expected) + assert guide.transform(0.).index == 0 + assert guide.transform(1.).index == 2 + for i in (0, 1): + boundary = math.fsum(guide.probabilities[:i + 1]) + assert guide.transform(np.nextafter(boundary, 0.)).index == i + assert guide.transform(boundary).index == i + 1 + assert guide.digest == CategoricalProposal(prior, guide.log_guide).digest + changed = CategoricalProposal(prior, (0., 0., 0.)) + assert changed.digest != guide.digest + assert changed.prior == guide.prior + assert changed.probabilities == pytest.approx(prior) + for unit in [math.nan, math.inf, -.1, 1.1]: + with pytest.raises(ValueError): + guide.transform(unit) + + +@pytest.mark.parametrize("prior,log_guide,mass", [((), (), .25), + ((1., ), (), .25), + ((0., 1.), (0., 0.), .25), + ((.2, .3), (0., 0.), .25), + ((math.nan, ), (0., ), .25), + ((1., ), (-math.inf, ), .25), + ((1., ), (math.inf, ), .25), + ((1., ), (math.nan, ), .25), + ((1., ), (0., ), 0.), + ((1., ), (0., ), 1.), + ((1., ), (0., ), math.nan)]) +def test_invalid_cases(prior: tuple, log_guide: tuple, mass: float) -> None: + """Invalid guides fail explicitly instead of discarding prior cases.""" + with pytest.raises(ValueError): + CategoricalProposal(prior, log_guide, mass) + + +def test_unrepresentable_interval_is_not_silently_dropped() -> None: + """Positive prior cases require representable sampling intervals.""" + with pytest.raises(ConditioningNumericalError): + CategoricalProposal((1., 1e-100), (0., 0.)) From a414f8b017dbb31e0c7992b396f5f72b3c88b16e Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 20:39:53 -0400 Subject: [PATCH 71/94] Record validated Bridge orientation support and remaining inference gates --- docs/uncertainty/bridge-glue-support.md | 37 +++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 4 ++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/uncertainty/bridge-glue-support.md b/docs/uncertainty/bridge-glue-support.md index 5aa072c67..7e38d74a0 100644 --- a/docs/uncertainty/bridge-glue-support.md +++ b/docs/uncertainty/bridge-glue-support.md @@ -126,5 +126,42 @@ It checks all 36 per-body density corrections and sums all 46,656 joint face com Moving-body coordinates remain unchanged and acquire no spurious resting-face factor. This validates a proposal boundary for the planned inference map; it does not supply a posterior population or satisfy the prediction gate. +## Complete conditional trajectories with corrected faces + +Pipeline `22703020` completed in 7:49, including 7,306 native actions, independent reach/joint verification and four complete output-accounting checks. +Every feasible diagnostic is run twice from a fresh native world, and the baseline reproduces the earlier supported artifact exactly. + +| Face intervention | Exact glue prefix supported | First unsupported action | Complete output log density | +| --- | ---: | ---: | ---: | +| Original sampled faces | 600 | None | -11,048,561.45 | +| Correct `leg1` only | 600 | None | -3,619,008.88 | +| Correct `span0` only | 86 | 87 | Zero complete likelihood | +| Correct `span1` only | 600 | None | -9,218,435.81 | +| Correct bottle only | 582 | 583 | Zero complete likelihood | +| All initial-observation-selected faces | 600 | None | 74,837.37 | +| All selected faces, dwell 30 | 582 | 583 | Zero complete likelihood | + +The combined face correction removes the dominant orientation discrepancy while preserving all 600 exact glue updates, with retained reach-event log probability -5.32478. +The complete composition, before the additional face-proposal correction, has log weight 109,642.31. +Its independent output-accounting discrepancy is at most 2.9104e-11. +The largest remaining negative output terms are the two span roll channels, so the improved score does not establish good prediction or calibrated uncertainty. +These are log densities at deterministic development points, not marginal evidence, posterior comparisons, solve rates or successful agent seeds. + +The face interventions cannot be assessed independently: changing the bottle or `span0` alone loses an exact event, whereas their combined correction with the other faces preserves the full prefix. +At the fixed radius, intersecting every compatible height interval on the corrected conditional path remains empty at action 87. +Thus this corrected scene does not yet justify removing the declared reach discrepancy and using a single deterministic height. + +The first complete reader attempt exposed cancellation for a roughly 5.55e-17 m interval between nearly equal reach thresholds. +Subtracting rounded Gaussian log-CDF values produced a log-mass error of about 0.094 for an incompatible interval. +The corrected evaluator subtracts the original boundaries before scaling and uses the narrow-interval density expansion when the standardized width times the midpoint magnitude is below 1e-5. +The independent quadrature reader likewise preserves the original interval width. +All seven native cases were rerun after that correction, with maximum independent interval log-mass discrepancy 5.3718e-11. +The earlier path/probability artifacts remain retained separately under the failed attempt. +The earlier scan path-resolution failure and an incorrect float32 assumption in its height-readout checker also remain recorded as setup/verification failures, not model or agent outcomes. + +Next, integrate the corrected face proposal into the complete parameter/scene target and validate an unconditional future generator for the explicit joint/reach laws. +Future reach perturbations must be sampled independently of future glue observations, with one posterior variance draw retained per joint throughout each future. +Only then should full posterior populations and their reserved-action forecasts be assessed against the incumbent. + Artifacts are in `logs/uncertainty_bridge_joint_glue_screen_20260913/`, `logs/uncertainty_bridge_local_scene_20260913/`, `logs/uncertainty_bridge_local_glue_screen_20260913/` and `logs/uncertainty_bridge_reach_discrepancy_20260913/`. Orientation diagnostics and the face proposal are in `logs/uncertainty_bridge_orientation_support_20260913/` and `logs/uncertainty_bridge_face_proposal_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 406f94305..4c6b62d65 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -30,7 +30,9 @@ The continuous-output composition and retained exact-rate factor now pass reader Three initial resting-face choices dominate that error, motivating a native orientation-support diagnostic before fitting. The initial-face scan and independent reader now cover all 36 body/face cases. The new prior-preserving categorical proposal passes sixteen functional tests and focused static/format checks; its Bridge adapter recovers seven native roots and verifies all 46,656 joint face weights. -Complete conditional-trajectory and output checks for these face changes are running as `22703020` after correcting a narrow-interval probability calculation exposed by the independent reader. +Complete conditional-trajectory and output checks for these face changes finished as `22703020`, with 7,306 native actions and independent probability/output checks. +The combined correction preserves all 600 exact glue updates and removes the dominant orientation penalty; individual face changes can still lose exact support. +An independently detected narrow-interval probability error is corrected and the failed artifacts are retained separately. The complete inference map, adequate exploration and future-prediction assessment are still required before a usable Bridge posterior can be claimed. The [Bridge glue attribution](bridge-glue-attribution.md) now proves a structural contradiction in the transferred parameterized program, beyond the original no-op control. From d486a95ea40e67b5b83821c548eefea09e4864d9 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 21:47:25 -0400 Subject: [PATCH 72/94] Add budgeted joint-support initialization for offline inference --- docs/uncertainty/bridge-causal-futures.md | 47 +++++ docs/uncertainty/bridge-joint-inference.md | 94 ++++++++++ docs/uncertainty/implementation-progress.md | 15 +- docs/uncertainty/support-initialization.md | 55 ++++++ .../code_sim_learning/inference_sampling.py | 43 ++++- .../test_inference_support_initialization.py | 170 ++++++++++++++++++ 6 files changed, 420 insertions(+), 4 deletions(-) create mode 100644 docs/uncertainty/bridge-causal-futures.md create mode 100644 docs/uncertainty/bridge-joint-inference.md create mode 100644 docs/uncertainty/support-initialization.md create mode 100644 tests/code_sim_learning/test_inference_support_initialization.py diff --git a/docs/uncertainty/bridge-causal-futures.md b/docs/uncertainty/bridge-causal-futures.md new file mode 100644 index 000000000..7c6d8957f --- /dev/null +++ b/docs/uncertainty/bridge-causal-futures.md @@ -0,0 +1,47 @@ +# Bridge causal futures from a supported prefix + +September 13, 2026. +This is an offline integration check for the [uncertainty simplification plan](simplification-proposal.md). +It uses the combined initial-face correction from the [Bridge support diagnostic](bridge-glue-support.md), with parameters chosen from the 600-action development prefix. +It is not a posterior population, an agent experiment or evidence of predictive adequacy. + +## Conditional prefix and unconditional continuation + +The driver reproduces the entire verified 600-action conditional prefix in a fresh native world, including initial state, observations, joint factors, reach-event masses and learned memory. +It then continues the same world through all 586 reserved actions. +The conditioning context contains exactly 601 observations, including the initial frame; accessing observation 601 raises an index error. +Future observed joint positions and glue readings are never supplied to generation. +The recorded future actions are fixed inputs to this offline forecast. + +For each of the nine robot joints, the driver draws one variance from its inverse-gamma posterior at the prefix boundary and retains that variance for the whole continuation. +After each future native step, it adds an independent zero-mean Gaussian position increment with that joint's sampled variance, retaining the native joint velocity. +Before each future learned glue update, it draws an independent Gaussian reach offset with standard deviation 0.005 m. +The unchanged learned update uses that offset without conditioning on any future glue reading. +The prefix's interval-conditioning algorithm is not used to select future reach offsets. + +The existing output model generates complete future observations from those causal physical predictions. +It conditions correlated scalar output errors only on the fitting prefix, preserves coupled robot-orientation errors and derives the exact finger readout from its generated source joint. +Variance, joint-increment, reach and output sampling use separate random streams derived from the numerical seed. +The output-model identity matches the independently checked prefix composition exactly. + +## Completed checks + +Native job `22703457` completed in 2:56. +Numerical seeds 710 and 711 each generate a distinct complete continuation, and both complete histories repeat exactly from fresh worlds. +The four histories total 4,744 native actions. +These numerical seeds are future draws at one fixed candidate, not solve-rate seeds. + +Independent reader `22703621` completed in 1:07. +For each continuation, it checks all 5,274 joint increments against the retained variance draws, all 586 reach draws and every literal glue update and learned-memory transition. +It also verifies that generated joint readouts match the corrected native states and that the output sampler covers all declared fields. +The previously validated output sampler round-trips both saved observation histories exactly; this part is an integration check rather than a new independent derivation of its statistical law. +Changed conditioning length, variance, reach offset, model memory and exact output values are all rejected. + +## Remaining work + +This supplies the causal generation component required by a future Bridge posterior forecast. +It does not supply a future-density estimator, a fitted population or a held-out comparison verdict. +Complete the [joint inference target](bridge-joint-inference.md), assess its numerical adequacy and evaluate population forecasts against the incumbent. +Keep the current agent estimator unchanged until the plan's prediction and live-validation gates pass. + +Artifacts are in `logs/uncertainty_bridge_future_20260913/`. diff --git a/docs/uncertainty/bridge-joint-inference.md b/docs/uncertainty/bridge-joint-inference.md new file mode 100644 index 000000000..16bd8fbe0 --- /dev/null +++ b/docs/uncertainty/bridge-joint-inference.md @@ -0,0 +1,94 @@ +# Bridge joint inference target + +September 13, 2026. +This implements a complete augmented parameter/initial-scene map for the revised Bridge program and its explicitly declared joint/reach discrepancy model. +It builds on the [original scene law](bridge-initial-scene.md), [conditional glue support](bridge-glue-support.md) and [causal future generator](bridge-causal-futures.md). +The acting agent still uses the incumbent estimator. +This is a development comparison with a changed discrepancy model, not an estimator-only ablation or a production replacement. + +## Original prior and conditioning + +The original six-parameter prior uses the public program's declared bounds. +`glue_latch`, `drip_r`, `drip_h`, `glue_rate` and `bond_dist` have independent uniform priors on their declared continuous intervals. +`bond_dwell` has a discrete uniform prior on the integers 1 through 120, inclusive. +The initial scene keeps the original rest/moving cases, six equiprobable resting faces, pose and velocity laws, robot-joint law, fixture support and unheld/unbonded reset-memory contract. +The whole-scene geometry normalizer is common and parameter-independent for this reset model; its unknown value prevents absolute-evidence claims. + +The first exact partial glue increment from zero fixes `glue_rate = 0.2`, retaining its original density `1 / 0.95` once. +A consecutive `0.2, 0.4, 1` progression requires `0.4 < glue_latch <= 0.2 + 0.2 + 0.2` under the literal floating-point update. +The latch coordinate is sampled uniformly within that interval, retaining its mass under the original uniform `(0.4, 1)` prior. +Subsequent exact glue outputs still require compatible causal transitions; parameter conditioning does not remove those checks or their probabilities. + +## Complete coordinate map + +| Proposal coordinates | Meaning | +| --- | --- | +| 0 through 139 | Existing scene coordinates, with the checked prior-preserving resting-face proposal. | +| 140 | Latch value within the supported interval. | +| 141 through 143 | Radius, reach height and bond distance under a defensive Gaussian-box proposal. | +| 144 | The continuous-parameter proposal's mixture coordinate. | +| 145 | Discrete bond dwell under a defensive categorical proposal. | + +The continuous proposal retains a 25% original-prior component and has standard deviation 0.01 in each physical parameter coordinate. +The dwell guide centers on the training-prefix support value 25 with scale 5, while retaining a 25% uniform component over all 120 original cases. +Every proposal supplies its original-prior/proposal correction. +Guide likelihoods and centers do not replace the original prior or remove observation factors. + +The joint result carries six physical parameters and 140 scene auxiliary coordinates for the pinned original scene decoder. +Those auxiliary coordinates must be reconstructed with the identified decoder and data; they are not standalone Cartesian state coordinates. +Original prior assumptions, conditioning/guidance, data, output/discrepancy model, program and runtime receive separate identities. +Only the 600 fitting actions and their 601 observations enter the inference data identity. + +## Target factors + +The base factor contains the original root correction, resting-face and parameter proposal corrections, exact-rate/latch conditioning factors and complete initial output likelihood. +The remaining likelihood contains all 600 conditional joint-transition factors, normalized exact glue-event masses and remaining output factors. +The output law retains correlated scalar errors, coupled robot-orientation errors, the checked finger readout, original noisy channels and all other exact constraints. +Joint variances are integrated across the entire prefix; reach offsets are marginalized only when compatible intervals have identical complete next memory and commands. + +Forbidden initial geometry has zero base and target support. +A later incompatible exact glue event retains a finite initial base where appropriate but has zero complete likelihood. +Setup failures and unsupported numerical operations raise errors; they are not silently converted into model likelihoods. + +## Native preflight and next gate + +Preflight `22704066` completed in 6:56 with 8,632 native actions. +The existing supported point is recovered exactly through the new coordinate map and agrees with its independently checked complete composition after adding the new proposal and latch factors. +Six nearby cases change radius, height, bond distance, latch, span position or bottle position; all have finite complete targets and repeat exactly. +Four broad draws are retained: two fail geometry and two contradict the first deposition at action 58. +The preflight also checks 32 independent parameter-proposal and conditional-factor calculations. +An earlier attempt failed before native replay because its frozen compute overlay omitted the target-evaluation module; the corrected allocation retains the same calculations and candidates. + +Independent reader `22704257` completed in 2:21, checking parameter corrections, literal conditional updates, joint factors and complete output sums for all eleven cases. +Its maximum complete-target factor discrepancy is 2.9104e-11. + +## Initial-population support + +Screen `22704578` evaluated 32 unit-uniform candidates for each of numerical seeds 810 and 811, before any resampling or rejuvenation. +It completed in 56 seconds with 1,865 native actions. +The populations respectively contain nine and twelve geometrically feasible candidates, but neither contains a finite complete target. +Most feasible candidates contradict the first deposition at action 58; later failures occur at actions 87, 122 and 200. +The existing proposal is therefore insufficient to initialize either full sampler, despite the independently verified support points. + +A separate proposal experiment preserves the same target and mixes a 25% full unit-uniform component with a normalized local product guide around the supported training-prefix point. +The local component selects its rest, face, mixture and dwell cases through restricted uniform intervals and guides 27 active continuous unit coordinates with standard deviation 0.02. +All other coordinates retain their uniform distributions. +These restrictions apply only to the local proposal; the broad component retains the entire original proposal support. +The additional factor is the negative log of the complete mixture density, not the density of the selected component. +Independent one-dimensional integration checks all 27 Gaussian normalizers, and both proposal branches are checked against separate density calculations. + +Local screen `22704778` completed in 3:12 with 14,100 native actions. +It finds three finite candidates in seed 810 and twelve in seed 811, out of 32 draws each. +However, their effective sample sizes at the original first temperature, `1/32`, are both approximately one. +On the finite-support subsets, base-only effective sample sizes are approximately 1.69 and 1.04, so choosing a smaller first temperature alone cannot resolve initialization concentration. +Independent proposal-inverse, weighting and serial/parallel native verification completed as `22705833`, including 1,200 native actions and exact repeated target calculations. +The maximum inverse-coordinate discrepancy is 1.111e-15. +Its earlier attempt compared tuple-valued in-memory metadata directly with JSON lists; the corrected reader canonicalizes serialization before comparing physical results. + +These are initialization screens, not posterior fits or agent experiments. +The next numerical work must improve supported-population initialization and weight balance before a full fitting run can be useful. +Any change must retain joint original-prior corrections and count rejected evaluations; conditioning proposals separately for each fixed parameter without the corresponding parameter-dependent acceptance factors would change the target. +Full sampling remains separate from numerical and predictive acceptance. + +Artifacts are in `logs/uncertainty_bridge_joint_inference_20260913/`. +The corrected local proposal and its diagnostics are in `logs/uncertainty_bridge_local_proposal_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 4c6b62d65..e8a2e42a5 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,9 +12,22 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. +It preserves the checked 600-action prefix and generates the 586-action suffix without future observations, retaining one sampled variance per joint throughout each continuation. +The [complete Bridge joint target](bridge-joint-inference.md) now composes the original parameter/scene priors, exact-rate/latch factors and corrected scene/parameter proposals. +Its eleven-case native preflight completed with seven finite targets, two geometry rejections and two exact-event rejections; all repeated calculations are exact. +Independent target verification passes within 2.9104e-11, but both first 32-candidate populations have zero complete support. +A target-preserving local mixture then finds three and twelve finite candidates; both populations still concentrate to effective sample size approximately one at the first temperature. +Base-weight concentration is already severe, so improved initialization is required in addition to any temperature-schedule change. +The optional [complete-support initializer](support-initialization.md) now redraws entire joint candidates with rejected evaluations charged to the same budget. +All 35 focused functional tests and 32 exact default-path comparisons pass, together with type, lint and pinned format checks; two native initialization fixtures and independent readers are submitted. +The Bridge fixture separately tempers all finite reduced-target factors while keeping exact support hard and preserving the final target. +These close further integration components while leaving posterior exploration, future-density evaluation and predictive acceptance open. + The shared-variance Boil forecast adapter has now completed its compute validation as `22699944`, with 2,376 native actions, exact checkpoint recovery, independent weighted summaries and twelve corruption rejections. Both replacement Boil fits completed all 32 stages with 7,357 and 7,293 evaluations, but each retains one initial ancestor. -Full forecast/reader jobs `22700066_0` and `22700067_1` are now running; comparison `22700111` depends on both forecasts succeeding. +Full forecast generation has completed for both populations, producing 288 saved histories each. +Reader jobs within `22700066_0` and `22700067_1` are still running; comparison `22700111` depends on both passing. These preserve the previous fitting and forecast seeds and compare the shared-variance law with the fixed-variance law and incumbent selected point. The [Bridge incumbent control](bridge-incumbent-control.md) has also completed its 600-action fit and independently verified 586-action forecast assessment. It records two suffix glue-reading mismatches, while other exact channels still disagree and the complete replacement target remains unfinished. diff --git a/docs/uncertainty/support-initialization.md b/docs/uncertainty/support-initialization.md new file mode 100644 index 000000000..7e4f058c1 --- /dev/null +++ b/docs/uncertainty/support-initialization.md @@ -0,0 +1,55 @@ +# Complete-support initialization for offline inference + +September 13, 2026. +The [Bridge joint-target screens](bridge-joint-inference.md) found zero complete-support candidates under the broad proposal and only three and twelve under the local mixture, with early weight concentration in both populations. +The offline sampler now has an optional `initialize_on_support` setting, defaulting to false. +The acting agent remains on the incumbent estimator. + +## Joint rejection and retained target + +Let `u` denote the complete proposal vector, `h(u)` its reduced unnormalized target density, and `S` its finite positive support. +Initialization draws complete vectors independently from the declared uniform proposal and rejects those outside `S` until the requested population is filled or the evaluation budget is exhausted. +The accepted proposal is the original proposal conditioned on `S`. +Its unknown acceptance probability is a single constant across the whole joint vector and cancels from normalized weights. +The existing target factors therefore remain valid for posterior sampling up to that common constant; this does not support absolute-evidence estimates. + +Rejecting only a scene at fixed parameters would instead introduce a parameter-dependent acceptance probability and generally change the target. +This implementation redraws the entire parameter, scene and auxiliary vector together. +Every evaluated rejection consumes the original budget, including partial final batches. +An incomplete initialization emits neither a checkpoint nor a usable posterior. +Impossible support is reported as budget exhaustion for this optional search, not as proof of model inconsistency. + +The default initialization path and its historical checkpoint signatures remain unchanged. +Supported initialization has a distinct checkpoint identity and validates complete finite support on recovery. + +## Separate Bridge tempering experiment + +The native Bridge fixture also changes the intermediate distributions, separately from the initialization method. +For each guided proposal, let `b` be its complete original base log factor, `l` its remaining conditional log factor and `c` its local-mixture log correction. +The endpoint remains `b + l + c`. +The experimental path has zero log base on complete support, negative infinity outside it, and tempered finite score `beta * (b + l + c)`. +At zero temperature its reference is the proposal conditioned on complete support; it is deliberately not the original physical prior. +At temperature one it recovers the same complete reduced target. +Exact constraints remain hard at every temperature, and no event is assigned artificial sensor noise. + +This differs from the earlier [guided tempering](guided-tempering.md) experiment, which moved only a proposal correction into the tempered factor. +It addresses concentration already present in Bridge's remaining base terms, without claiming that a balanced initial population guarantees later exploration. + +## Validation and pending physical evidence + +The analytical reference has density proportional to `x` on `0 < y < x < 1`, with an independent uninformed coordinate. +Joint rejection alone has means `E[x] = 2/3` and `E[y] = 1/3`; the final target has means `3/4` and `3/8`. +Tests exercise both the original finite-factor split and tempering all finite factors, verify moments of the uninformed coordinate, and check exact checkpoint recovery. +Additional cases exhaust the budget with possible or impossible support, using scalar and batched evaluation. + +The first compute check passed 35 functional tests and 32 exact default-path/checkpoint comparisons, then failed type checking in a deliberately invalid-input test. +That test's typing is corrected; replacement check `22706421` completed in 2:09 with all 35 functional tests, 32 exact default-path/checkpoint comparisons, type checking, lint and pinned format checks passing. +The dependent native allocations were cancelled before starting and produced no native results. + +Native fixture `22706430` is gated on the corrected checks and uses numerical seeds 810 and 811, 32 accepted candidates each and a maximum of 2,048 target evaluations per population. +It stops at the completed initialization checkpoint, retaining all rejected evaluations and reporting the weight concentration implied by several temperatures. +Independent readers `22706433` and `22706435` verify the rejection ledger, proposal inverses and mixture corrections, first-32-candidate parity with the original screen, and a fresh native target repetition per population. +These are submitted initialization checks, not posterior fits, predictive acceptance or agent results. + +The frozen native inputs and outputs are in `logs/uncertainty_bridge_supported_initialization_20260913/`. +The separate reader is in `logs/uncertainty_bridge_supported_verification_20260913/`. diff --git a/predicators/code_sim_learning/inference_sampling.py b/predicators/code_sim_learning/inference_sampling.py index 78f333e3e..155cff0b3 100644 --- a/predicators/code_sim_learning/inference_sampling.py +++ b/predicators/code_sim_learning/inference_sampling.py @@ -73,6 +73,14 @@ class SamplerConfig: refresh probability mixes in independent uniform proposals within the selected block. Zero preserves the original random-walk schedule. + + initialize_on_support redraws whole proposal vectors until every + initial particle has finite base and likelihood. This conditions the + proposal on the complete target support, with a common normalization + that cancels from normalized weights. It does not redraw state given + a fixed parameter or soften exact constraints. Every rejected target + evaluation consumes budget; an unfinished initialization publishes + neither a checkpoint nor posterior samples. """ particles: int = 512 temperatures: int = 32 @@ -83,8 +91,11 @@ class SamplerConfig: proposal_blocks: Tuple[Tuple[int, ...], ...] = () temperature_schedule: Tuple[float, ...] = () refresh_probability: float = 0. + initialize_on_support: bool = False def __post_init__(self) -> None: + if not isinstance(self.initialize_on_support, bool): + raise ValueError("Support initialization must be boolean") for value in (self.particles, self.temperatures, self.moves, self.max_evaluations): if not isinstance(value, int) or value <= 0: @@ -335,6 +346,8 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], if config.refresh_probability == 0.: # Keep default checkpoints compatible with the original schedule. del signature_config["refresh_probability"] + if not config.initialize_on_support: + del signature_config["initialize_on_support"] signature = content_digest( json.dumps( { @@ -395,6 +408,11 @@ def sample_batch(prior: Union[BoxPrior, ConditionedPrior], not math.isfinite(v) or v <= 0 or v > count + 1e-8 for v in ess_values): raise ValueError("Invalid checkpoint progress") + if config.initialize_on_support and ( + initial_finite != count + or not np.all(np.isfinite(base_weights)) + or not np.all(np.isfinite(likelihoods))): + raise ValueError("Invalid supported initialization checkpoint") completed = (config.temperature_schedule[completed_stage - 1] if config.temperature_schedule else completed_stage / config.temperatures) \ @@ -544,9 +562,28 @@ def result( try: if resume is None: - for i, trial in enumerate(evaluate_many(list(particles))): - likelihoods[i], base_weights[i], joints[i] = trial - initial_finite += int(math.isfinite(likelihoods[i])) + if config.initialize_on_support: + candidates = list(particles.copy()) + while initial_finite < count: + for candidate, trial in zip(candidates, + evaluate_many(candidates)): + likelihood, base, joint = trial + if math.isfinite(likelihood) and math.isfinite(base): + particles[initial_finite] = candidate + likelihoods[initial_finite] = likelihood + base_weights[initial_finite] = base + joints[initial_finite] = joint + initial_finite += 1 + if initial_finite < count: + candidates = list( + rng.uniform(lower, + upper, + size=(count - initial_finite, + len(proposal_prior.names)))) + else: + for i, trial in enumerate(evaluate_many(list(particles))): + likelihoods[i], base_weights[i], joints[i] = trial + initial_finite += int(math.isfinite(likelihoods[i])) if not initial_finite: # Finite initialization may simply have missed valid support. return result("no_particle_support") diff --git a/tests/code_sim_learning/test_inference_support_initialization.py b/tests/code_sim_learning/test_inference_support_initialization.py new file mode 100644 index 000000000..be56708de --- /dev/null +++ b/tests/code_sim_learning/test_inference_support_initialization.py @@ -0,0 +1,170 @@ +"""Joint support rejection preserves target mass and recovery semantics.""" +import math +from dataclasses import replace +from typing import List, Tuple, cast + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_checkpoint import \ + SamplerCheckpoint +from predicators.code_sim_learning.inference_data import InferenceIdentity, \ + content_digest +from predicators.code_sim_learning.inference_evaluation import BatchedTarget, \ + TargetEvaluation +from predicators.code_sim_learning.inference_sampling import BoxPrior, \ + ConditionedPrior, PriorPoint, SamplerConfig, sample_batch + + +def _setup() -> Tuple[ConditionedPrior, InferenceIdentity]: + digest = content_digest(b"triangular joint support reference") + prior = ConditionedPrior(("parameter", "state", "uninformed"), + digest, digest, + BoxPrior(("u", "v", "w"), ((0., 1.), ) * 3)) + return prior, InferenceIdentity(digest, digest, digest, prior.digest, + digest) + + +def _target(point: Tuple[float, ...], temper_all: bool) -> TargetEvaluation: + x, y, _ = point + if not 0 < y < x: + return TargetEvaluation(point, point, -math.inf, -math.inf) + return TargetEvaluation(point, point, 0. if temper_all else math.log(x), + math.log(x) if temper_all else 0.) + + +@pytest.mark.parametrize("seed", [18, 29]) +@pytest.mark.parametrize("temper_all", [False, True]) +def test_parameter_dependent_support(seed: int, temper_all: bool) -> None: + """For density x on 0 Tuple[TargetEvaluation, ...]: + return tuple(_target(point, temper_all) for point in points) + + config = SamplerConfig(particles=1800, + temperatures=8, + moves=2, + max_evaluations=40000, + initialize_on_support=True) + result = sample_batch(prior, + identity, + BatchedTarget(evaluate), + config, + seed, + checkpoint=saved.append) + assert result.status == "complete" + initial = saved[0].unpack() + assert initial["initial_finite"] == config.particles + assert config.particles < initial["evaluations"] < 3 * config.particles + np.testing.assert_allclose(np.mean(initial["particles"], axis=0), + [2 / 3, 1 / 3, .5], + atol=.025) + values = np.asarray(result.samples) + assert np.all((values[:, 1] > 0) & (values[:, 1] < values[:, 0])) + np.testing.assert_allclose(np.average(values, + axis=0, + weights=result.weights), + [.75, .375, .5], + atol=.025) + assert np.average(values[:, 0]**2, + weights=result.weights) == pytest.approx(.6, abs=.025) + assert np.average(values[:, 1]**2, + weights=result.weights) == pytest.approx(.2, abs=.025) + assert np.average((values[:, 2] - .5)**2, + weights=result.weights) == pytest.approx(1 / 12, abs=.01) + for boundary in (saved[0], saved[3]): + assert sample_batch(prior, + identity, + BatchedTarget(evaluate), + config, + seed, + resume=boundary) == result + + +@pytest.mark.parametrize("batched", [False, True]) +@pytest.mark.parametrize("possible", [False, True]) +def test_initialization_budget_is_not_a_posterior(batched: bool, + possible: bool) -> None: + """Count every rejection, without emitting partial initialization.""" + prior, identity = _setup() + saved: List[SamplerCheckpoint] = [] + calls = [] + + def target(point: Tuple[float, ...]) -> TargetEvaluation: + calls.append(point) + if not possible: + return TargetEvaluation(point, point, -math.inf, -math.inf) + return _target(point, True) + + def batch( + points: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple(target(point) for point in points) + + def condition(point: np.ndarray) -> PriorPoint: + row = target(tuple(point)) + return PriorPoint(row.joint, row.log_base) + + config = SamplerConfig(particles=20, + max_evaluations=13, + initialize_on_support=True) + if batched: + result = sample_batch(prior, + identity, + BatchedTarget(batch), + config, + 4, + checkpoint=saved.append) + else: + result = sample_batch(prior, + identity, + lambda p: math.log(p[0]), + config, + 4, + condition=condition, + checkpoint=saved.append) + assert result.status == "budget_exhausted" + assert result.evaluations == len(calls) == 13 + assert not saved and not result.samples and not result.weights + assert result.completed_temperature == 0 + expected = sum( + math.isfinite(_target(point, True).log_base) + for point in calls) if possible else 0 + assert result.initial_finite == expected + + +def test_support_checkpoint_contract() -> None: + """Initialization mode is part of checkpoint identity, with hard + support.""" + prior, identity = _setup() + config = SamplerConfig(particles=16, + temperatures=2, + moves=1, + initialize_on_support=True) + saved: List[SamplerCheckpoint] = [] + + def evaluate( + points: Tuple[Tuple[float, ...], + ...]) -> Tuple[TargetEvaluation, ...]: + return tuple(_target(point, True) for point in points) + + target = BatchedTarget(evaluate) + sample_batch(prior, identity, target, config, 9, checkpoint=saved.append) + with pytest.raises(ValueError, match="Checkpoint differs"): + sample_batch(prior, + identity, + target, + replace(config, initialize_on_support=False), + 9, + resume=saved[0]) + with pytest.raises(ValueError, match="must be boolean"): + replace(config, initialize_on_support=cast(bool, 1)) From 73de3f9d8d90ba9061c23f885371cf422e1d6048 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 22:05:22 -0400 Subject: [PATCH 73/94] Record Bridge inference validation and Boil forecast comparison --- docs/uncertainty/bridge-future-density.md | 45 +++++++++++++++++++ docs/uncertainty/bridge-joint-inference.md | 18 ++++++++ docs/uncertainty/implementation-progress.md | 13 ++++-- .../shared-variance-discrepancy.md | 31 +++++++++++-- docs/uncertainty/support-initialization.md | 11 +++-- 5 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 docs/uncertainty/bridge-future-density.md diff --git a/docs/uncertainty/bridge-future-density.md b/docs/uncertainty/bridge-future-density.md new file mode 100644 index 000000000..4238b59d0 --- /dev/null +++ b/docs/uncertainty/bridge-future-density.md @@ -0,0 +1,45 @@ +# Bridge conditional future-density validation + +September 13, 2026. +This extends the [causal future generator](bridge-causal-futures.md) with density evaluation under the same joint-motion, glue-transition and output laws. +It uses the same fixed training-prefix support point and retains all 600 fitting actions before evaluating the 586-action suffix. +This is a fixed-candidate development diagnostic, not a posterior or agent result. + +## Conditioning and factorization + +The generator never reads future observations. +The separate density evaluator conditions joint corrections on future joint readings and integrates the shared per-joint variance across the whole history. +Each future glue reading is conditioned through normalized Gaussian reach-interval masses; incompatible observations have zero density, and compatible intervals must give identical complete next memory and commands. +The remaining observation factors retain the output-error state conditioned on the fitting prefix. +Only suffix factors enter the reported conditional future score; fitting-prefix likelihood is not counted again. + +The density adapter changes the checked conditional replay's observation horizon and records its variance state at the fitting boundary. +The permitted source edits are enumerated and checked before replay. +Every case reconstructs the original prefix exactly, including geometry, all predicted features, learned memory, event factors and joint-variance state. + +## Native and independent checks + +Job `22706700` completed in 4:40 with 8,318 native actions. +Two generated suffixes, the original recorded suffix and an impossible exact glue reading are each repeated in fresh worlds. +Both generated cases reproduce the generator's full physical predictions, learned memory and pre-update observations exactly and have finite density. +The impossible glue reading is rejected at action 601. + +Reader `22706738` completed in 1:15. +It checks the full literal glue updates and memory, independently integrates reach probabilities, and compares the accumulated joint factors with a closed-form inverse-gamma integral conditioned on the prefix residuals. +It separately accumulates suffix scalar innovations, sensor factors and checked readouts; orientation uses its previously validated kernel with an independent rotation conversion. +The maximum individual joint-factor discrepancy is 1.367e-12 and maximum reach-interval discrepancy is 5.372e-11. + +| Future observations | Joint log density | Glue log probability | Remaining output log density | Total conditional log density | +| --- | ---: | ---: | ---: | ---: | +| Generated, numerical seed 710 | 34,157.56 | 0 | 80,760.62 | 114,918.18 | +| Generated, numerical seed 711 | 34,023.62 | 0 | 80,832.95 | 114,856.57 | +| Recorded suffix | 25,671.59 | 0 | -1,821,608.70 | -1,795,937.10 | +| Impossible glue at action 601 | 57.76 before rejection | Negative infinity | Not evaluated | Negative infinity | + +These log densities are tied to the declared coordinates and measurement units and are not probabilities or solve rates. +The fixed candidate assigns very low density to the recorded suffix, dominated by continuous output disagreement. +Exact glue compatibility therefore does not establish good predictions. +The paired full fits must be assessed through weighted posterior forecasts and decision-relevant errors before any production change. + +Source and native artifacts are in `logs/uncertainty_bridge_future_density_20260913/`. +The independent reader is in `logs/uncertainty_bridge_future_density_verification_20260913/`. diff --git a/docs/uncertainty/bridge-joint-inference.md b/docs/uncertainty/bridge-joint-inference.md index 16bd8fbe0..ebd69fac2 100644 --- a/docs/uncertainty/bridge-joint-inference.md +++ b/docs/uncertainty/bridge-joint-inference.md @@ -92,3 +92,21 @@ Full sampling remains separate from numerical and predictive acceptance. Artifacts are in `logs/uncertainty_bridge_joint_inference_20260913/`. The corrected local proposal and its diagnostics are in `logs/uncertainty_bridge_local_proposal_20260913/`. + +## Complete-support initialization and paired fits + +The [whole-joint support initializer](support-initialization.md) now supplies 32 accepted candidates for each numerical seed, and both independent native readers pass. +The original complete target is retained while all finite reduced factors are introduced through a geometric temperature schedule. +Array `22706932` runs the two matched fitting pilots with 64 temperatures from 0.000001 through one, eight moves per stage, proposal scale 0.05 and a 10% within-block independent refresh probability. +The eleven blocks separate two fixtures, the robot, six movable bodies, parameters and the outer mixture coordinate. +Each run has a 20,000-evaluation budget and a 16-CPU, 64-GB, eight-hour allocation on `node1412` in `mit_preemptable`. +Every stage writes a recoverable checkpoint and retains actual native-action cost alongside logical target-evaluation cost. + +The verified initialization evaluations are cached with exact proposal keys and original artifact checksums. +They remain charged as logical evaluations; the stage-zero checkpoint must exactly reproduce the checked initialization's particles, factors, weights, ancestry and random-generator state before any new native evaluations begin. +This saves repeated physical reconstruction without reusing an old posterior as a new prior. +Subsequent proposals use fresh native worlds, and all finite target factors and rejected evaluations are logged. +Independent readers `22707128` and `22707130` are queued behind the fits to replay complete numerical ledgers and freshly evaluate every final joint target. +Completion, numerical diversity and held-out predictions remain separate requirements. + +Fitting artifacts are in `logs/uncertainty_bridge_supported_fits_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index e8a2e42a5..ce8eb6a37 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -20,14 +20,21 @@ Independent target verification passes within 2.9104e-11, but both first 32-cand A target-preserving local mixture then finds three and twelve finite candidates; both populations still concentrate to effective sample size approximately one at the first temperature. Base-weight concentration is already severe, so improved initialization is required in addition to any temperature-schedule change. The optional [complete-support initializer](support-initialization.md) now redraws entire joint candidates with rejected evaluations charged to the same budget. -All 35 focused functional tests and 32 exact default-path comparisons pass, together with type, lint and pinned format checks; two native initialization fixtures and independent readers are submitted. +All 35 focused functional tests and 32 exact default-path comparisons pass, together with type, lint and pinned format checks. +Both native fixtures and independent readers now pass: 32 supported candidates are collected in 209 and 123 evaluations, with effective sample sizes 30.06 and 31.03 at temperature 0.0001. +Paired full fitting pilots `22706932` are running with 64 temperatures, blocked rejuvenation and a 20,000-evaluation budget per run. The Bridge fixture separately tempers all finite reduced-target factors while keeping exact support hard and preserving the final target. -These close further integration components while leaving posterior exploration, future-density evaluation and predictive acceptance open. +The [conditional future-density audit](bridge-future-density.md) also passes, with 8,318 native actions and independent variance, reach and output-factor checks. +The generated histories reproduce exactly, but the fixed starting candidate gives very low likelihood to the recorded suffix, dominated by continuous output disagreement. +These close further integration components while leaving posterior exploration and predictive acceptance open. The shared-variance Boil forecast adapter has now completed its compute validation as `22699944`, with 2,376 native actions, exact checkpoint recovery, independent weighted summaries and twelve corruption rejections. Both replacement Boil fits completed all 32 stages with 7,357 and 7,293 evaluations, but each retains one initial ancestor. Full forecast generation has completed for both populations, producing 288 saved histories each. -Reader jobs within `22700066_0` and `22700067_1` are still running; comparison `22700111` depends on both passing. +Reader jobs within `22700066_0` and `22700067_1` completed successfully in 1:33:57 and 1:34:04; comparison `22700111` has also completed. +The [verified Boil comparison](shared-variance-discrepancy.md#completed-boil-forecast-comparison) shows improved pose and bubbling predictions relative to fixed variance, but continued regression against the incumbent. +Shared-variance replicas predict final goal probabilities 0.649 and 0.189 for the observed successful suffix, while the incumbent predicts one; these are forecast probabilities, not agent solve rates. +Replica disagreement and one-ancestor populations keep the replacement gate open. These preserve the previous fitting and forecast seeds and compare the shared-variance law with the fixed-variance law and incumbent selected point. The [Bridge incumbent control](bridge-incumbent-control.md) has also completed its 600-action fit and independently verified 586-action forecast assessment. It records two suffix glue-reading mismatches, while other exact channels still disagree and the complete replacement target remains unfinished. diff --git a/docs/uncertainty/shared-variance-discrepancy.md b/docs/uncertainty/shared-variance-discrepancy.md index 0449f1502..78b8e2dae 100644 --- a/docs/uncertainty/shared-variance-discrepancy.md +++ b/docs/uncertainty/shared-variance-discrepancy.md @@ -71,7 +71,8 @@ Repeated fresh replay and complete target evaluation are exact. The resulting conditional root-mean-square finger scales range from approximately 0.127 to 0.288 mm on these selected candidates, below the former fixed 1 mm. Those candidates were selected before this diagnostic; they are not samples from a newly fitted posterior. -Fresh matched 32-particle fits `22699492_0` and `_1` are running on compute nodes. +Fresh matched 32-particle fits `22699492_0` and `_1` completed all 32 temperatures, using 7,357 and 7,293 evaluations respectively. +Both retain one initial ancestor, so their completion does not establish adequate exploration. Their complete model identity includes the numerical variance prior as well as the transition law. Both retain the original fitting-data and physical-prior definitions, while recording the changed discrepancy model separately. They remain unassessed until independent replicas, held-out predictions and computational cost are checked. @@ -90,12 +91,12 @@ Twelve deliberate corruption cases are rejected: missing and duplicate histories The complete forecast plan reconstructs the changed statistical and runtime identities from frozen inputs and checks the unchanged data, program, physical prior, scene map and sampler settings against the previous fits. It records expected final report and checkpoint locations without hashing a mutable running report as though it were complete. -Forecast jobs `22700066_0` and `22700067_1` are queued behind their respective fit jobs. +Forecast jobs `22700066_0` and `22700067_1` completed generation and independent verification in 1:33:57 and 1:34:04 respectively. Each retains every positive-weight particle and generates two banks of four 132-action futures, plus a separate conditional-density trajectory for the recorded future. The same numerical and forecast seeds are retained from the fixed-variance comparison. An independent reader reconstructs all weighted summaries from complete saved histories before the forecast job can succeed. -Dependent comparison `22700111` will assess differences between replicas, the previous fixed-variance populations and the incumbent selected-point forecast. +Comparison `22700111` completed the replica, fixed-variance and incumbent comparisons, including checked source identities and independent metric reconstruction. Independent headline metric reconstruction already matches both previous verified forecast reports. The comparison retains numerical collapse and simulation cost alongside prediction errors; a completed forecast alone is not numerical acceptance. The fixed/shared-variance comparison changes the discrepancy law, while the incumbent comparison additionally changes initial-state treatment and output discrepancy. @@ -103,3 +104,27 @@ Neither comparison is a new live-agent seed or establishes solve-rate non-regres Reducing perturbation scale alone does not resolve the remaining heating errors or establish agent non-regression. Frozen component checks are in `logs/uncertainty_shared_variance_20260913/`, fitting/preflight artifacts in `logs/uncertainty_boil_shared_variance_20260913/`, and future checks in `logs/uncertainty_boil_shared_variance_forecast_20260913/`. + +## Completed Boil forecast comparison + +The comparison uses one development recording, with a 132-action fitting prefix and a 132-action reserved suffix. +The table reports individual numerical replicas, not agent seeds or solve rates. +Pose and bubbling errors compare native predictive means with clean held-out measurements. +The final goal probability refers to the recorded suffix's final event, which is successful in the observed trajectory. + +| Method | Numerical seed | Jug x RMSE (m) | Bubbling-level RMSE | Final goal probability | +| --- | ---: | ---: | ---: | ---: | +| Incumbent selected point | Not applicable | 0.00170 | 0.01915 | 1.000 | +| Fixed joint variance | 410 | 0.12922 | 0.48328 | 0.223 | +| Fixed joint variance | 411 | 0.12632 | 0.45393 | 0.306 | +| Shared learned joint variance | 410 | 0.00911 | 0.28022 | 0.649 | +| Shared learned joint variance | 411 | 0.05747 | 0.30312 | 0.189 | + +Learning the shared variance improves pose and bubbling prediction over the fixed-variance model for both numerical seeds, but neither matches the incumbent's bubbling prediction. +The second shared-variance run assigns only 0.377 probability to the burner being off at the final step, reducing its joint goal probability despite predicting boiling more often. +Its conditional-density evaluation also retains 23 zero-density particles, while the other shared-variance run retains none. +The substantial replica disagreement and one-ancestor populations remain numerical concerns. +These results support the shared-variance modeling change as a development direction, but do not pass the inference replacement gate. + +The next Boil work must distinguish poor exploration from remaining state/model inadequacy, preserving the current comparison as evidence rather than rerunning until favorable. +The complete verified metrics and cost accounting are in `logs/uncertainty_boil_shared_variance_forecast_20260913/model-comparison.json`. diff --git a/docs/uncertainty/support-initialization.md b/docs/uncertainty/support-initialization.md index 7e4f058c1..9d4d224cf 100644 --- a/docs/uncertainty/support-initialization.md +++ b/docs/uncertainty/support-initialization.md @@ -46,10 +46,15 @@ The first compute check passed 35 functional tests and 32 exact default-path/che That test's typing is corrected; replacement check `22706421` completed in 2:09 with all 35 functional tests, 32 exact default-path/checkpoint comparisons, type checking, lint and pinned format checks passing. The dependent native allocations were cancelled before starting and produced no native results. -Native fixture `22706430` is gated on the corrected checks and uses numerical seeds 810 and 811, 32 accepted candidates each and a maximum of 2,048 target evaluations per population. +Native fixture `22706430` completed for numerical seeds 810 and 811, collecting 32 accepted candidates each in 209 and 123 evaluations, respectively, within the 2,048-evaluation limit. +The allocations took 9:12 and 8:12 and executed 35,332 and 28,389 native actions. It stops at the completed initialization checkpoint, retaining all rejected evaluations and reporting the weight concentration implied by several temperatures. -Independent readers `22706433` and `22706435` verify the rejection ledger, proposal inverses and mixture corrections, first-32-candidate parity with the original screen, and a fresh native target repetition per population. -These are submitted initialization checks, not posterior fits, predictive acceptance or agent results. +Independent readers `22706433` and `22706435` both completed in 1:02, verifying the rejection ledger, proposal inverses and mixture corrections, first-32-candidate parity with the original screen, and a fresh native target repetition per population. +Both readers reject all four corrupted-ledger controls, with maximum inverse-coordinate discrepancy 1.222e-15. +At temperature 0.0001, effective sample sizes are 30.06 and 31.03; at 0.001 they fall to 8.37 and 6.91, and direct weighting to temperature one concentrates on one candidate in each population. +The artificial starting distributions now contain supported, balanced populations, but gradual weighting and rejuvenation are still necessary. +These are completed initialization checks, not posterior fits, predictive acceptance or agent results. +Paired full fitting pilots `22706932` are now submitted with the unchanged complete target, 64 geometrically spaced temperatures, eight blocked moves per stage and a 20,000-evaluation budget per run. The frozen native inputs and outputs are in `logs/uncertainty_bridge_supported_initialization_20260913/`. The separate reader is in `logs/uncertainty_bridge_supported_verification_20260913/`. From b962d8225acfdae6f340ffef5de874d5b5a2649a Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 22:21:43 -0400 Subject: [PATCH 74/94] Validate Bridge joint forecast adapter and reserved assessment --- .../uncertainty/bridge-posterior-forecasts.md | 46 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 49 insertions(+) create mode 100644 docs/uncertainty/bridge-posterior-forecasts.md diff --git a/docs/uncertainty/bridge-posterior-forecasts.md b/docs/uncertainty/bridge-posterior-forecasts.md new file mode 100644 index 000000000..20ef718ea --- /dev/null +++ b/docs/uncertainty/bridge-posterior-forecasts.md @@ -0,0 +1,46 @@ +# Bridge forecasts from complete joint candidates + +September 13, 2026. +This prepares the running [Bridge joint fits](bridge-joint-inference.md) for held-out prediction checks. +The production estimator remains unchanged, and the full posterior forecast comparison is not complete. + +## Candidate preparation and causal continuation + +The adapter accepts the complete 147-coordinate guided proposal, preserving its six physical parameters and 140 original scene auxiliary coordinates through the checked map. +It reevaluates the candidate's original fitting target and checks its complete joint coordinates and score against the supplied fitting record. +A candidate with unsupported fitting observations is rejected rather than removed from a weighted population and renormalized away. + +The generalized generator retains the checked model and numerical operation sequence, replacing only the earlier single-witness assertion with the candidate's own complete fitted prefix. +It reconstructs all 600 fitting actions and requires exact agreement in the root, predictions, learned memory, joint-variance statistics and conditional event factors before continuing. +Future joint variances, step residuals, reach offsets and output errors use the same causal laws as the [original generator](bridge-causal-futures.md). +Future observations are available only to the separate [density evaluator](bridge-future-density.md). +The density evaluator must reconstruct that same fitting prefix and scores only the future factors. + +Fixture `22707346` applies generation and density evaluation to two different supported candidates from the checked initial populations. +Each candidate is prepared from its original complete target, then its 586-action generated future and corresponding density replay are each repeated in fresh worlds. +Both candidates pass exact prefix, target, generated-history and density checks. +The fixture completed 10,688 native actions in 7:44 of measured script time. +Independent reader `22707374` completed in 1:45, verifying shared-variance draws, joint and reach factors, literal memory updates, output sampling and future-density accounting for both candidates. +These selected candidates are test inputs, not an assessed posterior population. + +## Reserved assessment inputs + +Preparation `22707523` completed the 586-action clean and noisy assessment suffixes, with all data kept outside fitting and generation. +It reproduces the incumbent's fifteen block-position RMSE entries against the earlier verified assessment. +The public `Bridged` geometry predicate is evaluated on complete reconstructed public poses. +It is distinct from hidden attachment state; glue occupancy and consumption are observed-feature events rather than labels for actual welds. + +The clean recording satisfies `Bridged` only at action 1186, the final reserved step. +The noisy observations do not satisfy it at any reserved step. +Task-outcome predictions must therefore be assessed against the clean reference, while noisy observation errors are reported separately. +An earlier preparation attempt rejected equivalent tuple/list feature-key representations; the corrected comparison normalizes the key containers without altering measurements. + +## Remaining population comparison + +After the adapter reader passes, the complete-population driver must retain every positive-weight fitted candidate, its original weight and one complete joint state throughout each generated future. +Repeated future draws should be separated into independent banks so forecast sampling variability can be distinguished from differences between the two fitted populations. +The recorded-future density must retain zero-support contributions and combine candidates using their original weights. +Final assessment must include block-position errors, glue-state events, geometric goal probabilities, numerical replica differences and actual inference/forecast cost, alongside the existing incumbent comparison. +Neither completed simulation nor a finite future density is sufficient for predictive acceptance. + +Adapter artifacts are in `logs/uncertainty_bridge_forecast_adapter_20260913/`, its reader in `logs/uncertainty_bridge_forecast_adapter_verification_20260913/`, and reserved assessment inputs in `logs/uncertainty_bridge_forecast_assessment_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index ce8eb6a37..6ed018d7a 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -23,6 +23,9 @@ The optional [complete-support initializer](support-initialization.md) now redra All 35 focused functional tests and 32 exact default-path comparisons pass, together with type, lint and pinned format checks. Both native fixtures and independent readers now pass: 32 supported candidates are collected in 209 and 123 evaluations, with effective sample sizes 30.06 and 31.03 at temperature 0.0001. Paired full fitting pilots `22706932` are running with 64 temperatures, blocked rejuvenation and a 20,000-evaluation budget per run. +Both have completed their first two temperature steps with all 32 initial lineages retained; these very early diagnostics do not establish adequate exploration. +The [generalized forecast adapter](bridge-posterior-forecasts.md) passes native checks on two complete joint candidates, with 10,688 actions and exact generated/density histories; the independent reader also passes for both candidates. +The reserved assessment is prepared: the clean geometric bridge goal is true only at the final step, while direct evaluation of noisy observations misses it throughout the suffix. The Bridge fixture separately tempers all finite reduced-target factors while keeping exact support hard and preserving the final target. The [conditional future-density audit](bridge-future-density.md) also passes, with 8,318 native actions and independent variance, reach and output-factor checks. The generated histories reproduce exactly, but the fixed starting candidate gives very low likelihood to the recorded suffix, dominated by continuous output disagreement. From 7681ecfe1f712a6e53b61803302e2a836ae69d30 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 22:41:43 -0400 Subject: [PATCH 75/94] Validate and queue weighted Bridge posterior forecasts --- .../uncertainty/bridge-posterior-forecasts.md | 36 +++++++++++++++---- docs/uncertainty/implementation-progress.md | 3 ++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/uncertainty/bridge-posterior-forecasts.md b/docs/uncertainty/bridge-posterior-forecasts.md index 20ef718ea..2bf45e51e 100644 --- a/docs/uncertainty/bridge-posterior-forecasts.md +++ b/docs/uncertainty/bridge-posterior-forecasts.md @@ -35,12 +35,36 @@ The noisy observations do not satisfy it at any reserved step. Task-outcome predictions must therefore be assessed against the clean reference, while noisy observation errors are reported separately. An earlier preparation attempt rejected equivalent tuple/list feature-key representations; the corrected comparison normalizes the key containers without altering measurements. -## Remaining population comparison +## Weighted population driver and checks -After the adapter reader passes, the complete-population driver must retain every positive-weight fitted candidate, its original weight and one complete joint state throughout each generated future. -Repeated future draws should be separated into independent banks so forecast sampling variability can be distinguished from differences between the two fitted populations. -The recorded-future density must retain zero-support contributions and combine candidates using their original weights. -Final assessment must include block-position errors, glue-state events, geometric goal probabilities, numerical replica differences and actual inference/forecast cost, alongside the existing incumbent comparison. -Neither completed simulation nor a finite future density is sufficient for predictive acceptance. +The complete-population driver retains every positive-weight fitted candidate with its original weight and complete joint state throughout each generated future. +It checks that the fit and checkpoint are complete, that their weights and samples agree, and that data, program and probability-model identities match the reserved assessment. +Its source guard rejects ten malformed or unfinished input cases before generating a future. +These forecasts assess raw completed numerical pilots; they do not bypass the production interface's requirement for an assessed posterior. + +Each positive-weight candidate receives two independent banks of four complete generated futures and one separate conditional-density evaluation of the recorded suffix. +Generated tasks receive no reserved observations. +All candidate preparation and future simulation steps are counted. +Zero-density contributions remain in the weighted density mixture, with no removal or renormalization of their source mass. +The summaries include block-position errors, glue predicate probabilities, geometric goal probabilities, forecast-bank differences and complete recorded-future density. + +A finite-mixture reference verifies unequal weights and a zero-density contribution; eight corruption cases reject missing, duplicated, relabeled or reweighted histories. +Native fixture `22707973` completed in 2:47 with 13,060 simulator actions, using two fixed support points with weights 0.25 and 0.75 and two draws per bank. +These chosen weights are test inputs, not posterior estimates. +Independent readers `22708064` and `22708157` completed in 3:01 and 3:02, checking every history's factors and weighted sums and repeating a generated and recorded-future path in fresh simulators. +The second reader additionally checks original checkpoint weights for full-population inputs, all reported glue metrics and a roundoff-tolerant final probability comparison. +Separate check `22708224` reconstructs the bank-difference metrics through an independent dense weighted calculation. + +## Queued full comparison + +Forecast jobs `22708283` and `22708284` are queued behind the corresponding full-fit verifiers `22707128` and `22707130`. +They will be followed by forecast readers `22708285` and `22708286`, including bank checks, then comparison `22708287` against the existing incumbent forecast. +Each forecast and reader has a 16-CPU, 64-GB, three-hour allocation on `node1412` in `mit_preemptable`. +The final comparison also reports fitting cost, forecast cost, lineage loss and differences between the numerical replicas. + +The full fitted-population forecasts are not yet complete. +Neither completed simulation nor finite future density is sufficient for predictive acceptance. Adapter artifacts are in `logs/uncertainty_bridge_forecast_adapter_20260913/`, its reader in `logs/uncertainty_bridge_forecast_adapter_verification_20260913/`, and reserved assessment inputs in `logs/uncertainty_bridge_forecast_assessment_20260913/`. + +The weighted driver is in `logs/uncertainty_bridge_population_forecast_20260913/`; full verification and comparison use `logs/uncertainty_bridge_population_forecast_verification_v2_20260913/`. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 6ed018d7a..ff79c05c4 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -26,6 +26,9 @@ Paired full fitting pilots `22706932` are running with 64 temperatures, blocked Both have completed their first two temperature steps with all 32 initial lineages retained; these very early diagnostics do not establish adequate exploration. The [generalized forecast adapter](bridge-posterior-forecasts.md) passes native checks on two complete joint candidates, with 10,688 actions and exact generated/density histories; the independent reader also passes for both candidates. The reserved assessment is prepared: the clean geometric bridge goal is true only at the final step, while direct evaluation of noisy observations misses it throughout the suffix. +The weighted population driver now passes a 13,060-action native fixture, independent per-history and mixture checks, ten source guards, eight malformed-history rejections and separate forecast-bank checks. +Full forecasts `22708283` and `22708284`, their readers `22708285` and `22708286`, and incumbent comparison `22708287` are queued behind the live fit verifiers. +No full-population prediction result or replacement acceptance is available yet. The Bridge fixture separately tempers all finite reduced-target factors while keeping exact support hard and preserving the final target. The [conditional future-density audit](bridge-future-density.md) also passes, with 8,318 native actions and independent variance, reach and output-factor checks. The generated histories reproduce exactly, but the fixed starting candidate gives very low likelihood to the recorded suffix, dominated by continuous output disagreement. From f355822a9c2d2c1367363a1354083fab1b483096 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 23:05:02 -0400 Subject: [PATCH 76/94] Reproduce Boil inference collapse and validate supported fitting pilots --- docs/uncertainty/boil-supported-inference.md | 84 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 10 +++ 2 files changed, 94 insertions(+) create mode 100644 docs/uncertainty/boil-supported-inference.md diff --git a/docs/uncertainty/boil-supported-inference.md b/docs/uncertainty/boil-supported-inference.md new file mode 100644 index 000000000..497871543 --- /dev/null +++ b/docs/uncertainty/boil-supported-inference.md @@ -0,0 +1,84 @@ +# Boil: supported initialization and finite-factor tempering + +This is an offline numerical follow-up to the [shared-variance forecast comparison](shared-variance-discrepancy.md). +The current agent and its production estimator remain unchanged. +The previous replacement populations gave unstable predictions and did not meet the incumbent comparison gate. + +## Reproduced failure + +The original shared-variance fits use 32 particles on a fixed 132-action training prefix. +Numerical seeds 410 and 411 retain only three and one initial lineages after their first temperature update, respectively. +They eventually both retain one initial lineage. +These are sampler seeds on one development recording, not agent solve-rate seeds. + +Compute jobs `22708815_0` and `22708815_1` reproduce the original initial parameter means exactly, the original finite-candidate counts, and the first effective sample sizes. +The target worker, initial-scene map, simulator program, sensor model and physical/parameter priors are the same frozen sources used by the completed shared-variance fits. + +| Numerical seed | Original finite candidates / 32 | Initial base-weight ESS | First-temperature ESS | Lineages after first temperature | +|---|---:|---:|---:|---:| +| 410 | 11 / 32 | 2.2643 | 1.8032 | 3 | +| 411 | 7 / 32 | 1.0741 | 1.0572 | 1 | + +The first temperature is `1 / 32768`. +The initial base-weight calculation precedes the remaining trajectory-likelihood update. +Consequently, simply inserting smaller trajectory-temperature increments cannot remove the concentration already imposed by the untempered base factors. +This establishes an early numerical problem; it does not explain every later prediction error. + +## Target-preserving alternative + +Use the existing [whole-joint support initializer](support-initialization.md) to draw complete 84-dimensional proposal vectors until 32 have finite complete targets. +Reject the entire vector, including parameters, when it has zero target density. +Every rejection consumes the evaluation budget. +Do not hold parameters fixed while repeatedly redrawing scenes, which would require a parameter-dependent acceptance correction. + +On finite support, set the sampler's base log weight to zero and temper the sum of the original base log weight and remaining log likelihood. +Keep exact zero-support cases rejected at every temperature. +The final unnormalized target is unchanged, up to the single common normalization introduced by whole-joint proposal rejection. +No absolute-evidence estimate is claimed. +The data, program, uncertainty law and original priors are unchanged; the intermediate sampling distributions change. + +The two initializations collected 32 supported candidates in 132 and 128 evaluations, respectively. +Each performed 14,652 native simulator actions. +Their original first 32 proposals and initial summaries match the earlier runs. + +The following comparison uses exactly the same 32 retained candidates in each column, without temperature moves or resampling: + +| Seed | Temperature | ESS with original factor placement | ESS with all finite factors tempered | +|---|---:|---:|---:| +| 410 | 0.000001 | 2.7189 | 31.7001 | +| 411 | 0.000001 | 1.9439 | 31.8150 | +| 410 | 1 / 32768 | 2.0224 | 18.6341 | +| 411 | 1 / 32768 | 1.5069 | 18.6098 | +| 410 | 1 | 1.0000 | 1.0000 | +| 411 | 1 | 1.0000 | 1.0000 | + +The last two rows matter: refactoring the tempering path alone does not make the original candidates cover the final target. +Rejuvenation must move them into the important regions while the likelihood is introduced. +A nearly uniform early population is not evidence of a usable posterior. + +## Verification and next experiment + +The diagnostic source and original failed checker are preserved in `logs/uncertainty_boil_support_diagnostic_20260913`. +The first checker jobs, `22708819` and `22708820`, failed before native verification because the checker requested `original_factorization_max_weight` while the report field is `original_max_weight`. +This is a checker setup failure, not a simulator or agent failure. +The corrected reader lives separately in `logs/uncertainty_boil_support_verification_v2_20260913` and reads the unchanged diagnostic outputs. +Reader jobs `22709035` and `22709037` check the entire rejection/RNG ledger, checkpoint weights and joint targets, scalar ESS references, six deliberate corruptions, and fresh native evaluation of all 32 retained candidates. +Both corrected readers have completed successfully, with 4,224 fresh native actions each and exact agreement for every retained target. + +The next fitting driver is frozen in `logs/uncertainty_boil_supported_fits_20260913`. +Two small fixtures, `22709043` and `22709044`, are gated on the corrected initialization readers. +They exercise cached initialization, complete fitting, checkpoint/result agreement, and subsequent numerical-ledger and native-target verification. +Their reader jobs are `22709063` and `22709064`. +Both fixtures and both readers have completed successfully. +The fixtures each perform 6,204 new native actions; each reader adds 4,224 fresh actions and exactly reproduces the complete proposal/target/checkpoint trace and all final joint targets. +The two-temperature fixtures deliberately jump to the final target and collapse to one lineage; they validate mechanics, not numerical adequacy. +The cached initialization must exactly match every saved particle, joint coordinate, factor, weight, ancestor, RNG state and evaluation count. +Cached evaluations remain charged to the fitting budget; the original initialization's native cost is reported separately. + +After those checks, the paired full comparison will keep the original 32 particles, eight moves per temperature, proposal blocks, proposal scale and refresh probability. +It uses 64 geometrically spaced temperatures from `0.000001` to one and a maximum of 20,000 target evaluations per run. +The runs use compute nodes on `mit_preemptable` with the previously audited native runtime pinned to `node1412`. +Full fits `22709091` and `22709092` are now running after the fixture readers passed, with their own readers `22709093` and `22709094` dependent afterward. +Each full fit has 16 CPUs, 64 GB memory and an eight-hour allocation. +Independent replicas, numerical-budget sensitivity and reserved-future forecasts remain acceptance requirements. +No replacement posterior or agent advantage is established by this initialization diagnostic. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index ff79c05c4..d8b2e85eb 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,16 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Boil supported-inference diagnostic](boil-supported-inference.md) now reproduces the original first-update concentration on both shared-variance seeds. +The original base weights already have effective sample sizes 2.264 and 1.074 before the remaining trajectory likelihood is introduced. +Whole-joint rejection collects 32 finite candidates in 132 and 128 evaluations; placing all finite factors under tempering preserves the final target and gives initial-temperature effective sample sizes 31.700 and 31.815 on those same retained populations. +Both corrected independent readers pass all retained native targets, rejection/RNG and checkpoint checks, scalar ESS references, and six corruption controls. +The initial diagnostic performs 29,304 native actions across both seeds, and fresh reader verification adds 8,448. +The original reader's report-field naming error is preserved separately; it is a checker failure rather than a model or agent outcome. +Both small fitting fixtures and their numerical/native readers have now passed, including exact complete proposal/target/checkpoint traces and every final native target. +Full paired fits `22709091` and `22709092` are running on the compute node, with full-fit readers `22709093` and `22709094` dependent afterward. +These compare the same original Boil target under supported initialization and a 64-stage finite-factor tempering path; stable final inference and reserved-future prediction improvement remain unproven. + The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. It preserves the checked 600-action prefix and generates the 586-action suffix without future observations, retaining one sampled variance per joint throughout each continuation. The [complete Bridge joint target](bridge-joint-inference.md) now composes the original parameter/scene priors, exact-rate/latch factors and corrected scene/parameter proposals. From 75fabb23272ae346157274e77f807089bfb16e2c Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Sun, 13 Sep 2026 23:28:01 -0400 Subject: [PATCH 77/94] Validate weighted forecasts for supported Boil inference --- docs/uncertainty/boil-supported-inference.md | 41 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 5 +++ 2 files changed, 46 insertions(+) diff --git a/docs/uncertainty/boil-supported-inference.md b/docs/uncertainty/boil-supported-inference.md index 497871543..b5d19944c 100644 --- a/docs/uncertainty/boil-supported-inference.md +++ b/docs/uncertainty/boil-supported-inference.md @@ -82,3 +82,44 @@ Full fits `22709091` and `22709092` are now running after the fixture readers pa Each full fit has 16 CPUs, 64 GB memory and an eight-hour allocation. Independent replicas, numerical-budget sensitivity and reserved-future forecasts remain acceptance requirements. No replacement posterior or agent advantage is established by this initialization diagnostic. + +## Weighted future comparison + +The forecast adapter is implemented separately in `logs/uncertainty_boil_supported_forecasts_20260913`. +It reuses the previously verified shared-variance physical future generator, output model and per-history probability checks. +The completed-fit adapter restores the saved checkpoint without evaluating the fitting target, requires agreement with the independently verified source, and preserves every positive fitted weight and complete joint candidate. +It checks that each replay's original prefix base factor plus remaining likelihood equals the combined prefix score stored by the new sampler. +The factors retain their original statistical meanings even though their placement in the sampling schedule changed. + +Each candidate remains fixed through its complete 132-action continuation. +Full forecasts use two banks of four generated futures per positive-weight candidate, plus a separate conditional-density evaluation against the recorded future. +Generated histories have no future-observation lookup. +Zero-likelihood future contributions remain in the original mixture without discarding or renormalizing their source weight. +The assessment, feature/event definitions and future random seeds match the previous shared-variance comparison. + +Guard job `22709834` passed ten completed-source corruption checks and eight malformed-history checks, plus an independent nonuniform-weight mean/variance/density reference. +The reference explicitly tests a zero-density component carrying weight 0.3 and a finite component carrying weight 0.7. +The native adapter fixture, `22709851`, uses all 32 positive-weight candidates from the completed two-temperature fitting fixture, producing 96 histories before its repeated checks. +It completed successfully with 29,172 native simulator actions, preserving the saved complete-prefix scores and repeating its selected generated and density histories exactly. +Its reader, `22709867`, checks every saved history, independently rebuilds the weighted means, variances, feature errors, event probabilities, mixture densities and bank differences, and repeats one complete generated history and one density history. +That reader has completed successfully: all 96 histories and 228,096 joint factors pass, the weighted summaries agree, and 3,828 additional native actions include both exact fresh complete histories. +This fixture tests the complete adapter; its source fit is deliberately inadequate and is not a posterior-accuracy result. + +The original validation node has all 64 allocated cores occupied by the four ongoing fits. +A read-only compute probe, `22709822`, confirmed that `node1411` has the same AMD EPYC 7542 CPU model as `node1412`. +The new adapter checks run on `node1411`, requiring exact saved prefix-density and fresh-history comparisons before its full forecast jobs can proceed. +Matching the saved density does not by itself establish portability of every physical trajectory across machines; the reader additionally checks native prefixes and fresh complete histories on the new node. +The initially pending guard and fixture jobs `22709723` and `22709730` were cancelled before starting to move only this validation work; none of the running fits was changed. + +| Work | Jobs | Gate | +|---|---|---| +| Full Boil forecasts | `22709874`, `22709875` | Adapter fixture reader and corresponding full-fit reader | +| Full forecast readers | `22709879`, `22709880` | Corresponding full forecast | +| Incumbent/model/numerical comparison | `22709887` | Both full forecast readers | + +The final comparison retains the earlier incumbent, fixed-variance and shared-variance rows, adds both new numerical replicas, and reports replica disagreement and fitting/forecast cost. +Initialization native actions are reported separately from the cached full fit and must be included when assessing total fitting cost. +The new versus previous shared-variance comparison preserves the probability model while changing initialization, tempering and numerical budget. +The fixed-variance and incumbent controls retain their separately labelled differences in discrepancy or initial-state treatment. +The full forecasts and comparison remain incomplete until the gated jobs finish and their reports are verified. +The adapter gate has passed; the full forecasts are currently waiting for the full fits and their readers. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index d8b2e85eb..5e2155e40 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -21,6 +21,11 @@ The original reader's report-field naming error is preserved separately; it is a Both small fitting fixtures and their numerical/native readers have now passed, including exact complete proposal/target/checkpoint traces and every final native target. Full paired fits `22709091` and `22709092` are running on the compute node, with full-fit readers `22709093` and `22709094` dependent afterward. These compare the same original Boil target under supported initialization and a 64-stage finite-factor tempering path; stable final inference and reserved-future prediction improvement remain unproven. +The [weighted Boil future adapter](boil-supported-inference.md#weighted-future-comparison) is implemented with exact completed-checkpoint recovery, unchanged positive weights and complete joint histories, and separate generation and future-density paths. +All ten source guards, eight malformed-history checks and an independent nonuniform-weight reference pass. +Its native fixture and independent reader have passed: 96 histories, 29,172 generation/check actions, 228,096 verified joint factors and 3,828 additional reader actions, including two exact fresh complete histories. +Full forecasts `22709874` and `22709875`, forecast readers `22709879` and `22709880`, and comparison `22709887` are queued behind the still-running fits and their readers. +The validation work uses a second matching-CPU compute node with exact prefix/replay checks; none of the four running fits is modified. The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. It preserves the checked 600-action prefix and generates the 586-action suffix without future observations, retaining one sampled variance per joint throughout each continuation. From 1212d1475d1169f33741dcacda16c6b3a9b174a9 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 00:07:25 -0400 Subject: [PATCH 78/94] Verify Fan speed ambiguity against complete forecast populations --- docs/uncertainty/fan-reflection-diagnostic.md | 80 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 11 +++ 2 files changed, 91 insertions(+) create mode 100644 docs/uncertainty/fan-reflection-diagnostic.md diff --git a/docs/uncertainty/fan-reflection-diagnostic.md b/docs/uncertainty/fan-reflection-diagnostic.md new file mode 100644 index 000000000..48397b8af --- /dev/null +++ b/docs/uncertainty/fan-reflection-diagnostic.md @@ -0,0 +1,80 @@ +# Fan speed ambiguity and reflection diagnostic + +This is Stage B development work under the [simplification proposal](simplification-proposal.md). +It investigates disagreement between the two completed [tempered Fan populations](guided-tempering.md), without changing the program, prior, data or production estimator. +All results here are offline inference diagnostics, not agent solve-rate seeds. + +## Conditional speed profiles + +The diagnostic selects the highest-weight particle from each population before inspecting the speed profiles: seed 302 particle 53 and seed 303 particle 61. +For each selected scene it varies only `fan_speed`, holding all other 119 proposal coordinates fixed. +The 29 distinct speed settings per scene include its original speed, and each original trajectory is repeated once. +This produces 60 histories with 3,840 native actions. + +Native job `22710907` and independent reader `22710919` pass. +Both original 65-frame prefixes reproduce exactly, all other scene coordinates and initial observations remain unchanged, and the independent scalar likelihood reference agrees within `2.8422e-14`. +The frozen reports are in `logs/uncertainty_fan_speed_profiles_v2_20260913`. +An earlier tuple/list comparison failure is preserved separately; canonicalized saved trajectories match exactly. + +At both fixed scenes, every tested pair `v` and `1-v` has exactly the same prefix log likelihood. +Ten reflected pairs are checked per scene, spanning speeds 0.025 through 0.975. +For example, 0.0725 and 0.9275 yield the same score, as do 0.12 and 0.88. +The original high-speed candidate, 0.913752, scores only 0.0565 log units below the best low-speed grid point, 0.085, in its own scene. +Thus the high-speed branch is a plausible explanation of this prefix under the declared broad prior, rather than an obviously bad numerical point. +These are conditional profiles at two scenes, not marginalized posterior probabilities or a proof of universal symmetry. +The physical cause of the reflection relation has not yet been independently established. + +The independent reader also recomputes complete-future likelihoods and exact-event mismatches for all 128 original saved histories. +All 64 histories of seed 302 and 63 of seed 303 have zero complete-future density, attributable to disagreements in exact fan, switch or target indicators. +This explains the zero scores; it does not establish that changing the parameter sampler alone will fix the forecasts. + +## Full-population diagnostic + +The next audit replays both original and reflected speeds for every positive-weight particle in both populations, holding the remaining coordinates fixed. +It compares all 133 original frames against their saved histories before interpreting the reflected predictions. +The bounded workload is 128 pairs and 33,792 native actions, on four CPUs in `mit_preemptable`. + +For each original joint proposal coordinate `u`, define the reflection `R(u)` by replacing `u[0]` with `1-u[0]`. +The full fitting target in these coordinates is the prefix log likelihood plus the original-prior/proposal log correction. +The diagnostic computes `a = min(1, exp(log_target(R(u)) - log_target(u)))` from that target alone. +It then assigns weights `w * (1 - a/2)` and `w * a/2` to the original and reflected histories. +This is the expectation of one lazy symmetric Metropolis transition, with reflection proposed half the time. +The transformation preserves the target as a stationary distribution, including when the two target values differ. +It does not imply that the original approximation or the transformed population has converged. +Future observations do not enter these weights. + +The independent reader checks source identities, full original histories, unchanged scene coordinates, prefix and future likelihoods, geometric goal curves, normalized transition weights and an asymmetric two-state detailed-balance reference. +It reports changes in replica disagreement for position, event and goal predictions, while retaining all zero-future-density weight. +The frozen corrected bundle is `logs/uncertainty_fan_reflection_v2_20260913`, with native job `22711267` and dependent reader `22711276`. +The earlier job `22711251` stopped on a source-report field mismatch after matching original native histories; its failed bundle is retained separately. +The correction checks the stored first goal step and independently reconstructs the complete goal curve. + +## Completed full-population result + +Native job `22711267` completes in 3:06 and independent reader `22711276` completes in 1:25. +All 128 original complete histories reproduce exactly. +All 128 reflected candidates have exactly the same fitting-target score as their originals, so the transition splits each original weight equally between its two alternatives. +Ninety-four pairs have exactly equal observed prefixes, and 23 have exactly equal observed futures. +Bitwise trajectory differences in the remaining pairs do not establish a practically different forecast. + +The follow-up saved-history audit `22711446` checks those differences directly, without additional simulation. +Every pair has exactly the same future event indicators and geometric goal curve. +The maximum future ball-coordinate difference is `1.1098e-6` m, far below the 0.005 m position-noise scale. +Some robot readouts differ more, with the largest joint-coordinate difference approximately `2.4921e-4`. +Thus this test establishes speed ambiguity but does not identify it as the cause of the population-level prediction disagreement. + +| Difference between numerical replicas | Original populations | After reflection transition | +|---|---:|---:| +| Native mean-position RMS gap | 0.00669314753 m | 0.00669314772 m | +| Maximum goal-probability gap | 0.3710034 | 0.3710034 | +| Maximum event-probability gap | 0.4991637 | 0.4991637 | + +The independently reconstructed original mean-position, event and goal curves agree with the previously published source reports within `8.8818e-16`. +The final goal probabilities remain 0.8531 and 0.5456 for seeds 302 and 303, respectively. +The zero-future-density masses remain one and approximately 0.9781. +These probabilities concern predictions of the recorded suffix, not agent solve rates. + +This rules out a one-step correction of speed-branch representation as a remedy for the measured forecast disagreement in these two populations. +It does not establish adequate scene exploration, a universally prediction-equivalent parameterization, or a converged posterior. +The next diagnostic should hold the parameter fixed and exchange groups of sampled initial-scene coordinates to locate the remaining event and goal sensitivity. +No reflection option is added to the production agent or sampler on the strength of this negative result. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 5e2155e40..2562b06c0 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,17 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Fan reflection diagnostic](fan-reflection-diagnostic.md) now verifies a conditional ambiguity that helps explain disagreement between independent parameter fits. +At two fixed sampled scenes, ten tested speed pairs `v` and `1-v` per scene have exactly equal fitting-prefix likelihoods. +The original native prefixes reproduce exactly, and all 119 other coordinates remain unchanged across the profiles. +The independent reader passes all 60 profiles and attributes the zero future likelihoods in 127 of 128 original histories to exact event mismatches. +A full-population reflection audit `22711267` and independent reader `22711276` now pass all 128 pairs, totaling 33,792 native actions. +It preserves the declared target and uses only fitting-prefix data for a lazy Metropolis transition; future predictions remain a separate assessment. +All reflected candidates have equal fitting-target scores, but representing both speed branches leaves the event and goal forecasts unchanged. +A separate saved-history audit confirms identical event/goal curves within every pair and maximum future ball-coordinate changes of only `1.1098e-6` m. +The practical disagreement between populations therefore remains; initial-scene and event sensitivity is the next diagnostic, rather than a speed-reflection sampler change. +This is a numerical diagnostic, not an accepted posterior or agent-performance result. + The [Boil supported-inference diagnostic](boil-supported-inference.md) now reproduces the original first-update concentration on both shared-variance seeds. The original base weights already have effective sample sizes 2.264 and 1.074 before the remaining trajectory likelihood is introduced. Whole-joint rejection collects 32 finite candidates in 132 and 128 evaluations; placing all finite factors under tempering preserves the final target and gives initial-temperature effective sample sizes 31.700 and 31.815 on those same retained populations. From 4dad9e80dadeee15b9324f2db4b66687abd7edf4 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 00:30:06 -0400 Subject: [PATCH 79/94] Isolate Fan fixture uncertainty and validate static sensor inference --- docs/uncertainty/fan-reflection-diagnostic.md | 2 +- docs/uncertainty/fan-static-fixtures.md | 90 +++++++++++++++++++ docs/uncertainty/implementation-progress.md | 16 +++- 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 docs/uncertainty/fan-static-fixtures.md diff --git a/docs/uncertainty/fan-reflection-diagnostic.md b/docs/uncertainty/fan-reflection-diagnostic.md index 48397b8af..1eb6c1ba0 100644 --- a/docs/uncertainty/fan-reflection-diagnostic.md +++ b/docs/uncertainty/fan-reflection-diagnostic.md @@ -76,5 +76,5 @@ These probabilities concern predictions of the recorded suffix, not agent solve This rules out a one-step correction of speed-branch representation as a remedy for the measured forecast disagreement in these two populations. It does not establish adequate scene exploration, a universally prediction-equivalent parameterization, or a converged posterior. -The next diagnostic should hold the parameter fixed and exchange groups of sampled initial-scene coordinates to locate the remaining event and goal sensitivity. +The follow-up [fixed-parameter scene exchanges](fan-static-fixtures.md) locate the remaining event and goal sensitivity in fixture poses and evaluate a separate static-fixture discrepancy law. No reflection option is added to the production agent or sampler on the strength of this negative result. diff --git a/docs/uncertainty/fan-static-fixtures.md b/docs/uncertainty/fan-static-fixtures.md new file mode 100644 index 000000000..4bc3ba067 --- /dev/null +++ b/docs/uncertainty/fan-static-fixtures.md @@ -0,0 +1,90 @@ +# Fan fixture sensitivity and static observation model + +This is Stage B development work under the [simplification proposal](simplification-proposal.md). +The [reflection diagnostic](fan-reflection-diagnostic.md) ruled out speed-branch representation as a remedy for the measured event and goal disagreement between two fitted populations. +The following conditional experiments locate another source of that disagreement and assess an explicit alternative discrepancy model. +They are not agent results or evidence of an accepted posterior replacement. + +## Conditional scene exchanges + +The experiment retains the same two highest-weight anchors selected before the reflection profiles: seed 302 particle 53 and seed 303 particle 61. +Each anchor retains its original speed while receiving selected initial-scene coordinates from the other anchor. +These exchanged candidates receive no posterior weights. +Both fitting-prefix scores and reserved-future predictions are reported, with geometry rejections retained explicitly. + +The broad exchange audit `22712070` and independent reader `22712072` pass. +Twelve complete histories use 1,584 native actions, and two other candidates are rejected by the initial geometry check. +The groups are robot joints, fixture poses, ball state, switch articulations and fan rotors. +Exchanging robot joints, switch articulations or fan rotors leaves both anchors' ball, event and goal predictions unchanged in this test. +Exchanging all fixture poses in the seed 303 anchor changes 19 future goal labels and changes its final goal from false to true. +The reverse fixture exchange is geometry-rejected, as is the opposite isolated ball exchange, exposing coupling between fixture and ball positions. + +The individual-fixture audit `22712128` and independent reader `22712131` also pass. +Twenty-five complete histories use 3,300 native actions, and one candidate is geometry-rejected. +Only exchanging the target fixture changes the anchors' final goal outcomes. +It changes 18 and 19 future goal labels, respectively, and changes ball positions by approximately 0.0140 and 0.0151 m RMS. +Exchanging switch 2's fixture pose changes one activation-time label for the switch and its fan, without changing the ball or goal curve. +Jointly exchanging fixtures and ball state is geometrically supported in both directions and transfers the final-goal difference. + +| Target-pose exchange | Fitting-target log-score change | Changed future goal labels | Final predicted goal | +|---|---:|---:|---| +| Seed 302 anchor receives seed 303 target | -1.0727 | 18 | True to false | +| Seed 303 anchor receives seed 302 target | +1.0727 | 19 | False to true | + +The target z coordinates in these two scenes are 0.3965203 and 0.4062214 m, a difference of approximately 9.7 mm. +The target is a physical pad, so uncertainty in its pose can change contact dynamics as well as the geometric goal test. +A follow-up coordinate audit separates x, y, z and yaw for the target and switch 2: native job `22712378` and reader `22712380` both pass. +All 20 histories complete, using 2,640 native actions. +Exchanging only target z changes the final goal in both directions, with 18 and 20 changed future goal labels and ball-position RMS changes of approximately 0.0133 and 0.0151 m. +Exchanging only switch 2 y transfers its one-step activation-time difference in both directions. +Target x changes one goal label in the seed 302 anchor without changing the ball trajectory or final goal; the other coordinate exchanges do not change the checked event or goal curves. +All of these are conditional comparisons at two scenes; they do not establish a population-wide causal decomposition. + +## Static fixtures and the discrepancy law + +The frozen public environment source gives the walls and target zero mass and creates switches with fixed bases. +The frozen candidate program writes only the ball's velocity. +The fixture xyz observation fields therefore describe fixed base positions in this comparison. +The existing model nevertheless assigns an additive AR discrepancy process to these 30 fixture coordinates, in addition to their original 0.005 m sensor noise and sampled initial pose. +That is a modeling choice, rather than a requirement imposed by their dynamics. + +The alternative removes only those 30 discrepancy factors, allowing the unchanged sensor model to score their measurements directly. +It retains all dynamic output factors, angular factors, exact events, physical priors and candidate programs. +This is explicitly a change to the observation/discrepancy model, not a numerical sampler correction. +It is justified by the fixed-fixture source contract and evaluated separately from changes to initialization or sampling budgets. + +The saved-history audit `22712247` and independent reader `22712284` pass all 128 original histories and 3,840 fixture factors without new simulation. +Every checked fixture coordinate is exactly constant throughout its complete native history. +An independent scalar Kalman calculation and Gaussian sensor likelihood reproduce the composite likelihood change within `9.0949e-13`. +The unchanged noise injector gives independent sensor measurements at these recorded steps. + +For one fixture coordinate and 65 fitting observations, the sensor-only constant-location likelihood has standard deviation `0.005 / sqrt(65)`, approximately 0.0006202 m. +The previous discrepancy law gives a corresponding constant-location likelihood width of approximately 0.0033115 m. +These are likelihood widths, not an assertion about the final marginal posterior after all geometry and trajectory constraints. + +Reweighting the existing approximate populations with the fitting-likelihood change gives effective sample sizes approximately 1.0000 and 1.0123 out of 64. +Those weights never use future measurements. +Their concentration prevents treating this reweighting as a reliable posterior comparison and motivates fresh inference under the alternative law. + +The old initialization also does not reproduce Boil's severe base-weight concentration. +The checked seed 302 initial state has 15 supported particles with essentially equal weights and effective sample size 15. +Seed 303's recorded initial effective sample size is 23.0957 among 26 supported particles. +No additional support-initialization or finite-factor-tempering treatment is introduced in this matched model comparison. + +## Fresh matched inference + +The new frozen fit bundle is `logs/uncertainty_fan_static_fits_20260914`. +It retains the original 64 particles, 32 temperatures, eight moves per stage, proposal blocks, broad speed prior and 16,448-evaluation budget, with numerical replicas 302 and 303. +Only the static-fixture discrepancy factors and their corresponding normalized defensive proposal change. +The proposal uses the sensor-only constant-location likelihood while retaining the original proposal-mixture component and original-prior/proposal correction. +Future observations are unavailable to the fitting target. + +Native preflight `22712355` passes all 64 fresh target evaluations across serial and parallel execution and compares complete initial sampler checkpoints, using 8,192 native actions. +Its initial population has 28 finite candidates and weight effective sample size 24.0957, which is an initialization diagnostic rather than evidence of final exploration. +Independent reader `22712364` also passes, reconstructing each physical candidate, rerunning its fitting prefix, checking its complete target factors and verifying the fixed-fixture observation partition with another 4,096 native actions. +Array `22712476` submits the full paired fits for seeds 302 and 303 on `mit_preemptable`, with 16 CPUs and 64 GB per task on the matching-CPU node1411. +The serial/parallel native preflight establishes unchanged evaluation results across the tested execution modes; worker count changes allocation throughput rather than the ordered sampler target or random stream. +Full fits preserve complete-stage checkpoints for preemption recovery and retain the original 16,448-evaluation budget. +The weighted future adapter and independent prediction comparison for this new observation model are still required. +Prediction stability, comparison with the incumbent and adequate numerical exploration remain unproven. +The production estimator remains unchanged. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2562b06c0..0fbd14232 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -1,6 +1,6 @@ # Uncertainty simplification: implementation progress -Updated September 13, 2026. +Updated September 14, 2026. This tracks implementation of the [simplification proposal](simplification-proposal.md). The incumbent estimator remains the production default. @@ -12,6 +12,18 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Fan fixture diagnostic](fan-static-fixtures.md) now isolates a practical forecast sensitivity after the speed-reflection result. +Two verified scene-exchange audits use 4,884 native actions and retain three geometry-rejected candidates. +At the two selected scenes, exchanging the target pad's pose transfers the final-goal difference, while switch 2's pose changes one activation-time label. +The target heights differ by approximately 9.7 mm. +A third verified audit uses 2,640 native actions and isolates the final-goal change to target z; switch 2 y accounts for its one-step activation difference. +An alternative observation model removes motion-discrepancy factors only from 30 source-established fixed-fixture coordinates and retains the original sensor noise. +Its saved-history audit and independent scalar reader pass 128 histories and 3,840 fixture factors, but importance reweighting collapses to approximately one effective sample per population. +Fresh matched inference has passed native preflight `22712355` with 8,192 actions and exact initial-state comparisons, plus 4,096 independent target-reader actions in `22712364`. +The two full fits are submitted as array `22712476` on compute nodes, retaining the original 64 particles, 32 temperatures, eight moves and evaluation budget. +The new model's weighted future adapter and independent prediction comparison remain to be completed. +This is an explicitly changed discrepancy law, not a claimed sampler-only improvement or accepted posterior replacement. + The [Fan reflection diagnostic](fan-reflection-diagnostic.md) now verifies a conditional ambiguity that helps explain disagreement between independent parameter fits. At two fixed sampled scenes, ten tested speed pairs `v` and `1-v` per scene have exactly equal fitting-prefix likelihoods. The original native prefixes reproduce exactly, and all 119 other coordinates remain unchanged across the profiles. @@ -20,7 +32,7 @@ A full-population reflection audit `22711267` and independent reader `22711276` It preserves the declared target and uses only fitting-prefix data for a lazy Metropolis transition; future predictions remain a separate assessment. All reflected candidates have equal fitting-target scores, but representing both speed branches leaves the event and goal forecasts unchanged. A separate saved-history audit confirms identical event/goal curves within every pair and maximum future ball-coordinate changes of only `1.1098e-6` m. -The practical disagreement between populations therefore remains; initial-scene and event sensitivity is the next diagnostic, rather than a speed-reflection sampler change. +The practical disagreement between populations therefore remains; this motivated the initial-scene and event diagnostics reported above rather than a speed-reflection sampler change. This is a numerical diagnostic, not an accepted posterior or agent-performance result. The [Boil supported-inference diagnostic](boil-supported-inference.md) now reproduces the original first-update concentration on both shared-variance seeds. From 860d2ca7abd4367aed9611681fb59009b8bd9ab5 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 00:45:12 -0400 Subject: [PATCH 80/94] Validate and queue weighted Fan static-fixture forecasts --- docs/uncertainty/boil-supported-inference.md | 6 +++-- docs/uncertainty/fan-static-fixtures.md | 25 +++++++++++++++++++- docs/uncertainty/implementation-progress.md | 11 +++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/uncertainty/boil-supported-inference.md b/docs/uncertainty/boil-supported-inference.md index b5d19944c..560470eea 100644 --- a/docs/uncertainty/boil-supported-inference.md +++ b/docs/uncertainty/boil-supported-inference.md @@ -78,7 +78,9 @@ Cached evaluations remain charged to the fitting budget; the original initializa After those checks, the paired full comparison will keep the original 32 particles, eight moves per temperature, proposal blocks, proposal scale and refresh probability. It uses 64 geometrically spaced temperatures from `0.000001` to one and a maximum of 20,000 target evaluations per run. The runs use compute nodes on `mit_preemptable` with the previously audited native runtime pinned to `node1412`. -Full fits `22709091` and `22709092` are now running after the fixture readers passed, with their own readers `22709093` and `22709094` dependent afterward. +Full fits `22709091` and `22709092` have completed all 64 stages, with 15,001 and 14,944 target evaluations and 1,577,004 and 1,638,648 native actions respectively, excluding their separately reported initialization costs. +Their independent readers `22709093` and `22709094` also pass, exactly reproducing the numerical traces and all final native targets with 4,224 additional native actions each. +Both completed populations retain one original particle lineage; this diagnostic alone neither proves adequate exploration nor substitutes for the pending prediction comparison. Each full fit has 16 CPUs, 64 GB memory and an eight-hour allocation. Independent replicas, numerical-budget sensitivity and reserved-future forecasts remain acceptance requirements. No replacement posterior or agent advantage is established by this initialization diagnostic. @@ -122,4 +124,4 @@ Initialization native actions are reported separately from the cached full fit a The new versus previous shared-variance comparison preserves the probability model while changing initialization, tempering and numerical budget. The fixed-variance and incumbent controls retain their separately labelled differences in discrepancy or initial-state treatment. The full forecasts and comparison remain incomplete until the gated jobs finish and their reports are verified. -The adapter gate has passed; the full forecasts are currently waiting for the full fits and their readers. +The adapter and full-fit reader gates have passed; both full forecasts are now running. diff --git a/docs/uncertainty/fan-static-fixtures.md b/docs/uncertainty/fan-static-fixtures.md index 4bc3ba067..f6f3a429c 100644 --- a/docs/uncertainty/fan-static-fixtures.md +++ b/docs/uncertainty/fan-static-fixtures.md @@ -85,6 +85,29 @@ Independent reader `22712364` also passes, reconstructing each physical candidat Array `22712476` submits the full paired fits for seeds 302 and 303 on `mit_preemptable`, with 16 CPUs and 64 GB per task on the matching-CPU node1411. The serial/parallel native preflight establishes unchanged evaluation results across the tested execution modes; worker count changes allocation throughput rather than the ordered sampler target or random stream. Full fits preserve complete-stage checkpoints for preemption recovery and retain the original 16,448-evaluation budget. -The weighted future adapter and independent prediction comparison for this new observation model are still required. +The weighted future adapter and independent prediction checks are now implemented and validated as described below. Prediction stability, comparison with the incumbent and adequate numerical exploration remain unproven. The production estimator remains unchanged. + +## Weighted future pipeline + +The frozen adapter is in `logs/uncertainty_fan_static_forecasts_20260914`. +It recovers a completed source checkpoint without another fitting evaluation and verifies the new proposal-to-canonical-coordinate mapping for every positive-weight particle. +Each particle retains its complete sampled scene, original weight and complete 132-action trajectory. +The changed static-fixture likelihood is used consistently in prefix checks and future scoring. +The full history is generated before future observations are scored, and all zero-density contributions remain in the mixture. + +The short fitting fixture `22712635` completes with 7,808 native actions. +Its full forecast fixture `22712670` completes 64 weighted histories plus one exact repeat, using 8,580 native actions. +Independent reader `22712683` verifies all complete histories and their weights, the completed checkpoint, every prefix and future likelihood, and the ball's conditional output moments. +It independently reconstructs mixture variance using within-component variance plus deviations from the weighted mean, checks a nonuniform two-component reference, and recomputes feature errors, event scores and goal scores. +All eight corruption checks pass, rejecting a dropped particle, changed weights, native predictions, events, goals, density, conditional means and conditional variances. +The deliberately short fixture is numerically unassessed, and its prediction metrics are not evidence for the alternative model. + +Full forecasts `22712813` and `22712819` are queued behind the respective running fits and the passed fixture reader. +Their independent readers are `22712820` and `22712825`. +Comparison `22712836` depends on both readers and on the comparison fixture `22712826`. +The comparison fixture has passed, reproducing both earlier tempered populations and correctly preserving the two new rows as pending. +It runs on the matching-CPU node1412 after only its pending resource request was changed; no frozen code or running fit was modified. +The full comparison retains both original and new numerical replicas and the unchanged incumbent forecast, checking identical data, program, original physical prior and sampling budget while recording the changed discrepancy model. +No full replacement forecast or acceptance result is available yet. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 0fbd14232..6802b8473 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -21,7 +21,9 @@ An alternative observation model removes motion-discrepancy factors only from 30 Its saved-history audit and independent scalar reader pass 128 histories and 3,840 fixture factors, but importance reweighting collapses to approximately one effective sample per population. Fresh matched inference has passed native preflight `22712355` with 8,192 actions and exact initial-state comparisons, plus 4,096 independent target-reader actions in `22712364`. The two full fits are submitted as array `22712476` on compute nodes, retaining the original 64 particles, 32 temperatures, eight moves and evaluation budget. -The new model's weighted future adapter and independent prediction comparison remain to be completed. +The new model's weighted future adapter now passes its native fixture and independent reader: 64 full weighted histories plus one exact repeat, 8,580 native actions, full checkpoint/density/moment checks and eight corruption rejections. +Full forecasts `22712813` and `22712819`, readers `22712820` and `22712825`, and comparison `22712836` are queued behind the fits and passed validation. +The comparison fixture reproduces the two earlier populations and explicitly retains the new results as pending. This is an explicitly changed discrepancy law, not a claimed sampler-only improvement or accepted posterior replacement. The [Fan reflection diagnostic](fan-reflection-diagnostic.md) now verifies a conditional ambiguity that helps explain disagreement between independent parameter fits. @@ -42,13 +44,14 @@ Both corrected independent readers pass all retained native targets, rejection/R The initial diagnostic performs 29,304 native actions across both seeds, and fresh reader verification adds 8,448. The original reader's report-field naming error is preserved separately; it is a checker failure rather than a model or agent outcome. Both small fitting fixtures and their numerical/native readers have now passed, including exact complete proposal/target/checkpoint traces and every final native target. -Full paired fits `22709091` and `22709092` are running on the compute node, with full-fit readers `22709093` and `22709094` dependent afterward. +Full paired fits `22709091` and `22709092` and full-fit readers `22709093` and `22709094` have completed successfully. +They use 15,001 and 14,944 evaluations, respectively, and each retains one original particle lineage; final numerical and predictive adequacy remains unestablished. These compare the same original Boil target under supported initialization and a 64-stage finite-factor tempering path; stable final inference and reserved-future prediction improvement remain unproven. The [weighted Boil future adapter](boil-supported-inference.md#weighted-future-comparison) is implemented with exact completed-checkpoint recovery, unchanged positive weights and complete joint histories, and separate generation and future-density paths. All ten source guards, eight malformed-history checks and an independent nonuniform-weight reference pass. Its native fixture and independent reader have passed: 96 histories, 29,172 generation/check actions, 228,096 verified joint factors and 3,828 additional reader actions, including two exact fresh complete histories. -Full forecasts `22709874` and `22709875`, forecast readers `22709879` and `22709880`, and comparison `22709887` are queued behind the still-running fits and their readers. -The validation work uses a second matching-CPU compute node with exact prefix/replay checks; none of the four running fits is modified. +Full forecasts `22709874` and `22709875` are running, with forecast readers `22709879` and `22709880` and comparison `22709887` dependency-queued. +The validation and forecast work uses a second matching-CPU compute node with exact prefix/replay checks; the frozen fitting inputs are unchanged. The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. It preserves the checked 600-action prefix and generates the 586-action suffix without future observations, retaining one sampled variance per joint throughout each continuation. From f0ea4400bd9a21f421e56f60c8253ca29d3b9dc2 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 01:08:16 -0400 Subject: [PATCH 81/94] Record posterior prediction diagnostics and recover Bridge fits --- .../balloons-transition-sensitivity.md | 53 +++++++++++++++++++ docs/uncertainty/boil-supported-inference.md | 26 ++++++++- docs/uncertainty/bridge-memory-recovery.md | 39 ++++++++++++++ docs/uncertainty/fan-static-fixtures.md | 25 ++++++++- docs/uncertainty/implementation-progress.md | 19 +++++-- 5 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 docs/uncertainty/balloons-transition-sensitivity.md create mode 100644 docs/uncertainty/bridge-memory-recovery.md diff --git a/docs/uncertainty/balloons-transition-sensitivity.md b/docs/uncertainty/balloons-transition-sensitivity.md new file mode 100644 index 000000000..5a6540f23 --- /dev/null +++ b/docs/uncertainty/balloons-transition-sensitivity.md @@ -0,0 +1,53 @@ +# Balloons: sensitivity to future dynamics noise + +September 14, 2026. +This diagnostic follows the failed [prefix posterior forecasts](balloons-prefix-forecasts.md). +It tests conditional continuations, not a refitted probability model or agent performance. + +## Controlled comparison + +The current discrepancy model perturbs arm-joint positions with standard deviation 0.001 and samples a box-velocity transition after every native action. +That velocity transition has a 0.1 probability of setting all three linear velocity components to zero; otherwise it adds Gaussian noise with standard deviation 0.01 per component. +These physical interventions affect subsequent contacts and can therefore change irreversible balloon bursts. + +We select the highest-weight particle from each completed prefix population, independently of future prediction error. +These are particle 48 of fit seed 620, with weight 0.05047, and particle 18 of fit seed 621, with weight 0.05974. +Each particle retains its complete 206-coordinate parameter, initial-state and conditional-direction vector. +Four archived future random seeds per particle are crossed with four interventions: both noise effects, joints only, velocity only, and neither. +All 32 histories reconstruct the same first 64 actions and then continue through the remaining 171 actions in one fresh world. +Suppressed interventions still consume their random draws, preserving the paired random schedules. +Future observations are absent from generation and enter only the subsequent assessment. + +The frozen bundle is `logs/uncertainty_balloons_transition_sensitivity_20260914`. +Native job `22713764` completed in 2:32 on a compute node with the same CPU model as the archived runs. +It executed 9,400 native actions, including eight complete repeated histories. +All eight original-treatment paths reproduce their archived complete histories exactly, and all interventions preserve their original fitting-prefix trajectories and scores exactly. +Independent reader `22713765` completed in 12 seconds. +It checks all 5,472 future joint vectors and velocity draws, recomputes event labels and feature errors, and rejects four deliberately corrupted prefix, joint, velocity and event records. + +## Results + +Each row uses four simulated continuations from one selected fitted state. +Errors are averages of the four individual future-trajectory RMSEs against the reserved clean development trajectory. +These draws are not agent seeds, and their goal counts are not solve rates. + +| Fitting seed | Active future noise | Box-height RMSE (m) | Box-speed RMSE (m/s) | Final burst count | Final predicted goal count | +|---|---|---:|---:|---:|---:| +| 620 | Joint and velocity | 0.18992 | 0.22886 | 0/4 | 0/4 | +| 620 | Joint only | 0.20728 | 0.26137 | 0/4 | 0/4 | +| 620 | Velocity only | 0.20678 | 0.25179 | 0/4 | 0/4 | +| 620 | Neither | 0.22072 | 0.23738 | 0/4 | 0/4 | +| 621 | Joint and velocity | 0.22680 | 0.65371 | 4/4 | 0/4 | +| 621 | Joint only | 0.10548 | 0.41266 | 1/4 | 0/4 | +| 621 | Velocity only | 0.36394 | 0.88007 | 4/4 | 0/4 | +| 621 | Neither | 0.11345 | 0.37631 | 0/4 | 0/4 | + +At the selected seed-621 state, the velocity intervention contributes to spurious bursts and large motion errors. +Removing both interventions eliminates bursts in these four continuations, but still does not recover the goal. +At the selected seed-620 state, removing noise does not improve either reported error. +Thus future transition noise explains part of the regression, but does not explain all of the fitted-state and dynamics error. +There is no basis for promoting a simple noise-disable change to production. + +The next model investigation should separate the velocity reset atom from continuous velocity noise and inspect errors already present in the conditioned prefix. +Any proposed replacement must use a coherent law during both fitting and forecasting, then be refitted and assessed across complete weighted populations. +These two selected states cannot establish posterior-wide improvement, numerical convergence, or the Stage B acceptance gate. diff --git a/docs/uncertainty/boil-supported-inference.md b/docs/uncertainty/boil-supported-inference.md index 560470eea..400244870 100644 --- a/docs/uncertainty/boil-supported-inference.md +++ b/docs/uncertainty/boil-supported-inference.md @@ -123,5 +123,27 @@ The final comparison retains the earlier incumbent, fixed-variance and shared-va Initialization native actions are reported separately from the cached full fit and must be included when assessing total fitting cost. The new versus previous shared-variance comparison preserves the probability model while changing initialization, tempering and numerical budget. The fixed-variance and incumbent controls retain their separately labelled differences in discrepancy or initial-state treatment. -The full forecasts and comparison remain incomplete until the gated jobs finish and their reports are verified. -The adapter and full-fit reader gates have passed; both full forecasts are now running. +Both full forecasts, independent readers and comparison `22709887` have now completed. +All comparison source hashes have been rechecked. + +## Completed reserved-future comparison + +These are offline forecast probabilities from two numerical fits to the same development recording, not agent solve rates. +The original supported initialization and full-target computations have passed their checks, but this does not establish mixing or predictive adequacy. + +| Configuration | Fitting seed | Bubbling RMSE | Final goal probability | Surviving original lineages | +|---|---|---:|---:|---:| +| Incumbent selected point | N/A | 0.01915 | 1.0000 | N/A | +| Earlier shared-variance fit | 410 | 0.28022 | 0.6488 | 1 | +| Earlier shared-variance fit | 411 | 0.30312 | 0.1895 | 1 | +| Supported initialization and finite-factor tempering | 410 | 0.33856 | 0.5637 | 1 | +| Supported initialization and finite-factor tempering | 411 | 0.31636 | 0.7691 | 1 | + +The two new forecasts disagree by 0.2054 in final-goal probability and by up to 0.2302 on the boiled predicate over the future trajectory. +Their future-mixture log-density estimates are -7,542.19 and 17,414.58, another substantial disagreement requiring investigation. +Both retain one original lineage; this alone is not a proof of numerical failure, but neither independent replication nor the prediction comparison supports acceptance. +The new initialization and tempering changes do not fix the practical bubbling-prediction regression. +Some position coordinates improve, so retain the complete feature comparison in `logs/uncertainty_boil_supported_forecasts_20260913/model-comparison.json`. + +Forecasting uses 89,364 and 113,124 native actions, excluding the separately reported fit, initialization and independent verification costs. +Stage B remains open, and the production estimator remains unchanged. diff --git a/docs/uncertainty/bridge-memory-recovery.md b/docs/uncertainty/bridge-memory-recovery.md new file mode 100644 index 000000000..e9437d2fb --- /dev/null +++ b/docs/uncertainty/bridge-memory-recovery.md @@ -0,0 +1,39 @@ +# Bridge inference: checkpoint recovery after memory exhaustion + +September 14, 2026. +Both full supported-initialization fits in array `22706932` terminated with scheduler state `OUT_OF_MEMORY` after 2:51:51. +Each allocation reached approximately 64 GiB of resident memory. +These are infrastructure interruptions, not completed inference results or agent failures. + +Both last complete checkpoints are at temperature stage 26 of 64. +Seed 810 has 4,535 logical evaluations and seed 811 has 4,348; both retain 32 original particle lineages at that stage. +The original interrupted reports, ledgers and checkpoints remain unchanged. +The recovery bundle, `logs/uncertainty_bridge_memory_resume_20260914`, contains checksummed copies of the complete checkpoint prefixes and explicit links to the interrupted artifacts. +The 38 and 69 evaluations logged after the respective complete checkpoints are retained in the original ledgers and excluded from resumed sampler history. +Their resource cost remains part of the interrupted allocations. + +Validation job `22713841` reproduces both entire numerical traces through their saved checkpoints, including sampler random state. +Four fresh native target checks also reproduce their saved joint values and scores exactly, using 2,400 native actions. +This validates the recovery point rather than claiming that the incomplete posterior is adequate. + +The first resume array, `22713869`, failed before sampling because its frozen loader compared JSON lists with tuples from dataclass serialization. +The checkpoint, model identity and configuration values were unchanged. +An isolated launcher canonicalizes report-only dataclass serialization before calling the frozen fitter; the probability model and sampler objects remain unchanged. +Both end-to-end loader preflights in array `22714028` pass, reaching the sampler with the exact saved identity, configuration and checkpoint. +The launcher and preflight outputs have separate source hashes in the recovery bundle. +The failed first resume allocations and their cancelled dependent pipelines remain recorded as setup failures. + +Array `22714037` resumes these checkpoints with the unchanged frozen fitter, probability model, sampler configuration, 16 ordered workers, and original evaluation budget. +Each new allocation requests 128 GiB instead of 64 GiB on the original compute node. +The underlying memory-growth cause has not been isolated; the larger allocation provides headroom for the remaining stages. +Readers `22714038` and `22714039` follow the fits and will independently replay the combined pre-interruption and resumed ledgers. + +The replacement forecast pipeline is frozen in `logs/uncertainty_bridge_resumed_forecasts_v2_20260914`. +Its forecast and summary calculations are unchanged; source paths point to the resumed fits and new output directory. +Forecasts `22714051` and `22714054` depend on their fit readers, followed by readers `22714052` and `22714055` and comparison `22714056`. +Every full forecast reruns the existing summary guards, and its reader checks complete histories and fresh native repetitions. +The old downstream jobs cannot proceed after their failed dependencies and are terminal. + +The resumed fit report's elapsed-time field covers only its new allocation. +Total inference cost must also include the interrupted allocation, initialization and recovery validation; do not quote the new elapsed-time field as total cost. +The numerical and predictive gates remain open. diff --git a/docs/uncertainty/fan-static-fixtures.md b/docs/uncertainty/fan-static-fixtures.md index f6f3a429c..26fbc5c6f 100644 --- a/docs/uncertainty/fan-static-fixtures.md +++ b/docs/uncertainty/fan-static-fixtures.md @@ -110,4 +110,27 @@ Comparison `22712836` depends on both readers and on the comparison fixture `227 The comparison fixture has passed, reproducing both earlier tempered populations and correctly preserving the two new rows as pending. It runs on the matching-CPU node1412 after only its pending resource request was changed; no frozen code or running fit was modified. The full comparison retains both original and new numerical replicas and the unchanged incumbent forecast, checking identical data, program, original physical prior and sampling budget while recording the changed discrepancy model. -No full replacement forecast or acceptance result is available yet. +Both fits, full forecasts, independent readers and comparison `22712836` have now completed. +The following results supersede the pending status above. +All new forecast hashes match their independent verification reports. + +## Completed static-fixture comparison + +| Configuration | Fitting seed | Ball-position RMSE (m) | Goal Brier score | Final goal probability | Zero-density histories | +|---|---|---:|---:|---:|---:| +| Incumbent selected point | N/A | 0.007072 | 0.044118 | 1.0000 | N/A | +| Original discrepancy | 302 | 0.005348 | 0.017019 | 0.8531 | 64/64 | +| Original discrepancy | 303 | 0.008733 | 0.064487 | 0.5456 | 63/64 | +| Static-fixture sensor model | 302 | 0.005858 | 0.018993 | 0.9844 | 57/64 | +| Static-fixture sensor model | 303 | 0.006059 | 0.020533 | 0.8645 | 62/64 | + +The new model reduces between-fit native position RMS disagreement from 0.006693 m to 0.000619 m. +Both new goal Brier scores are better than the incumbent's on this development recording, and the formerly weaker numerical replica improves. +However, maximum goal-probability disagreement over the future remains 0.36726, compared with 0.37100 previously. +Most positive-weight histories still contradict at least one exact future observation, and the two fits retain one and two original lineages. +The model change therefore improves some forecasts without closing the numerical or predictive gate. +These are offline predictions, not agent solve-rate results. + +The new fits use 15,453 and 15,492 target evaluations, close to the original 15,455 and 15,550. +Their shorter elapsed times also reflect the increase from four to sixteen compute workers and must not be attributed solely to the observation-model change. +Each forecast uses 8,580 native actions including its repeated history, separate from fitting and reader costs. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 6802b8473..2f3fa62a1 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,15 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. +At one selected fitted state, removing future joint and velocity noise eliminates bursts in four continuations, but does not recover the goal; the other selected state does not improve. +This isolates a contribution from the physical discrepancy model without establishing a replacement posterior or an agent improvement. + +The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. +Both complete numerical prefixes and four fresh native targets reproduce exactly. +After fixing and checking a JSON-container comparison in the resume launcher, array `22714037` now resumes the same fits with 128 GiB per allocation, with replacement fit readers and the complete forecast/verification pipeline queued. +This is recovery from an infrastructure interruption; the incomplete fits are not model or agent outcomes. + The [Fan fixture diagnostic](fan-static-fixtures.md) now isolates a practical forecast sensitivity after the speed-reflection result. Two verified scene-exchange audits use 4,884 native actions and retain three geometry-rejected candidates. At the two selected scenes, exchanging the target pad's pose transfers the final-goal difference, while switch 2's pose changes one activation-time label. @@ -22,8 +31,10 @@ Its saved-history audit and independent scalar reader pass 128 histories and 3,8 Fresh matched inference has passed native preflight `22712355` with 8,192 actions and exact initial-state comparisons, plus 4,096 independent target-reader actions in `22712364`. The two full fits are submitted as array `22712476` on compute nodes, retaining the original 64 particles, 32 temperatures, eight moves and evaluation budget. The new model's weighted future adapter now passes its native fixture and independent reader: 64 full weighted histories plus one exact repeat, 8,580 native actions, full checkpoint/density/moment checks and eight corruption rejections. -Full forecasts `22712813` and `22712819`, readers `22712820` and `22712825`, and comparison `22712836` are queued behind the fits and passed validation. -The comparison fixture reproduces the two earlier populations and explicitly retains the new results as pending. +Full forecasts `22712813` and `22712819`, readers `22712820` and `22712825`, and comparison `22712836` have completed. +The revised static-fixture model reduces between-fit position disagreement from 6.69 mm to 0.619 mm, and both goal Brier scores improve over the incumbent on this recording. +However, maximum goal-probability disagreement across the future remains 0.3673, and 57/64 and 62/64 retained histories assign zero density to the recorded future. +The practical improvement is partial and does not establish numerical or predictive acceptance. This is an explicitly changed discrepancy law, not a claimed sampler-only improvement or accepted posterior replacement. The [Fan reflection diagnostic](fan-reflection-diagnostic.md) now verifies a conditional ambiguity that helps explain disagreement between independent parameter fits. @@ -50,7 +61,9 @@ These compare the same original Boil target under supported initialization and a The [weighted Boil future adapter](boil-supported-inference.md#weighted-future-comparison) is implemented with exact completed-checkpoint recovery, unchanged positive weights and complete joint histories, and separate generation and future-density paths. All ten source guards, eight malformed-history checks and an independent nonuniform-weight reference pass. Its native fixture and independent reader have passed: 96 histories, 29,172 generation/check actions, 228,096 verified joint factors and 3,828 additional reader actions, including two exact fresh complete histories. -Full forecasts `22709874` and `22709875` are running, with forecast readers `22709879` and `22709880` and comparison `22709887` dependency-queued. +Full forecasts `22709874` and `22709875`, readers `22709879` and `22709880`, and comparison `22709887` have completed. +Both new fits remain worse than the incumbent on bubbling prediction, and their final-goal probabilities differ by 0.2054. +Supported initialization improves the initial population but has not established adequate final inference or prediction. The validation and forecast work uses a second matching-CPU compute node with exact prefix/replay checks; the frozen fitting inputs are unchanged. The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. From f90aa568f5237ff2ca848c55312d0bc532394b5d Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 01:34:32 -0400 Subject: [PATCH 82/94] Diagnose Balloons inference errors and test default-centered guidance --- .../balloons-transition-sensitivity.md | 106 ++++++++++++++++++ docs/uncertainty/implementation-progress.md | 6 + 2 files changed, 112 insertions(+) diff --git a/docs/uncertainty/balloons-transition-sensitivity.md b/docs/uncertainty/balloons-transition-sensitivity.md index 5a6540f23..22504288e 100644 --- a/docs/uncertainty/balloons-transition-sensitivity.md +++ b/docs/uncertainty/balloons-transition-sensitivity.md @@ -51,3 +51,109 @@ There is no basis for promoting a simple noise-disable change to production. The next model investigation should separate the velocity reset atom from continuous velocity noise and inspect errors already present in the conditioned prefix. Any proposed replacement must use a coherent law during both fitting and forecasting, then be refitted and assessed across complete weighted populations. These two selected states cannot establish posterior-wide improvement, numerical convergence, or the Stage B acceptance gate. + +## Separating reset events from continuous velocity noise + +The follow-up bundle `logs/uncertainty_balloons_velocity_components_20260914` crosses joint noise on/off with either the velocity reset branch or the Gaussian branch. +Suppressing a reset retains the native velocity on that branch; it does not replace the skipped reset with a new Gaussian draw. +Suppressing a Gaussian branch still draws its noise and then retains native velocity. +This preserves the original paired random schedules and isolates interventions; it does not define or fit a new posterior. + +Native job `22714229` completed in 2:45 with 11,280 actions, including eight original-treatment baseline histories and eight complete repeats. +All original-treatment histories and all fitting prefixes reproduce exactly. +Reader `22714231` completed in 10 seconds, checking the 32 new histories, 5,472 future joint vectors and velocity draws, and four corruption controls. + +| Fitting seed | Active future noise | Box-height RMSE (m) | Box-speed RMSE (m/s) | Final burst count | Final predicted goal count | +|---|---|---:|---:|---:|---:| +| 620 | Joint and Gaussian velocity | 0.20953 | 0.23888 | 0/4 | 0/4 | +| 620 | Gaussian velocity only | 0.19419 | 0.26363 | 0/4 | 0/4 | +| 620 | Joint and velocity reset | 0.40998 | 1.27533 | 2/4 | 0/4 | +| 620 | Velocity reset only | 0.23846 | 0.63505 | 1/4 | 0/4 | +| 621 | Joint and Gaussian velocity | 0.18292 | 0.34631 | 4/4 | 0/4 | +| 621 | Gaussian velocity only | 0.20205 | 0.35242 | 4/4 | 0/4 | +| 621 | Joint and velocity reset | 0.10594 | 0.42441 | 2/4 | 0/4 | +| 621 | Velocity reset only | 0.11305 | 0.38959 | 0/4 | 0/4 | + +At the selected seed-621 state, continuous velocity noise alone suffices to cause the four observed forecast bursts, while resets alone do not. +At the selected seed-620 state, reset interventions can also cause bursts. +The effects are nonlinear and depend on the fitted state; the reset branch is not the sole cause. +Neither intervention recovers the goal. + +The same reader finds substantial discrepancies during the conditioned fitting prefix. +The two selected histories require speed corrections with RMS 0.26270 and 0.06844 m/s, compared with the declared 0.01 m/s continuous velocity scale. +All 64 fitted speed observations in each history are positive, so none uses the zero-speed atom during conditioning. +The nominal discrepancy scale should not be interpreted as a bound on these conditioned corrections. +These errors motivate investigation of the sampled parameters and initial states in addition to future noise. + +## Conditional parameter profiles + +The frozen program's own decision record describes an earlier overdamped optimizer solution that matched mean heights but lost the observed oscillation. +The selected posterior states have drag values 23.20 and 21.20, compared with the program's default 1.979. +This motivates a diagnostic rather than a conclusion that changing drag alone will fix the problem. + +Bundle `logs/uncertainty_balloons_parameter_profiles_20260914` evaluates eight predefined parameter profiles at each selected initial state and latent direction path. +Profiles include the selected parameters, all existing program defaults, selected single-parameter changes and three interpolations in the original prior's unit coordinates. +All 196 non-parameter coordinates remain fixed within each profile. +The parameters remain inside the same original prior bounds. +The program defaults already contain historical development-data choices and are not independent evidence or a newly asserted prior. + +Native job `22714315` completed in 1:41 with 5,808 actions. +Every fitting prefix repeats exactly and matches its separately generated full-history prefix. +Independent reader `22714316` completed in 19 seconds, verifying all 1,024 radial speed factors, joint factors, output-score composition, parameter edits and future intervention semantics. +Future evaluation uses native continuation with both physical noise interventions disabled, separately from the unchanged conditioned-prefix score. + +| Fitting seed | Parameter profile | Prefix log score | Prefix height RMSE against noisy observations (m) | Future height RMSE against clean observations (m) | Future speed RMSE (m/s) | +|---|---|---:|---:|---:|---:| +| 620 | Selected | -14,534.08 | 0.02552 | 0.22072 | 0.23738 | +| 620 | Program defaults | 4,920.01 | 0.01378 | 0.01279 | 0.05321 | +| 621 | Selected | 7,411.31 | 0.02291 | 0.11345 | 0.37631 | +| 621 | Program defaults | -1,477.20 | 0.02682 | 0.01383 | 0.05748 | + +The prefix scores are conditional likelihoods at fixed initial states and latent paths, not marginal parameter evidence or posterior weights. +At the selected seed-620 state, defaults improve the score by about 19,454 and greatly improve the native future prediction. +This exposes a much better conditional fitting point than that particular retained sample. +At the seed-621 state, defaults improve future motion prediction but reduce the fitted output score enough to lower the total prefix score. +Other profiles have large discontinuities and can produce violent trajectories; the complete report retains those failures. +Changing drag alone barely changes seed 621's prefix score yet severely worsens its future prediction. +Thus both numerical exploration and the relationship between the conditional scoring model and future dynamics remain concerns. +None of these profiles predicts the final goal correctly. + +The next proposal audit uses prefix observations before the first exact clip/tie change to estimate initial-location proposal centers, and tests those centers with existing program defaults and selected parameters. +It retains the original prior, feasibility rules and likelihood, including their treatment of motion. +The use of a quiet-prefix mean is a proposal heuristic, not an assumption that those measurements are independent initial-state observations in the likelihood. +The audit must verify normalized proposal densities and native target scores before any new fitting run uses it. + +## Verified proposal audit and matched fits + +The audit in `logs/uncertainty_balloons_prefix_guidance_20260914` uses observations 0 through 22, before the first exact clip/tie change at step 23, for its optional initial-location means. +It compares four proposal centers with the same twelve random inputs per guide, plus one evaluation of each center. +Native job `22714432` completed in 1:46 with 4,864 actions, including repeated evaluations of all 52 points. +Geometry or exact-output failures remain explicit unsupported candidates. + +| Proposal center | Finite random draws | Best random-draw prefix log score | Center prefix log score | +|---|---:|---:|---:| +| Original guide | 3/12 | -1,252,294.47 | -268,373.62 | +| Original scene, program-default parameters | 8/12 | 6,559.01 | -651.49 | +| Prefix location means, program-default parameters | 9/12 | 4,516.61 | Unsupported | +| Prefix location means, selected seed-621 parameters | 5/12 | 5,960.21 | Unsupported | + +These small counts describe proposal support and are not posterior-quality estimates. +The independent reader verifies all 52 mixture proposal densities and 38 evaluated conditional likelihoods, including the original correction for the full mixture density. +The first reader, `22714433`, encountered a 2.78e-17 inverse-CDF coordinate difference between AMD and Intel processors while checking exact proposal provenance. +Reader `22714522` passes the unchanged checks on the same Intel CPU model as generation. +This is a reader/runtime precision issue, not an agent or model failure. + +The next comparison changes only the original guide's ten parameter-center coordinates to the frozen program defaults. +It retains the original scene center; the additional mean-location change is not included in these fits. +The original broad proposal component and normalized mixture correction remain intact, so this changes how the target is explored rather than changing the declared prior or likelihood. + +The frozen fitting bundle is `logs/uncertainty_balloons_default_guided_fits_20260914`. +Native target validation `22714576` has passed: archived prefix histories reproduce, future-data corruption leaves the fitting data unchanged, 14 of 16 new proposal cases have finite support, and serial/parallel target evaluations agree exactly. +The maximum checked factorization error is 2.27e-13. +Array `22714577` runs numerical seeds 620 and 621 with the original 64 particles, 32 temperatures, eight moves, four workers and 16,448-evaluation budget. +Fit readers `22714628` and `22714629` will recover each complete checkpoint and freshly repeat every retained native target. + +The two new fits remain separate from the earlier populations with the same numerical seed labels. +Complete weighted reserved-future forecasts and numerical-stability checks are still required before assessing the changed guide. +In particular, adequate inference must retain prior uncertainty for parameters that the recorded prefix does not inform; a better conditional fitting score alone is insufficient. +There is no new agent-performance result or production-estimator change. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2f3fa62a1..0a1b48ec7 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -15,6 +15,12 @@ The full plan remains incomplete, and the incumbent estimator remains the produc The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. At one selected fitted state, removing future joint and velocity noise eliminates bursts in four continuations, but does not recover the goal; the other selected state does not improve. This isolates a contribution from the physical discrepancy model without establishing a replacement posterior or an agent improvement. +The subsequent velocity-component audit verifies another 32 paired interventions and shows that both reset events and continuous noise can cause bursts, depending on the fitted state. +Sixteen verified conditional parameter profiles expose both a much better fitting point than one retained sample and disagreement between fitted score and future quality at the other selected state. +An unchanged-target proposal audit verifies 52 proposal densities and motivates a matched guide-center-only comparison. +Native validation `22714576` passes exact archived replay, future-data isolation, factorization and serial/parallel checks. +Array `22714577` now runs the two original numerical seeds with only the parameter proposal center changed to the fixed program defaults; original priors, likelihood and sampling budgets remain unchanged. +Independent complete-checkpoint and final-target readers `22714628` and `22714629` are queued; new weighted future comparisons remain outstanding. The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. Both complete numerical prefixes and four fresh native targets reproduce exactly. From 63f1604f2bbc6e291e53787d03d139e38a6354af Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 02:02:13 -0400 Subject: [PATCH 83/94] Diagnose Balloons parameter concentration and test selective guidance --- .../balloons-selective-guidance.md | 80 +++++++++++++++++++ .../balloons-transition-sensitivity.md | 37 +++++++++ docs/uncertainty/implementation-progress.md | 10 ++- 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/balloons-selective-guidance.md diff --git a/docs/uncertainty/balloons-selective-guidance.md b/docs/uncertainty/balloons-selective-guidance.md new file mode 100644 index 000000000..5da2d58eb --- /dev/null +++ b/docs/uncertainty/balloons-selective-guidance.md @@ -0,0 +1,80 @@ +# Balloons: keeping unobserved parameters broad during inference + +September 14, 2026. +This is an offline numerical investigation for Stages A/B of the [simplification proposal](simplification-proposal.md). +The production estimator is unchanged. + +## Reproduced dependence and concentration + +The fixed Balloons program reads a balloon's color-specific lift coefficient only when it is tied and live, and selects box mass by its material. +This suggests that a recording which activates only one color on one material cannot constrain every parameter in the program. +The native audit tests that suggestion on the actual 64-action fitting prefix at two supported retained states from the original numerical fits. + +At each state, it evaluates the baseline, each of ten parameters at prior-unit coordinates 0.05 and 0.95, and two simultaneous interventions on the five suspected inactive parameters. +All 196 non-parameter coordinates remain fixed. +Every candidate repeats in a fresh native trajectory; complete observations, transition corrections, support outcomes and likelihood components are retained for comparison. + +Native job `22715205` completed all 46 cases in 2:28, using 5,888 actions. +Independent reader `22715206` completed in 59 seconds with 1,792 additional native actions, checking all recorded interventions, all twenty parameter summaries, and six deliberate corruptions. +Its source checksum matches the native report. +Artifacts are in `logs/uncertainty_balloons_parameter_dependence_20260914`. + +At both states, changing `lift_c0`, `lift_c1`, `lift_c2`, `mass_pine`, or `mass_teak`, separately or together, preserves the complete evaluation exactly. +The other five parameters change the evaluation, so the audit also has positive sensitivity controls. +These are finite local checks; they are not a proof of global conditional independence over every feasible latent state. + +The original completed fits nevertheless retain extremely narrow distributions for these locally inactive parameters. +The following discrepancies compare the weighted empirical distribution with the declared prior after transforming that prior to a uniform unit coordinate. +A uniform coordinate has standard deviation approximately 0.289. +The CDF gap is descriptive; it is not a p-value or a complete convergence test. + +| Parameter | Seed 620 unit SD | Seed 621 unit SD | Seed 620 maximum CDF gap | Seed 621 maximum CDF gap | +|---|---:|---:|---:|---:| +| Red lift | 0.0091 | 0.0102 | 0.5283 | 0.5367 | +| Blue lift | 0.0105 | 0.0165 | 0.5403 | 0.4902 | +| Green lift | 0.0066 | 0.0037 | 0.5121 | 0.6722 | +| Pine mass | 0.0105 | 0.0144 | 0.5118 | 0.5522 | +| Teak mass | 0.0067 | 0.0104 | 0.5257 | 0.4950 | + +This supports a concrete numerical concern: concentration inherited from the initial proposal and resampling can persist even along directions where the checked likelihood does not change. +A higher likelihood at a retained point cannot establish that its surrounding parameter uncertainty is correct. +The new default-centered fits remain a separate ongoing comparison and are not replaced by this diagnostic. + +## Target-preserving numerical change + +Bundle `logs/uncertainty_balloons_selective_guidance_20260914` adds a selective version of the existing normalized joint proposal. +The five diagnosed coordinates are sampled uniformly in every proposal component, while the existing local guidance remains on the other coordinates. +The broad component and full mixture density correction are preserved. +An empty selection reproduces the original proposal. + +The fitting driver also mixes a 20% independent uniform proposal into each existing coordinate block's random-walk kernel. +Both moves are symmetric in the proposal chart and receive the existing Metropolis acceptance check with the complete target density. +Every evaluated candidate continues to incur its full native fitting cost. +The prior, physical simulator program, observation and dynamics-discrepancy laws, fitting prefix, and future-data boundary remain unchanged. +The paired numerical seeds, particle count, temperature schedule, number of moves and evaluation budget remain unchanged. + +Neither change requires treating the selected parameters as globally independent of the data. +If a selected parameter does affect an untested history, the native likelihood and density-corrected acceptance decision still account for that effect. +No fitted marginal is manually replaced by the prior, and no state or parameter coordinate is removed from the inference problem. + +Native validation `22715307` completed in 1:48 on the matching Intel compute node. +It passes 32 exact default-parity cases, 32 independent full-chart mixture-density cases, uniform-coordinate mappings, normalized importance integration and four malformed-mask rejections. +The corrected integral 0.27571630757858434 agrees with independent quadrature 0.2757163075785848. +An additional likelihood that depends on a selected uniform coordinate preserves evidence 2 and conditional mean 7/12, checking that selecting a coordinate does not remove its likelihood. +The existing exact prefix replays pass, 14 of 16 new guide candidates have finite support, serial and parallel targets agree exactly, and future-data corruption leaves the fitting target unchanged. +The maximum target-factorization discrepancy is 9.09e-13. + +Array `22715405` runs paired fitting seeds 620 and 621 on `mit_preemptable`, each with four CPUs and 20 GiB. +It retains 64 particles, 32 cubic temperatures, eight moves, the original blocks and a 16,448-evaluation cap. +Final-target readers `22715406` and `22715407` are dependency-queued to recover each complete checkpoint and freshly evaluate every retained particle. +Their frozen bundle is `logs/uncertainty_balloons_selective_verification_20260914`. +A complete weighted future adapter for this selective proposal is still required; the existing default-guide adapter must not decode its particles without the new coordinate selection. + +## Remaining acceptance work + +Compare these populations with the original and default-centered fits, keeping each treatment's identity and proposal mapping separate. +Check parameter marginals, independent-run agreement, budget sensitivity and full weighted reserved-action predictions. +Broader marginals alone do not establish adequate inference in the remaining scene and dynamics coordinates. +Improving numerical exploration also does not fix an inadequate dynamics or discrepancy model. +The [Balloons transition diagnostics](balloons-transition-sensitivity.md) remain relevant to that separate predictive issue. +No Stage B acceptance or live-agent improvement follows from this experiment's launch. diff --git a/docs/uncertainty/balloons-transition-sensitivity.md b/docs/uncertainty/balloons-transition-sensitivity.md index 22504288e..b411fbfba 100644 --- a/docs/uncertainty/balloons-transition-sensitivity.md +++ b/docs/uncertainty/balloons-transition-sensitivity.md @@ -157,3 +157,40 @@ The two new fits remain separate from the earlier populations with the same nume Complete weighted reserved-future forecasts and numerical-stability checks are still required before assessing the changed guide. In particular, adequate inference must retain prior uncertainty for parameters that the recorded prefix does not inform; a better conditional fitting score alone is insufficient. There is no new agent-performance result or production-estimator change. + + +## Weighted comparison for the new guide + +Bundle `logs/uncertainty_balloons_default_guided_forecasts_20260914` binds the forecast adapter to the new guide and fitting identities. +It preserves every retained positive-weight particle and its full joint history, and separates future generation from conditioning on reserved observations for density assessment. +The frozen forecasting budget remains two banks of four generation draws and eight density draws per positive-weight particle. +Source preparation waits for independent complete-checkpoint and final-target verification, then publishes checksummed immutable copies of the reports and checkpoints. +It resolves the verified final attempt rather than copying live files or assuming attempt zero will finish. + +Native adapter job `22715007` completed in 1:18 with 3,012 actions. +Independent reader `22715008` completed in 29 seconds with 192 additional prefix actions. +The checks cover three distinct physical candidates across six generation/density histories, including two candidates drawn through local components of the new guide. +They also retain twelve malformed-result rejections and verify that the old guide cannot decode a new local particle as the stored physical state. +The reader's source checksum matches the completed adapter report. +These results establish the tested adapter mechanics, not the quality of the pending fitted posterior. + +Full forecast jobs `22715115` and `22715117` wait for fit readers `22714628`/`22714629` and the adapter reader. +Independent forecast readers `22715116` and `22715118` and final comparison `22715120` are dependency-queued. +Comparison fixture `22715119` completes successfully, verifying the original results and incumbent while retaining missing new forecasts as pending. +The comparison preserves the original physical prior, recording, simulator program, likelihood, numerical seeds and fitting budget across the old and new guides. +It reports replica agreement, predictive errors, future density, ancestry and fit/forecast cost without treating a higher fitting score as acceptance. +Cost records include available interrupted-attempt reports and explicitly exclude native work that an interrupted process never returned. + +## Checking parameter dependence + +The fixed program selects box mass by material and reads a balloon's color-specific lift coefficient when that balloon is tied and live. +This suggests that the fitting prefix may leave several parameters unobserved. +Bundle `logs/uncertainty_balloons_parameter_dependence_20260914` tests this using the same two supported fitted scenes and latent direction paths as the conditional profiles. +At each scene it evaluates the original point, each of the ten parameter coordinates at prior-unit values 0.05 and 0.95, and two simultaneous interventions on the five suspected inactive parameters. +All non-parameter coordinates remain fixed, and every evaluation repeats in a fresh native trajectory. +The report preserves both changed and unchanged predictions, likelihoods and support outcomes rather than assuming the suspected independence is true. +It also compares the original completed populations' weighted parameter marginals with the declared prior in unit coordinates. +These finite interventions are local dependence evidence; they do not by themselves prove a global likelihood factorization. +Native job `22715205` and independent reader `22715206` have completed on the matching Intel compute node, with 5,888 and 1,792 native actions respectively. +All five suspected inactive parameters preserve both evaluated histories and scores exactly, while the other parameters have detectable effects. +The [selective-guidance follow-up](balloons-selective-guidance.md) records the unexpectedly narrow fitted marginals and a target-preserving numerical intervention. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 0a1b48ec7..5721abc0d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -20,7 +20,15 @@ Sixteen verified conditional parameter profiles expose both a much better fittin An unchanged-target proposal audit verifies 52 proposal densities and motivates a matched guide-center-only comparison. Native validation `22714576` passes exact archived replay, future-data isolation, factorization and serial/parallel checks. Array `22714577` now runs the two original numerical seeds with only the parameter proposal center changed to the fixed program defaults; original priors, likelihood and sampling budgets remain unchanged. -Independent complete-checkpoint and final-target readers `22714628` and `22714629` are queued; new weighted future comparisons remain outstanding. +Independent complete-checkpoint and final-target readers `22714628` and `22714629` are queued. +The new weighted forecast adapter and independent reader `22715007`/`22715008` have passed, checking both broad and local proposal mappings, complete native prefixes, generation and density histories, and rejection of the old guide when decoding new particles. +Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` are dependency-queued behind the verified completed fits. +The comparison retains both earlier fits and the incumbent, including prediction metrics and computation costs; its fixture `22715119` passes while correctly leaving the new outcomes pending. +The [parameter-dependence audit](balloons-selective-guidance.md) and independent reader `22715205`/`22715206` have completed all 46 native cases and twenty marginal checks. +Changing five parameters separately or together leaves both tested fitting histories and scores exactly unchanged, while the original fits retain very narrow marginals for them. +A selective guide with uniform proposals on those coordinates and 20% block refresh moves now passes native validation `22715307`, including 32 default-parity and 32 independent mixture-density checks. +Paired fits `22715405` are running with final-target readers `22715406`/`22715407` dependent on completion; weighted future validation for this selective proposal remains outstanding. +It retains the complete prior, likelihood and Metropolis correction and does not assume global parameter independence. The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. Both complete numerical prefixes and four fresh native targets reproduce exactly. From 720c1b8f0582838a41498bd7d37337df4342f1a0 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 02:09:42 -0400 Subject: [PATCH 84/94] Validate selective Balloons posterior forecasts and queue comparisons --- .../balloons-selective-guidance.md | 30 ++++++++++++++++++- docs/uncertainty/implementation-progress.md | 6 +++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/balloons-selective-guidance.md b/docs/uncertainty/balloons-selective-guidance.md index 5da2d58eb..0460fe405 100644 --- a/docs/uncertainty/balloons-selective-guidance.md +++ b/docs/uncertainty/balloons-selective-guidance.md @@ -68,7 +68,8 @@ Array `22715405` runs paired fitting seeds 620 and 621 on `mit_preemptable`, eac It retains 64 particles, 32 cubic temperatures, eight moves, the original blocks and a 16,448-evaluation cap. Final-target readers `22715406` and `22715407` are dependency-queued to recover each complete checkpoint and freshly evaluate every retained particle. Their frozen bundle is `logs/uncertainty_balloons_selective_verification_20260914`. -A complete weighted future adapter for this selective proposal is still required; the existing default-guide adapter must not decode its particles without the new coordinate selection. +The selective-proposal weighted future adapter in `logs/uncertainty_balloons_selective_forecasts_20260914` has passed native validation and independent verification. +Its particle decoder binds the new guide and all five uniform-coordinate selections; the default-guide decoder cannot be substituted for it. ## Remaining acceptance work @@ -78,3 +79,30 @@ Broader marginals alone do not establish adequate inference in the remaining sce Improving numerical exploration also does not fix an inadequate dynamics or discrepancy model. The [Balloons transition diagnostics](balloons-transition-sensitivity.md) remain relevant to that separate predictive issue. No Stage B acceptance or live-agent improvement follows from this experiment's launch. + + +## Weighted prediction pipeline + +The new pipeline keeps the native generation, density integration and weighted summary methods from the previously verified default-guide adapter. +It binds the new fitting identities and selective proposal, preserving the physical coordinates, complete histories, and positive weights of every retained particle. +It uses the unchanged two banks of four future generations and eight conditional-density draws per particle. +The source-publication step waits for independent completed-fit verification and then creates immutable report and checkpoint copies. + +Native adapter `22715541` completed in 1:12 with 3,012 actions, and independent reader `22715542` completed in 27 seconds with another 192 prefix actions. +They pass broad and local component checks, repeatability, full checkpoint recovery, weighted summaries, density denominators and twelve malformed-result rejections. +Both completed reports match the checksums of their frozen inputs and source artifacts. +The reader successfully rejects both the old proposal center and the correct center with its uniform-coordinate selection omitted. +This distinction matters because omitting the selection changes the physical state represented by the same saved proposal coordinates. + +Full forecasts `22715569` and `22715571` are queued behind the completed-fit readers and successful adapter verification. +Independent forecast readers `22715570` and `22715572` follow them. +Comparison fixture `22715573` and final comparison `22715574` cover the original, default-centered and selective treatments, retaining the incumbent estimator as a separate reference. +The full comparison also waits for the earlier default-guide forecast readers, so unfinished treatments cannot be silently omitted. + +The comparison checks identical fitting budgets and forecast draw counts across all treatments, allowing only the declared refresh-probability difference in the sampler configuration. +It reports all ten weighted parameter marginals in prior-unit coordinates, including duplicate sample masses, alongside prediction errors, event probabilities, replica agreement and available computation costs. +Comparison fixture `22715573` completed in seven seconds. +Its synthetic weighted-distribution check and six deliberate invalid-budget, invalid-refresh, invalid-weight and out-of-prior cases pass. +It independently reproduces the original two fits' parameter summaries and retains all four new forecasts as pending. +An incomplete comparison fixture is expected while new forecasts are unavailable; full comparison requires all six verified forecasts. +These reports remain offline evidence and do not establish agent solve rates. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 5721abc0d..c16c48076 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -27,7 +27,11 @@ The comparison retains both earlier fits and the incumbent, including prediction The [parameter-dependence audit](balloons-selective-guidance.md) and independent reader `22715205`/`22715206` have completed all 46 native cases and twenty marginal checks. Changing five parameters separately or together leaves both tested fitting histories and scores exactly unchanged, while the original fits retain very narrow marginals for them. A selective guide with uniform proposals on those coordinates and 20% block refresh moves now passes native validation `22715307`, including 32 default-parity and 32 independent mixture-density checks. -Paired fits `22715405` are running with final-target readers `22715406`/`22715407` dependent on completion; weighted future validation for this selective proposal remains outstanding. +Paired fits `22715405` are running with final-target readers `22715406`/`22715407` dependent on completion. +The selective proposal's weighted future adapter and independent reader `22715541`/`22715542` now pass, including rejection of a decoder that omits the selective mask. +Forecasts `22715569`/`22715571`, independent readers and the six-population comparison `22715574` are dependency-queued. +The comparison fixture `22715573` passes its marginal and configuration guards, retaining the four pending forecasts explicitly. +The comparison preserves the original and default-centered controls and reports parameter spread alongside future predictions. It retains the complete prior, likelihood and Metropolis correction and does not assume global parameter independence. The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. From 388c6223e0590174c2d3e4ade5a761182cbc6c53 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 02:36:09 -0400 Subject: [PATCH 85/94] Integrate unobserved Boil heating parameters in weighted forecasts --- docs/uncertainty/boil-unobserved-heating.md | 108 ++++++++++++++++++++ docs/uncertainty/implementation-progress.md | 9 ++ 2 files changed, 117 insertions(+) create mode 100644 docs/uncertainty/boil-unobserved-heating.md diff --git a/docs/uncertainty/boil-unobserved-heating.md b/docs/uncertainty/boil-unobserved-heating.md new file mode 100644 index 000000000..07adaeb76 --- /dev/null +++ b/docs/uncertainty/boil-unobserved-heating.md @@ -0,0 +1,108 @@ +# Boil: unobserved heating parameters and conditional marginalization + +September 14, 2026. +This is an offline follow-up to the [supported Boil comparison](boil-supported-inference.md). +It separates numerical parameter concentration from the information available in the fitting prefix. +The production estimator is unchanged. + +## What the fitting prefix observes + +The 132-action fitting prefix never turns on the burner. +The burner state is an exact observation, verified against the frozen sensor contract. +The selected native continuations first turn it on at action 185. + +The frozen program initializes heat to zero and increments it only when the burner is on, the jug is near enough, the jug is not held, and sufficient water is present. +Burner radius is used only in that heating condition. +Heating onset and width are used only in the clipped readout of accumulated heat. +Their prior supports are positive, and the physical simulator histories are independent of these feature-only parameters. + +Consequently, for this program and any supported history of this all-burners-off prefix, the likelihood does not depend on these three parameters. +The original target uses independent uniform priors on burner radius `[0.04, 0.25]`, onset `[5, 80]`, and width `[1, 40]`. +Writing the remaining unknowns as `phi`, the target therefore factorizes as + +```text +p(phi, radius, onset, width | prefix) + = p(phi | prefix) p(radius) p(onset) p(width). +``` + +This statement depends on the exact observation contract, the frozen program, zero initial heat, and this particular prefix. +It does not apply after heating observations arrive or to an arbitrary simulator program. +The native transition and output-discrepancy factors on the remaining coordinates are unchanged. + +The incumbent's own fitting report identifies burner radius, onset and width as unconstrained and retains defaults `0.12`, `29.5` and `7.0`. +Its good future prediction does not show that this prefix identified those values. +The posterior replacement should retain the declared prior uncertainty rather than reproduce a default point through unexplained concentration. + +## Native reproduction and parameter interventions + +Bundle `logs/uncertainty_boil_heating_diagnostic_20260914` selects the highest-weight retained particle from each completed numerical fit, 410 and 411, and four archived future random draws per selected particle. +It keeps each physical history and all non-heating parameters fixed. +Profiles are the selected parameters, incumbent onset/width, incumbent onset/width/radius, and two boundary profiles for the three heating parameters. +These are forty fixed-history interventions, not new fits or agent seeds. + +Native diagnostic `22715829` completed in 3:55 with 792 simulator actions, exactly reproducing one complete source continuation per selected particle. +All eight archived histories reproduce the selected program's complete readouts and memory. +Every intervention preserves the entire fitting-prefix prediction, memory and likelihood exactly. +Independent reader `22715830` completed in five seconds, reconstructing 10,560 rule frames with literal arithmetic, checking 1,056 integrated future frames and rejecting altered parameters and predictions. +Its source checksum matches the completed diagnostic. + +The following RMSE is averaged over the four selected future draws per numerical seed. +All four draws give the same bubbling RMSE within each listed profile. + +| Numerical fitting seed | Selected onset/width | Incumbent onset/width | Uniform-prior onset/width integration | +|---|---:|---:|---:| +| 410 | 0.58826 | 0.01915 | 0.28497 | +| 411 | 0.49246 | 0.01915 | 0.28497 | + +Changing only onset and width recovers the incumbent's bubbling error on these selected histories, without changing their physical motion or fitting score. +This identifies the retained heating parameters as the source of the selected histories' bubbling error. +It does not show that motion uncertainty is harmless for every particle or future path. +The two-parameter integration in this diagnostic holds the selected radius fixed and is not the complete posterior predictive calculation. + +## Integrating all three heating parameters + +Bundle `logs/uncertainty_boil_heating_marginalization_20260914` applies the factorization to all 512 generated histories from the two completed 32-particle fits. +Every original positive particle weight and both four-draw forecast banks are preserved. +The approximate posterior marginal over all other parameters and scene coordinates is retained. +The calculation replaces the accidental fitted heating-coordinate values with their declared independent priors. + +For each fixed physical history, distances to the burner partition its radius prior into intervals with constant heating decisions. +A single radius interval determines the complete accumulated-heat trajectory, preserving the assumption of fixed parameters through time. +The clipped bubbling readout is integrated over onset analytically and over width by one-dimensional quadrature. +Interval probabilities then integrate burner radius. +Goal probabilities also retain the same history's filling, spilling and burner-off conditions. + +Independent verification uses radius order statistics at each future step, rather than the producer's interval enumeration, and a different clipped-ramp integral. +It additionally checks every stored full-trajectory heat-count sequence, the exact burner observation contract, zero initial heat, source weights and aggregate summaries. +The calculation produces means and event probabilities; it does not claim a new joint future-density estimate or refit the remaining unknowns. + +Integration job `22716011` completed in 57 seconds, and reader `22716012` completed in 1:24. +Both use compute nodes and require zero additional native simulator actions. +The reader checks all 512 histories, 67,584 future frames and 133 readout-integral values, and rejects a deliberately changed marginal mean. +The verified result checksum is `f918a4b451ec540a746feadc002b79ed6dd411ad4f4cfce525833ca3da87f9f5`. + +| Method | Numerical seed | Bubbling RMSE of weighted mean | Final goal probability | +|---|---|---:|---:| +| Original supported fit | 410 | 0.33856 | 0.5637 | +| Heating priors integrated | 410 | 0.28731 | 0.7097 | +| Original supported fit | 411 | 0.31636 | 0.7691 | +| Heating priors integrated | 411 | 0.29648 | 0.6849 | +| Incumbent retained point | N/A | 0.01915 | 1.0000 | + +The final-goal disagreement falls from 0.2054 to 0.0247 between the two approximate populations. +Both bubbling means improve, but remain worse than the incumbent's retained point on this recording. +The remaining difference cannot be diagnosed solely as simulator failure: this is also an extrapolation of heating behavior that the fitting prefix has not observed. +The retained posterior over scene and other parameters remains an approximation requiring numerical assessment. +No agent solve rate or Stage B acceptance is established here. + +## Implications for the simplification + +For a certified independent parameter block, retaining its analytical prior factor is simpler and more reliable than estimating that factor through resampling and limited random-walk moves. +A useful next numerical comparison can remove this block from the fitting coordinates and restore its prior when drawing complete parameter sets or integrating predictions. +That requires explicit factorization provenance and a result representation that does not mislabel fixed placeholder values as posterior samples. +The block must return to joint inference when new observations can constrain it. + +Keep the original 132-action extrapolation comparison as a prior-retention and prediction case. +A separate prefix after heating experience can test whether the replacement learns the heating response from informative data. +Neither setting should silently replace the other, and program defaults must not be introduced as a new prior merely to improve the observed future score. +The full five-domain numerical, predictive and live-agent acceptance work remains open. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index c16c48076..a173b4a0d 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -84,6 +84,15 @@ Both new fits remain worse than the incumbent on bubbling prediction, and their Supported initialization improves the initial population but has not established adequate final inference or prediction. The validation and forecast work uses a second matching-CPU compute node with exact prefix/replay checks; the frozen fitting inputs are unchanged. +The [Boil heating investigation](boil-unobserved-heating.md) now identifies an information limitation in that 132-action prefix: its exact burner observations are all off, so the frozen model cannot constrain burner radius, heating onset or heating width. +Forty verified fixed-history interventions preserve the full fitting score; changing only onset and width to the incumbent defaults removes the bubbling error on all eight selected future paths. +The incumbent itself reports these parameters as unconstrained and retains their defaults. +An explicit conditional-prior integration now covers all three heating parameters across all 512 weighted generated histories, with independent order-statistic and integral verification over 67,584 future frames. +It uses no additional native actions, improves bubbling RMSE from 0.3386/0.3164 to 0.2873/0.2965, and reduces between-fit final-goal disagreement from 0.2054 to 0.0247. +These moment calculations preserve the other fitted coordinates and weights; they neither establish full numerical adequacy nor supply a new future-density estimate. +This motivates an explicitly factorized inference representation and an additional informative-heating prefix, while retaining the original extrapolation case and incumbent default. + + The [Bridge causal future generator](bridge-causal-futures.md) now passes four complete native histories and an independent joint/reach/memory reader, totaling 4,744 actions. It preserves the checked 600-action prefix and generates the 586-action suffix without future observations, retaining one sampled variance per joint throughout each continuation. The [complete Bridge joint target](bridge-joint-inference.md) now composes the original parameter/scene priors, exact-rate/latch factors and corrected scene/parameter proposals. From 4c3242527370accf54bb9facf187b7bf42905cba Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 03:04:07 -0400 Subject: [PATCH 86/94] Represent independent prior factors and validate reduced Boil target --- docs/uncertainty/implementation-progress.md | 13 +- docs/uncertainty/independent-prior-factors.md | 74 ++++++ .../inference_factorization.py | 219 ++++++++++++++++++ .../test_inference_factorization.py | 197 ++++++++++++++++ 4 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 docs/uncertainty/independent-prior-factors.md create mode 100644 predicators/code_sim_learning/inference_factorization.py create mode 100644 tests/code_sim_learning/test_inference_factorization.py diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index a173b4a0d..bfb737b89 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -12,6 +12,15 @@ The active work is Stage B offline comparison; Stage C live posterior use and St Stage D execution smoothing remains optional and deferred. The full plan remains incomplete, and the incumbent estimator remains the production default. +The [independent-prior representation](independent-prior-factors.md) now supports an explicitly checked product of a sampled joint marginal and independent uniform prior factors. +Compute-node check `22717200` passes 27 focused tests, typing, repository lint and formatting checks. +The consumer preserves retained correlations, exact analytic quantiles, full-target provenance and unavailable-assessment behavior; production integration is unchanged. +The reduced Boil target removes the three unobserved heating coordinates while preserving the original full target score. +Native preflight `22717213` passes 56 assignments across eight source states, with 792 native actions and four contract guards. +Independent reader `22717214` also passes, with 264 fresh native actions and a rejected placeholder coordinate. +Two fresh small fitting fixtures are running; full fits `22717256` and `22717258` are dependency-queued behind their independent readers, with final-target verification also queued. +No full reduced fit or new agent result is available yet. + The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. At one selected fitted state, removing future joint and velocity noise eliminates bursts in four continuations, but does not recover the goal; the other selected state does not improve. This isolates a contribution from the physical discrepancy model without establishing a replacement posterior or an agent improvement. @@ -20,7 +29,9 @@ Sixteen verified conditional parameter profiles expose both a much better fittin An unchanged-target proposal audit verifies 52 proposal densities and motivates a matched guide-center-only comparison. Native validation `22714576` passes exact archived replay, future-data isolation, factorization and serial/parallel checks. Array `22714577` now runs the two original numerical seeds with only the parameter proposal center changed to the fixed program defaults; original priors, likelihood and sampling budgets remain unchanged. -Independent complete-checkpoint and final-target readers `22714628` and `22714629` are queued. +The default-centered seed 621 fit `22714577_1` and its reader `22714629` have completed; all 64 final native targets and the checkpoint reproduce, with one surviving original lineage. +This does not establish mixing or prediction quality. +The paired seed 620 fit remains running, with reader `22714628` dependent on completion. The new weighted forecast adapter and independent reader `22715007`/`22715008` have passed, checking both broad and local proposal mappings, complete native prefixes, generation and density histories, and rejection of the old guide when decoding new particles. Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` are dependency-queued behind the verified completed fits. The comparison retains both earlier fits and the incumbent, including prediction metrics and computation costs; its fixture `22715119` passes while correctly leaving the new outcomes pending. diff --git a/docs/uncertainty/independent-prior-factors.md b/docs/uncertainty/independent-prior-factors.md new file mode 100644 index 000000000..b82161a18 --- /dev/null +++ b/docs/uncertainty/independent-prior-factors.md @@ -0,0 +1,74 @@ +# Independent prior factors in the inference result + +September 14, 2026. +This extends the offline parameter-inference representation used by the [simplification proposal](simplification-proposal.md). +The production estimator remains unchanged. + +## Motivation and scope + +The [Boil heating diagnostic](boil-unobserved-heating.md) establishes that the pinned program's all-burners-off fitting prefix cannot constrain burner radius, heating onset or heating width. +A finite resampled population can nevertheless appear concentrated in those coordinates. +Representing their exact independent priors explicitly avoids that approximation error without choosing a narrower prior or using future observations to select parameter values. + +The result has the form `p(retained | data) * p(independent)`. +The retained block includes correlated parameters and uncertain initial-state coordinates, and stays a joint sampled distribution. +The independent block currently supports normalized uniform box priors in explicit physical coordinates. +Log-uniform laws, correlated factors, and parameters whose independence is only observed at a few fitted states are outside this interface's declared scope. + +## Representation and checks + +`inference_factorization.py` introduces a factorization declaration, a scoped evidence check, and a parameter consumer for the resulting product posterior. +The declaration identifies the full and reduced targets, the complete retained coordinate schema, and the independent prior law. +Both targets must refer to the same model program, fitting data and observation model. +The evidence scope changes when either identity, the retained schema, or the independent prior changes. +A matching digest binds the evidence to the declaration; it does not establish mathematical independence by itself. +The supporting argument must apply throughout the target's support. + +The reduced posterior must independently pass its declared numerical assessment, and the factorization check must pass before any parameter draws or intervals are available. +Even a request containing only independent coordinates cannot bypass an unavailable reduced posterior. +Predictive failures remain visible, with distinct full-target and reduced-target scopes, and do not silently erase a numerically adequate posterior. +None of these checks approves an action or publishes a production fit. + +The consumer returns exact uniform marginal quantiles for independent coordinates and weighted empirical quantiles for retained coordinates. +Joint draws resample whole retained rows according to their weights, then independently draw the analytic block. +Aliases of one coordinate share the same draw. +The resulting existing `ParameterEnsemble` carries equal Monte Carlo weights, the sampling seed, the original retained-particle indices, the full target identity and the scoped checks. +There is no finite exact `weighted_samples()` table for a continuous analytic factor; callers explicitly choose a resampling count and seed. +The existing sampled-only parameter consumer remains unchanged. + +## Reduced Boil experiment + +The target adapter in `logs/uncertainty_boil_reduced_target_20260914` removes proposal coordinates 79, 81 and 82 and output-joint coordinates 3, 5 and 6. +This leaves 81 jointly sampled coordinates, while retaining the original heating priors separately. +The frozen native evaluator uses midpoint values internally for the removed parameters because its unchanged fitting score is independent of them. +Those midpoints never appear in the reduced sampled joint and are not claimed as posterior samples. +The adapter retains the full original target score and all scene proposal corrections; each removed normalized uniform prior integrates to one. + +The structural argument is tied to the exact reviewed program, empty initial memory, strictly positive onset and width bounds, and exact burner-off observations throughout the 132-action prefix. +Changes to that program, noisy burner-state observations, nonzero initial memory, or a prefix containing burner activation require a new argument or restoration of joint inference. +Native preflight compares seven parameter assignments at each selected source state, including finite and rejected targets, and checks unchanged scores and projected coordinates. +An independent reader uses a separate coordinate mapping, verifies physical prior bounds, rejects a retained midpoint coordinate and repeats native histories. +These are mechanical checks, not numerical mixing or prediction acceptance. + +The fresh fitting bundle `logs/uncertainty_boil_reduced_fits_20260914` retains the original 32 particles, 64 geometric temperatures, eight moves, proposal scale 0.05, refresh probability 0.5 and 20,000-evaluation cap. +Removing the three independent singleton blocks leaves ten proposal blocks, with uniform selection among those remaining blocks. +Numerical seeds 410 and 411 are retained, but changing dimension changes the random streams. +No 84-dimensional checkpoint is resumed as an 81-dimensional fit. +Full fits depend on completed native preflight and a verified small fitting fixture for each seed. +Completed sampler artifacts remain unavailable as assessed posteriors until independent numerical replication and budget checks establish adequacy. +The original extrapolation comparison and incumbent results remain separate and preserved. + +## Acceptance still required + +Compute-node check `22717200` passes 27 focused tests, mypy, repository pytest-pylint checks, and pinned formatting checks. +Earlier check attempts exposed test typing and lint issues, which were corrected before this passing frozen run. +Native preflight `22717213` has completed: 56 parameter assignments at eight source states preserve the expected reduced target, with 792 native actions and four rejected changes to the factorization assumptions. +Independent reader `22717214` also passes all 56 assignments, rejects a retained placeholder coordinate and performs 264 fresh native actions. +Its verified source checksum is `7ec3f083e8a866ccad24db8242e53b6860593a73c60e010c82b5b38d7f6884de`. +Small fitting fixtures `22717216` and `22717220` are running, with readers `22717219` and `22717221` dependent on completion. +Full fits `22717256` and `22717258` are queued behind the corresponding successful fixture readers, with full-fit readers `22717257` and `22717259` dependent on completion. +All jobs use compute nodes on `mit_preemptable`. +The Boil adapter and each fitting fixture require independent verification before full fitting. +Future comparisons must restore the full heating priors when drawing parameter sets or integrate them explicitly, while preserving correlations and weights in the retained block. +An informative prefix containing heating remains a separate learning test; it does not replace the all-off extrapolation case. +Stable five-domain inference, live planning integration and final non-regression evidence remain required before retiring existing uncertainty mechanisms. diff --git a/predicators/code_sim_learning/inference_factorization.py b/predicators/code_sim_learning/inference_factorization.py new file mode 100644 index 000000000..d8a0f0f98 --- /dev/null +++ b/predicators/code_sim_learning/inference_factorization.py @@ -0,0 +1,219 @@ +"""Parameter consumers for a sampled marginal times an analytic prior factor. + +Factorization is an explicit, checked model/data claim. Flat numerical +slices do not establish it. This module neither discovers independence +nor fits, publishes, or silently changes a posterior's prior. +""" +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass, replace +from typing import Dict, Tuple + +import numpy as np + +from predicators.code_sim_learning.inference_assessment import \ + AssessedInference, InferenceCheck, validated_posterior +from predicators.code_sim_learning.inference_data import InferenceIdentity, \ + content_digest +from predicators.code_sim_learning.inference_parameters import \ + ParameterEnsemble, UnavailableParameterPosterior +from predicators.code_sim_learning.inference_sampling import BatchPosterior, \ + BoxPrior + + +@dataclass(frozen=True) +class IndependentPriorFactorization: + """Declare p(full | data) = p(retained | data) times a fixed box prior. + + Full and reduced targets must share data, sensor and program. Their + prior and runtime identities can differ because the reduced target + integrates out coordinates. The declaration identifies the complete + retained schema, including episode state, and an independent uniform + factor in explicit physical coordinates. It is not a proof. + """ + + full_identity: InferenceIdentity + reduced_identity: InferenceIdentity + retained_coordinates: Tuple[str, ...] + independent_prior: BoxPrior + + def __post_init__(self) -> None: + coordinates = tuple(self.retained_coordinates) + if not coordinates or len(set(coordinates)) != len(coordinates) or \ + any(not isinstance(name, str) or not name + for name in coordinates): + raise ValueError("Retained coordinates must be distinct names") + if set(coordinates).intersection(self.independent_prior.names): + raise ValueError("Independent and retained coordinates overlap") + for name in ("data", "sensor", "program"): + if getattr(self.full_identity, name) != \ + getattr(self.reduced_identity, name): + raise ValueError( + "Factorization changes data, sensor or program") + object.__setattr__(self, "retained_coordinates", coordinates) + + @property + def digest(self) -> str: + """Bind evidence to the identities, complete schema and factor law.""" + return content_digest( + json.dumps({ + "schema": 1, + "factorization": asdict(self) + }, + sort_keys=True).encode("utf-8")) + + +@dataclass(frozen=True) +class CheckedFactorization: + """An identified independence check for one exact declaration. + + The evidence must establish the factorization over the target's + support, not just at selected fitted states. A new data prefix or + model requires a new declaration and check. The caller is + responsible for that evidence; a digest match alone does not prove + independence. + """ + + declaration: IndependentPriorFactorization + checked_declaration: str + check: InferenceCheck + + def __post_init__(self) -> None: + if self.checked_declaration != self.declaration.digest: + raise ValueError("Factorization evidence has a different scope") + + +@dataclass(frozen=True) +class FactorizedParameterPosterior: + """Project one product posterior into parameter intervals and draws. + + The reduced numerical assessment and factorization check must both + permit use. Analytic factors never turn an unavailable sampled + marginal into an available full posterior. Predictive diagnostics + retain their full or reduced target scope and do not approve + actions. + """ + + reduced: AssessedInference + factorization: CheckedFactorization + names: Tuple[str, ...] + coordinates: Tuple[str, ...] = () + predictive_checks: Tuple[InferenceCheck, ...] = () + + def __post_init__(self) -> None: + names = tuple(self.names) + coordinates = tuple(self.coordinates) if self.coordinates else names + if len(set(names)) != len(names) or any( + not isinstance(name, str) or not name for name in names): + raise ValueError("Parameter names must be distinct strings") + if len(coordinates) != len(names) or any( + not isinstance(name, str) or not name for name in coordinates): + raise ValueError("One joint coordinate per parameter required") + declaration = self.factorization.declaration + available = set(declaration.retained_coordinates) | \ + set(declaration.independent_prior.names) + if not set(coordinates) <= available: + raise ValueError("Unknown full posterior coordinate") + checks = tuple(self.predictive_checks) + if len({check.name for check in checks}) != len(checks): + raise ValueError("Duplicate full posterior predictive check") + object.__setattr__(self, "names", names) + object.__setattr__(self, "coordinates", coordinates) + object.__setattr__(self, "predictive_checks", checks) + self._validate_source() + + def _validate_source(self) -> BatchPosterior | None: + checked = self.factorization + declaration = checked.declaration + if checked.checked_declaration != declaration.digest or \ + self.reduced.identity != declaration.reduced_identity: + raise ValueError("Factorization and reduced source disagree") + posterior = validated_posterior(self.reduced) + if posterior is not None and \ + posterior.prior.names != declaration.retained_coordinates: + raise ValueError("Reduced posterior coordinate schema differs") + return posterior + + def _posterior(self) -> BatchPosterior: + posterior = self._validate_source() + if posterior is None: + raise UnavailableParameterPosterior("Reduced posterior is " + + self.reduced.availability) + if self.factorization.check.status != "pass": + raise UnavailableParameterPosterior( + "Factorization is " + self.factorization.check.status) + return posterior + + @property + def identity(self) -> InferenceIdentity: + """Identify the full target, separately from its sampled marginal.""" + return self.factorization.declaration.full_identity + + def marginal_quantiles( + self, probabilities: Tuple[float, ...] = (.05, .5, .95) + ) -> Dict[str, Tuple[float, ...]]: + """Keep analytic quantiles exact and sampled quantiles weighted.""" + posterior = self._posterior() + if not probabilities or any(not math.isfinite(p) or not 0 <= p <= 1 + for p in probabilities): + raise ValueError("Quantile probabilities must lie in [0, 1]") + prior = self.factorization.declaration.independent_prior + bounds = dict(zip(prior.names, prior.bounds)) + result = {} + for name, coordinate in zip(self.names, self.coordinates): + if coordinate in bounds: + lo, hi = bounds[coordinate] + result[name] = tuple(lo if p == 0 else hi if p == 1 else lo + + p * (hi - lo) for p in probabilities) + else: + result[name] = posterior.marginal_quantiles( + coordinate, probabilities) + return result + + def resample(self, count: int, seed: int) -> ParameterEnsemble: + """Draw retained rows jointly and the certified factor independently. + + A repeated source coordinate, including an analytic coordinate, + shares one value across its aliases. Output rows have equal + Monte Carlo weights and retain their sampled marginal's original + particle indices. This does not create new fitting evidence. + """ + if not isinstance(count, int) or isinstance(count, bool) or count <= 0: + raise ValueError("Resampling count must be a positive integer") + if not isinstance(seed, int) or isinstance(seed, bool) or seed < 0: + raise ValueError("Resampling seed must be a nonnegative integer") + posterior = self._posterior() + rng = np.random.default_rng(seed) + weights = np.asarray(posterior.weights, dtype=float) + weights /= math.fsum(posterior.weights) + indices = tuple( + int(i) for i in rng.choice(len(weights), size=count, p=weights)) + prior = self.factorization.declaration.independent_prior + lower, upper = np.asarray(prior.bounds).T + independent = rng.uniform(lower, upper, size=(count, len(prior.names))) + rows = [] + for index, draw in zip(indices, independent): + values = dict(zip(posterior.prior.names, posterior.samples[index])) + values.update(zip(prior.names, draw)) + rows.append(tuple( + float(values[name]) for name in self.coordinates)) + protocol = content_digest( + json.dumps( + { + "schema": 1, + "reduced_protocol": self.reduced.protocol.source, + "factorization": self.factorization.checked_declaration, + "check": asdict(self.factorization.check) + }, + sort_keys=True).encode("utf-8")) + checks = tuple( + replace(check, name="reduced/" + check.name) + for check in self.reduced.predictive_checks) + tuple( + replace(check, name="full/" + check.name) + for check in self.predictive_checks) + return ParameterEnsemble(self.identity, protocol, + checks, self.names, self.coordinates, + tuple(rows), (1. / count, ) * count, indices, + seed) diff --git a/tests/code_sim_learning/test_inference_factorization.py b/tests/code_sim_learning/test_inference_factorization.py new file mode 100644 index 000000000..a3686021d --- /dev/null +++ b/tests/code_sim_learning/test_inference_factorization.py @@ -0,0 +1,197 @@ +"""Checked product posteriors preserve analytic factors and sampled joints.""" +from dataclasses import replace + +import numpy as np +import pytest + +from predicators.code_sim_learning.inference_assessment import \ + AssessedInference, AssessmentProtocol, InferenceCheck, assess_inference +from predicators.code_sim_learning.inference_data import InferenceIdentity, \ + content_digest +from predicators.code_sim_learning.inference_factorization import \ + CheckedFactorization, FactorizedParameterPosterior, \ + IndependentPriorFactorization +from predicators.code_sim_learning.inference_parameters import \ + UnavailableParameterPosterior +from predicators.code_sim_learning.inference_sampling import BatchPosterior, \ + BoxPrior, SamplerConfig + + +def _source() -> AssessedInference: + # Known discrete joint: b=a*a, with one zero-mass outlier. + prior = BoxPrior(("a", "start", "b"), + ((-10., 10.), (-100., 100.), (0., 100.))) + digest = content_digest(b"enumerated joint") + identity = InferenceIdentity(digest, digest, digest, prior.digest, digest) + posterior = BatchPosterior(identity, prior, SamplerConfig(particles=4), 0, + "complete", ((-9., 80., 81.), (-2., 3., 4.), + (1., 4., 1.), (3., 5., 9.)), + (0., .1, .3, .6), 4, 1., 3, (2., ), 0, 0, 3, 0) + protocol = AssessmentProtocol(digest, ("enumeration", )) + checks = (InferenceCheck("enumeration", "pass", "Exact reference", + digest), ) + predictive = (InferenceCheck("future", "fail", "Missing dynamics", + digest), ) + return assess_inference(posterior, protocol, checks, predictive) + + +def _result() -> FactorizedParameterPosterior: + source = _source() + full = replace(source.identity, prior=content_digest(b"product prior")) + declaration = IndependentPriorFactorization( + full, source.identity, ("a", "start", "b"), + BoxPrior(("radius", "heat"), ((.04, .25), (5., 80.)))) + check = InferenceCheck("independence", "pass", "Exact product reference", + content_digest(b"factor proof")) + factor = CheckedFactorization(declaration, declaration.digest, check) + predictive = (InferenceCheck("future", "fail", "Prior extrapolation", + content_digest(b"full prediction")), ) + return FactorizedParameterPosterior(source, + factor, ("a", "b", "radius", "heat"), + predictive_checks=predictive) + + +def test_product_forecasts_preserve_dependence_and_prior_law() -> None: + """A known nonlinear joint times two uniforms yields reference moments.""" + result = _result() + ensemble = result.resample(20000, 17) + rows = np.asarray(ensemble.values) + assert ensemble == result.resample(20000, 17) + assert ensemble != result.resample(20000, 18) + assert set(ensemble.source_indices) == {1, 2, 3} + assert ensemble.weights == (1 / 20000, ) * 20000 + assert ensemble.resampling_seed == 17 + assert np.all(rows[:, 1] == rows[:, 0]**2) + assert np.mean(rows[:, 0]) == pytest.approx(1.9, abs=.04) + assert np.mean(rows[:, 2]) == pytest.approx(.145, abs=.002) + assert np.mean(rows[:, 3]) == pytest.approx(42.5, abs=.5) + assert np.all((rows[:, 2] >= .04) & (rows[:, 2] <= .25)) + assert np.all((rows[:, 3] >= 5.) & (rows[:, 3] <= 80.)) + # Joint event separates retained mass from both independent factors. + event = (rows[:, 0] == 3.) & (rows[:, 2] < .145) & (rows[:, 3] < 42.5) + assert ensemble.expectation(event) == pytest.approx(.15, abs=.01) + assert ensemble.identity == result.identity + assert ensemble.identity != result.reduced.identity + assert [(check.name, check.status) for check in ensemble.predictive_checks + ] == [("reduced/future", "fail"), ("full/future", "fail")] + assert ensemble.assessment_protocol != result.reduced.protocol.source + maps = ensemble.as_dicts() + maps[0]["heat"] = -100. + assert ensemble.as_dicts()[0]["heat"] >= 5. + + +def test_exact_quantiles_and_coordinate_aliases() -> None: + """Analytic intervals do not inherit accidental particle concentration.""" + result = _result() + quantiles = result.marginal_quantiles((0., .5, 1.)) + assert quantiles == { + "a": (-2., 3., 3.), + "b": (1., 9., 9.), + "radius": (.04, .145, .25), + "heat": (5., 42.5, 80.) + } + aliases = replace(result, + names=("left", "right", "x", "y"), + coordinates=("heat", "heat", "a", "a")) + assert all(left == right and x == y + for left, right, x, y in aliases.resample(100, 9).values) + empty = replace(result, names=(), coordinates=()) + assert not empty.marginal_quantiles() + assert empty.resample(2, 1).as_dicts() == ({}, {}) + + +@pytest.mark.parametrize("status", ["numerical_failure", "unevaluated"]) +def test_analytic_factor_cannot_bypass_unavailable_fit(status) -> None: + """Even analytic-only requests require an available whole posterior.""" + result = _result() + unavailable = replace(result.reduced, availability=status, posterior=None) + result = replace(result, + reduced=unavailable, + names=("heat", ), + coordinates=("heat", )) + with pytest.raises(UnavailableParameterPosterior, match=status): + result.marginal_quantiles() + with pytest.raises(UnavailableParameterPosterior, match=status): + result.resample(2, 1) + assert result.reduced.predictive_checks[0].status == "fail" + + +@pytest.mark.parametrize("status", ["fail", "unevaluated"]) +def test_unverified_factor_cannot_supply_parameters(status) -> None: + """An independence declaration alone is insufficient for use.""" + result = _result() + check = replace(result.factorization.check, status=status) + result = replace(result, + factorization=replace(result.factorization, check=check)) + with pytest.raises(UnavailableParameterPosterior, match=status): + result.marginal_quantiles() + with pytest.raises(UnavailableParameterPosterior, match=status): + result.resample(2, 1) + + +def test_factor_evidence_cannot_follow_changed_target_or_law() -> None: + """New observations, program, prior law or schema invalidate old checks.""" + result = _result() + factor = result.factorization + declaration = factor.declaration + changed = content_digest(b"new data or program") + declarations = [ + replace(declaration, retained_coordinates=("b", "start", "a")), + replace(declaration, + independent_prior=BoxPrior(("radius", "heat"), + ((.04, .3), (5., 80.)))) + ] + for field in ("data", "sensor", "program"): + with pytest.raises(ValueError, match="changes data"): + replace(declaration, + full_identity=replace(declaration.full_identity, + **{field: changed})) + declarations.append( + replace(declaration, + full_identity=replace(declaration.full_identity, + **{field: changed}), + reduced_identity=replace(declaration.reduced_identity, + **{field: changed}))) + for revised in declarations: + with pytest.raises(ValueError, match="different scope"): + replace(factor, declaration=revised) + wrong_schema = declarations[0] + checked = replace(factor, + declaration=wrong_schema, + checked_declaration=wrong_schema.digest) + with pytest.raises(ValueError, match="schema differs"): + replace(result, factorization=checked) + new_data = declarations[-1] + checked = replace(factor, + declaration=new_data, + checked_declaration=new_data.digest) + with pytest.raises(ValueError, match="source disagree"): + replace(result, factorization=checked) + + +def test_invalid_sources_and_requests_reject() -> None: + """Malformed or falsely available results never reach a consumer.""" + result = _result() + with pytest.raises(ValueError, match="needs a posterior"): + replace(result, reduced=replace(result.reduced, posterior=None)) + with pytest.raises(ValueError, match="available posterior"): + replace(result, reduced=replace(result.reduced, numerical_checks=())) + for coordinates in (("missing", ) * 4, ("a", )): + with pytest.raises(ValueError): + replace(result, coordinates=coordinates) + with pytest.raises(ValueError, match="distinct"): + replace(result, names=("a", ) * 4) + with pytest.raises(ValueError, match="Duplicate"): + replace(result, predictive_checks=result.predictive_checks * 2) + declaration = result.factorization.declaration + with pytest.raises(ValueError, match="overlap"): + replace(declaration, independent_prior=BoxPrior(("a", ), ((0., 1.), ))) + for coordinates in ((), ("a", "a"), ("", )): + with pytest.raises(ValueError, match="distinct"): + replace(declaration, retained_coordinates=coordinates) + for count, seed in ((0, 1), (True, 1), (1, -1), (1, True)): + with pytest.raises(ValueError): + result.resample(count, seed) + for probabilities in ((), (-.1, ), (1.1, ), (float("nan"), )): + with pytest.raises(ValueError): + result.marginal_quantiles(probabilities) From d1609861f9bd085e8dc602379a2673873460ed45 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 03:20:21 -0400 Subject: [PATCH 87/94] Validate reduced Boil forecasts with independent heating priors --- docs/uncertainty/implementation-progress.md | 12 +++- docs/uncertainty/independent-prior-factors.md | 5 +- docs/uncertainty/reduced-boil-forecasts.md | 71 +++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 docs/uncertainty/reduced-boil-forecasts.md diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index bfb737b89..2dfbb0402 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -18,7 +18,14 @@ The consumer preserves retained correlations, exact analytic quantiles, full-tar The reduced Boil target removes the three unobserved heating coordinates while preserving the original full target score. Native preflight `22717213` passes 56 assignments across eight source states, with 792 native actions and four contract guards. Independent reader `22717214` also passes, with 264 fresh native actions and a rejected placeholder coordinate. -Two fresh small fitting fixtures are running; full fits `22717256` and `22717258` are dependency-queued behind their independent readers, with final-target verification also queued. +Both small fitting fixtures and readers now pass, including 4,224 fresh native actions per reader. +Full fits `22717256` and `22717258` are running with final-target verification queued. +The [reduced Boil forecast adapter](reduced-boil-forecasts.md) restores independently sampled thermal priors per complete trajectory while preserving retained joint rows and weights. +Its twelve source guards, ten malformed-history checks and nonuniform density/variance reference pass in `22717364`. +Native forecast fixture `22717373` completed 128 histories and 40,656 native actions. +Independent reader `22717375` also passes, checking 304,128 joint factors and using 6,864 additional native actions, including two fresh complete histories. +Both full forecasts, their independent readers and the final comparison are dependency-queued behind successful fit and fixture verification. +Comparison fixture `22717399` preserves all seven earlier controls while explicitly retaining both pending new outcomes. No full reduced fit or new agent result is available yet. The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. @@ -31,7 +38,8 @@ Native validation `22714576` passes exact archived replay, future-data isolation Array `22714577` now runs the two original numerical seeds with only the parameter proposal center changed to the fixed program defaults; original priors, likelihood and sampling budgets remain unchanged. The default-centered seed 621 fit `22714577_1` and its reader `22714629` have completed; all 64 final native targets and the checkpoint reproduce, with one surviving original lineage. This does not establish mixing or prediction quality. -The paired seed 620 fit remains running, with reader `22714628` dependent on completion. +The paired seed 620 fit `22714577_0` and reader `22714628` have also completed, with the same final-target reproducibility and one surviving original lineage. +Both default-centered posterior forecasts remain pending or running; their comparison is incomplete. The new weighted forecast adapter and independent reader `22715007`/`22715008` have passed, checking both broad and local proposal mappings, complete native prefixes, generation and density histories, and rejection of the old guide when decoding new particles. Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` are dependency-queued behind the verified completed fits. The comparison retains both earlier fits and the incumbent, including prediction metrics and computation costs; its fixture `22715119` passes while correctly leaving the new outcomes pending. diff --git a/docs/uncertainty/independent-prior-factors.md b/docs/uncertainty/independent-prior-factors.md index b82161a18..a82b062f6 100644 --- a/docs/uncertainty/independent-prior-factors.md +++ b/docs/uncertainty/independent-prior-factors.md @@ -65,8 +65,9 @@ Earlier check attempts exposed test typing and lint issues, which were corrected Native preflight `22717213` has completed: 56 parameter assignments at eight source states preserve the expected reduced target, with 792 native actions and four rejected changes to the factorization assumptions. Independent reader `22717214` also passes all 56 assignments, rejects a retained placeholder coordinate and performs 264 fresh native actions. Its verified source checksum is `7ec3f083e8a866ccad24db8242e53b6860593a73c60e010c82b5b38d7f6884de`. -Small fitting fixtures `22717216` and `22717220` are running, with readers `22717219` and `22717221` dependent on completion. -Full fits `22717256` and `22717258` are queued behind the corresponding successful fixture readers, with full-fit readers `22717257` and `22717259` dependent on completion. +Small fitting fixtures `22717216` and `22717220` and readers `22717219` and `22717221` have completed successfully. +Full fits `22717256` and `22717258` are now running, with full-fit readers `22717257` and `22717259` dependent on completion. +The subsequent [forecast validation](reduced-boil-forecasts.md) restores the three thermal priors explicitly. All jobs use compute nodes on `mit_preemptable`. The Boil adapter and each fitting fixture require independent verification before full fitting. Future comparisons must restore the full heating priors when drawing parameter sets or integrate them explicitly, while preserving correlations and weights in the retained block. diff --git a/docs/uncertainty/reduced-boil-forecasts.md b/docs/uncertainty/reduced-boil-forecasts.md new file mode 100644 index 000000000..c4379cc18 --- /dev/null +++ b/docs/uncertainty/reduced-boil-forecasts.md @@ -0,0 +1,71 @@ +# Reduced Boil inference and forecasts with independent heating priors + +September 14, 2026. +This follows the [independent-prior representation](independent-prior-factors.md) and its exact reduction of the all-burners-off Boil fitting target. +The work remains an offline Stage B comparison; the production estimator is unchanged. + +## Fitting validation + +Both 81-coordinate fitting fixtures completed successfully, using 219 and 180 target evaluations for numerical seeds 410 and 411. +Independent readers `22717219` and `22717221` replay the complete numerical ledgers and checkpoints and freshly verify all 32 final native targets per fixture, using 4,224 native actions each. +The verified source checksums are `8490860cad3bb895298296e523cb171ac3f737b56d931679107306bb322fc23f` and `e4b1360238f326cb6c990ceab3f3ac1c79d665a770da0d6739310b0ad6c826b7`. +These deliberately small two-temperature fixtures establish implementation consistency, not posterior adequacy. + +Full fits `22717256` and `22717258` are running on compute node `node1411` in `mit_preemptable`. +They retain the original 32 particles, 64 temperatures, eight moves and 20,000-evaluation cap, with the three independent singleton blocks removed. +Their independent readers `22717257` and `22717259` are dependency-queued. +The reports retain an unavailable numerical assessment until replication and budget-stability requirements are met. + +## Restoring the independent priors in forecasts + +The frozen adapter in `logs/uncertainty_boil_reduced_forecasts_20260914` first reconstructs each complete reduced checkpoint without performing any additional fitting. +It verifies the expected target, configuration, seed, complete retained coordinate schema, factorization declaration and matching fit reader. +Each positive-weight retained joint row keeps its original weight and correlations. + +For each generated trajectory, the adapter draws burner radius, onset and width independently from their original normalized uniform priors. +A separate deterministic random seed identifies that thermal draw. +The resulting complete parameter vector stays fixed throughout the trajectory, including its fitting prefix and future continuation. +The remaining parameter and scene coordinates are copied from one retained joint row. +No internal midpoint used during reduced fitting is interpreted as an inferred heating parameter. +The original physical continuation law and future simulator random-seed convention are retained. + +Full forecasts use two generation banks with four continuations per positive-weight particle in each bank. +Each continuation therefore includes both the retained uncertainty and a thermal prior draw. +Output means, total variances and event probabilities include their combined variability. +Generation does not receive reserved future observations. + +Future-data density is assessed separately, with eight independent thermal prior draws per retained particle. +The adapter averages densities over those draws before mixing particles according to their original weights. +Zero-density contributions stay in the denominator. +The reported log mixture is the logarithm of a Monte Carlo mean density; it is neither an exact integral nor an unbiased log-density estimate. +A particle whose sampled density draws are all zero is not thereby proved to have zero density under its full continuous prior. +The fixture uses two density draws per particle and one continuation per generation bank solely to test the implementation. + +## Independent checks + +Compute-node guard job `22717364` passes twelve source-corruption checks and ten malformed-history checks. +These include changed factor bounds, stale factorization scope, incomplete checkpoints, altered weights, missing or duplicated density draws, and negative conditional variances. +A separate nonuniform-weight reference verifies the total-variance calculation and averaging denominator when three of four prior-density draws contribute zero. + +Native forecast fixture `22717373` completed all 128 histories, using 40,656 native simulator actions. +Independent reader `22717375` also passes: all 128 histories, 304,128 joint factors, independent thermal-prior density and weighted-summary checks, and two fresh complete histories. +It uses 6,864 additional native actions. +The verified forecast source checksum is `bbbb8324640aaf3a348a6888e4e21e6de44a539c8698477b2e489c023192bd35`. +The reader reconstructs the 81-to-84 coordinate mapping separately, regenerates thermal draws from their recorded seeds, checks every native history and readout, and repeats complete generation and density histories. +It independently computes the thermal density average and weighted prediction summaries. +Full forecasts `22717437` and `22717439` are dependency-queued behind the successful fixture reader and the corresponding full-fit reader. +Their independent readers are `22717438` and `22717440`, followed by final comparison `22717441`. +These gates precede full forecast execution. + +## Comparison and remaining acceptance + +The comparison bundle `logs/uncertainty_boil_reduced_comparison_20260914` retains all existing Boil comparison rows, including the original full supported fits and the incumbent. +New reduced-target rows must match the original program, fitting observations and sensor model, with the declared full prior preserved by factorization. +It checks that the only fitting configuration difference from the full supported fits is removal and remapping of the three singleton proposal blocks. +The dimension change also changes random streams and is not described as identical stochastic execution. + +The comparison reports fitting and forecast costs, between-fit prediction differences, within-forecast bank differences, and the explicitly approximate density calculation. +Comparison fixture `22717399` completed its checks and retained all seven original rows while correctly reporting both new full comparisons as pending. +Missing results remain pending and cannot support an improvement claim. +A completed mechanical comparison would still require numerical and predictive adequacy before live planning integration or retirement of the incumbent. +A separate fitting prefix containing heating remains necessary to test learning from informative data; it will not replace the original all-off extrapolation comparison. From fc0d905a00bcd0b9b1bf2e99da8c4ac0aa77f619 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:39:51 -0400 Subject: [PATCH 88/94] Finalize noisy five-domain sweep results --- .../make_noisy_sweep_table.py | 4 +- .../noisy-sweep-snapshot.json | 204 +++++++++--------- .../noisy-sweep-summary.tsv | 10 +- docs/uncertainty-results/noisy-sweep-table.md | 38 ++-- 4 files changed, 130 insertions(+), 126 deletions(-) diff --git a/docs/uncertainty-results/make_noisy_sweep_table.py b/docs/uncertainty-results/make_noisy_sweep_table.py index 96dceda48..c60862ea6 100644 --- a/docs/uncertainty-results/make_noisy_sweep_table.py +++ b/docs/uncertainty-results/make_noisy_sweep_table.py @@ -183,7 +183,9 @@ def refresh(manifest_path=MANIFEST): lines = [ "", "# MB versus MF: five-domain noisy sweep", "", f"Updated {stamp}.", "", - "The result watcher refreshes this table every two minutes while the sweep is active.", + "This table is regenerated from verified scorecards; the periodic watcher is disabled.", + "This is the original cohort, before the Bridge observation-boundary fixes and the Balloons sustained-hover rule.", + "The corrected pilots are reported separately in [Bridge/Balloons integrity results](../comparisons/bridge-balloons-integrity-results.md).", "Each domain and arm has three planned seeds; aggregates remain provisional until all three finish.", "Whole-run solve rate means winning every training and test level.", "Both solve rates and mean resets use finished agent runs, with their count shown.", diff --git a/docs/uncertainty-results/noisy-sweep-snapshot.json b/docs/uncertainty-results/noisy-sweep-snapshot.json index ee20dc245..a84ff45e1 100644 --- a/docs/uncertainty-results/noisy-sweep-snapshot.json +++ b/docs/uncertainty-results/noisy-sweep-snapshot.json @@ -1,6 +1,6 @@ { "generated_by": "make_noisy_sweep_table.py; do not edit manually", - "updated_at": "2026-09-11T11:39:41.036197+00:00", + "updated_at": "2026-09-14T19:39:34.906642+00:00", "manifest": "/home/ycliang/predicators/logs/noisy_sweep_20260910/launch-manifest.json", "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", "rows": [ @@ -60,17 +60,17 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456627_0", + "job": "22625667_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 2408, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Bridge", @@ -101,17 +101,17 @@ "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456627_2", + "job": "22625597_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 1988, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Fan", @@ -186,17 +186,17 @@ "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456628_1", + "job": "22625598_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 842, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Fan", @@ -271,17 +271,17 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456629_0", + "job": "22625599_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 1, "levels": 2, - "steps": 0, + "steps": 712, "resets": 0, "success": false, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json", - "end_reason": null + "end_reason": "agent_ended" }, { "domain": "Domino", @@ -305,17 +305,17 @@ "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456629_2", + "job": "22639804_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 1, "levels": 2, - "steps": 0, + "steps": 821, "resets": 0, "success": false, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json", - "end_reason": null + "end_reason": "level_lost" }, { "domain": "Boil", @@ -373,51 +373,51 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_0", + "job": "22639805_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 1, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 593, + "steps": 738, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Boil", "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_1", + "job": "22639805_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 1076, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Boil", "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_2", + "job": "22625600_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 1, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 570, + "steps": 988, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", @@ -475,51 +475,51 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_0", + "job": "22639807_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 441, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_1", + "job": "22625601_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 476, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_2", + "job": "22625601_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 334, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" } ], "aggregate": [ @@ -538,13 +538,13 @@ { "domain": "Bridge", "arm": "MF", - "finished": 1, + "finished": 3, "expected": 3, - "successful": 1, + "successful": 3, "whole_run_solve_pct": 100.0, "level_solve_pct": 100.0, - "mean_steps_successful": 2413, - "steps_n": 1, + "mean_steps_successful": 2269.6666666666665, + "steps_n": 3, "mean_resets_finished": 0 }, { @@ -562,13 +562,13 @@ { "domain": "Fan", "arm": "MF", - "finished": 2, + "finished": 3, "expected": 3, - "successful": 2, + "successful": 3, "whole_run_solve_pct": 100.0, "level_solve_pct": 100.0, - "mean_steps_successful": 901, - "steps_n": 2, + "mean_steps_successful": 881.3333333333334, + "steps_n": 3, "mean_resets_finished": 0 }, { @@ -586,7 +586,7 @@ { "domain": "Domino", "arm": "MF", - "finished": 1, + "finished": 3, "expected": 3, "successful": 0, "whole_run_solve_pct": 0.0, @@ -610,14 +610,14 @@ { "domain": "Boil", "arm": "MF", - "finished": 0, + "finished": 3, "expected": 3, - "successful": 0, - "whole_run_solve_pct": null, - "level_solve_pct": null, - "mean_steps_successful": null, - "steps_n": 0, - "mean_resets_finished": null + "successful": 3, + "whole_run_solve_pct": 100.0, + "level_solve_pct": 100.0, + "mean_steps_successful": 934, + "steps_n": 3, + "mean_resets_finished": 0 }, { "domain": "Balloons", @@ -634,14 +634,14 @@ { "domain": "Balloons", "arm": "MF", - "finished": 0, + "finished": 3, "expected": 3, - "successful": 0, - "whole_run_solve_pct": null, - "level_solve_pct": null, - "mean_steps_successful": null, - "steps_n": 0, - "mean_resets_finished": null + "successful": 3, + "whole_run_solve_pct": 100.0, + "level_solve_pct": 100.0, + "mean_steps_successful": 417, + "steps_n": 3, + "mean_resets_finished": 1 } ] } diff --git a/docs/uncertainty-results/noisy-sweep-summary.tsv b/docs/uncertainty-results/noisy-sweep-summary.tsv index 50a5c5393..31839adcf 100644 --- a/docs/uncertainty-results/noisy-sweep-summary.tsv +++ b/docs/uncertainty-results/noisy-sweep-summary.tsv @@ -1,11 +1,11 @@ domain arm finished expected successful whole_run_solve_pct level_solve_pct mean_steps_successful steps_n mean_resets_finished Bridge MB 3 3 3 100.0 100.0 2214 3 0 -Bridge MF 1 3 1 100.0 100.0 2413 1 0 +Bridge MF 3 3 3 100.0 100.0 2269.6666666666665 3 0 Fan MB 3 3 3 100.0 100.0 425 3 0 -Fan MF 2 3 2 100.0 100.0 901 2 0 +Fan MF 3 3 3 100.0 100.0 881.3333333333334 3 0 Domino MB 3 3 3 100.0 100.0 404.6666666666667 3 0 -Domino MF 1 3 0 0.0 50.0 NA 0 0 +Domino MF 3 3 0 0.0 50.0 NA 0 0 Boil MB 3 3 3 100.0 100.0 668.3333333333334 3 0 -Boil MF 0 3 0 NA NA NA 0 NA +Boil MF 3 3 3 100.0 100.0 934 3 0 Balloons MB 3 3 3 100.0 100.0 365.6666666666667 3 0.3333333333333333 -Balloons MF 0 3 0 NA NA NA 0 NA +Balloons MF 3 3 3 100.0 100.0 417 3 1 diff --git a/docs/uncertainty-results/noisy-sweep-table.md b/docs/uncertainty-results/noisy-sweep-table.md index 82fd7a237..e41829624 100644 --- a/docs/uncertainty-results/noisy-sweep-table.md +++ b/docs/uncertainty-results/noisy-sweep-table.md @@ -1,9 +1,11 @@ # MB versus MF: five-domain noisy sweep -Updated 2026-09-11T11:39:41.036197+00:00. +Updated 2026-09-14T19:39:34.906642+00:00. -The result watcher refreshes this table every two minutes while the sweep is active. +This table is regenerated from verified scorecards; the periodic watcher is disabled. +This is the original cohort, before the Bridge observation-boundary fixes and the Balloons sustained-hover rule. +The corrected pilots are reported separately in [Bridge/Balloons integrity results](../comparisons/bridge-balloons-integrity-results.md). Each domain and arm has three planned seeds; aggregates remain provisional until all three finish. Whole-run solve rate means winning every training and test level. Both solve rates and mean resets use finished agent runs, with their count shown. @@ -14,48 +16,48 @@ Balloons MB seeds 0 and 1 are reused from the identical agent runtime and flags; | Domain | Arm | Finished seeds | Whole-run solve rate | Level solve rate | Mean steps, successful runs (n) | Mean resets | |---|---|---:|---:|---:|---:|---:| | Bridge | MB | 3/3 | 100.0% (3/3) | 100.0% | 2,214 (n=3) | 0 | -| Bridge | MF | 1/3 | 100.0% (1/1) | 100.0% | 2,413 (n=1) | 0 | +| Bridge | MF | 3/3 | 100.0% (3/3) | 100.0% | 2,269.7 (n=3) | 0 | | Fan | MB | 3/3 | 100.0% (3/3) | 100.0% | 425 (n=3) | 0 | -| Fan | MF | 2/3 | 100.0% (2/2) | 100.0% | 901 (n=2) | 0 | +| Fan | MF | 3/3 | 100.0% (3/3) | 100.0% | 881.3 (n=3) | 0 | | Domino | MB | 3/3 | 100.0% (3/3) | 100.0% | 404.7 (n=3) | 0 | -| Domino | MF | 1/3 | 0.0% (0/1) | 50.0% | - (n=0) | 0 | +| Domino | MF | 3/3 | 0.0% (0/3) | 50.0% | - (n=0) | 0 | | Boil | MB | 3/3 | 100.0% (3/3) | 100.0% | 668.3 (n=3) | 0 | -| Boil | MF | 0/3 | Pending | Pending | - (n=0) | - | +| Boil | MF | 3/3 | 100.0% (3/3) | 100.0% | 934 (n=3) | 0 | | Balloons | MB | 3/3 | 100.0% (3/3) | 100.0% | 365.7 (n=3) | 0.33 | -| Balloons | MF | 0/3 | Pending | Pending | - (n=0) | - | +| Balloons | MF | 3/3 | 100.0% (3/3) | 100.0% | 417 (n=3) | 1 | | Domain | Arm | Seed | Status | Wins | Steps | Resets | Source | |---|---|---:|---|---:|---:|---:|---| | Bridge | MB | 0 | Finished | 2/2 | 2306 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed0/run_20260910_052215/scorecard.json) | | Bridge | MB | 1 | Finished | 2/2 | 1950 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed1/run_20260910_052213/scorecard.json) | | Bridge | MB | 2 | Finished | 2/2 | 2386 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed2/run_20260910_052219/scorecard.json) | -| Bridge | MF | 0 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | +| Bridge | MF | 0 | Finished | 2/2 | 2408 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | | Bridge | MF | 1 | Finished | 2/2 | 2413 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/scorecard.json) | -| Bridge | MF | 2 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | +| Bridge | MF | 2 | Finished | 2/2 | 1988 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | | Fan | MB | 0 | Finished | 2/2 | 312 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed0/run_20260910_062635/scorecard.json) | | Fan | MB | 1 | Finished | 2/2 | 493 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed1/run_20260910_062648/scorecard.json) | | Fan | MB | 2 | Finished | 2/2 | 470 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed2/run_20260910_062710/scorecard.json) | | Fan | MF | 0 | Finished | 2/2 | 1396 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_164313/scorecard.json) | -| Fan | MF | 1 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json) | +| Fan | MF | 1 | Finished | 2/2 | 842 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json) | | Fan | MF | 2 | Finished | 2/2 | 406 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_164315/scorecard.json) | | Domino | MB | 0 | Finished | 2/2 | 395 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed0/run_20260910_073549/scorecard.json) | | Domino | MB | 1 | Finished | 2/2 | 310 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed1/run_20260910_073544/scorecard.json) | | Domino | MB | 2 | Finished | 2/2 | 509 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed2/run_20260910_073545/scorecard.json) | -| Domino | MF | 0 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json) | +| Domino | MF | 0 | Finished | 1/2 | 712 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json) | | Domino | MF | 1 | Finished | 1/2 | 430 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_174127/scorecard.json) | -| Domino | MF | 2 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json) | +| Domino | MF | 2 | Finished | 1/2 | 821 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json) | | Boil | MB | 0 | Finished | 2/2 | 721 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed0/run_20260910_133006/scorecard.json) | | Boil | MB | 1 | Finished | 2/2 | 775 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed1/run_20260910_133006/scorecard.json) | | Boil | MB | 2 | Finished | 2/2 | 509 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed2/run_20260910_132947/scorecard.json) | -| Boil | MF | 0 | In progress | 1/2 | 593 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json) | -| Boil | MF | 1 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json) | -| Boil | MF | 2 | In progress | 1/2 | 570 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json) | +| Boil | MF | 0 | Finished | 2/2 | 738 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json) | +| Boil | MF | 1 | Finished | 2/2 | 1076 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json) | +| Boil | MF | 2 | Finished | 2/2 | 988 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json) | | Balloons | MB | 0 | Finished | 3/3 | 466 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed0/run_20260909_171523/scorecard.json) (reused) | | Balloons | MB | 1 | Finished | 3/3 | 338 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed1/run_20260909_171520/scorecard.json) (reused) | | Balloons | MB | 2 | Finished | 3/3 | 293 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_noise_sweep_r1/seed2/run_20260910_043043/scorecard.json) | -| Balloons | MF | 0 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | -| Balloons | MF | 1 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | -| Balloons | MF | 2 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | +| Balloons | MF | 0 | Finished | 3/3 | 441 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | +| Balloons | MF | 1 | Finished | 3/3 | 476 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | +| Balloons | MF | 2 | Finished | 3/3 | 334 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | ## Duplicate execution audit From 137e83b91e238429e20244f9b8c18ac87da75de3 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:39:58 -0400 Subject: [PATCH 89/94] Document continual baseline integrity results --- .../bridge-balloons-integrity-results.json | 135 ++++++++++++++++++ .../bridge-balloons-integrity-results.md | 25 ++++ .../comparisons/domain-contract-validation.md | 55 +++++++ docs/comparisons/fixed-balloons-mf-notes.md | 46 ++++++ .../mf-integrity-audit-20260912.md | 100 +++++++++++++ 5 files changed, 361 insertions(+) create mode 100644 docs/comparisons/bridge-balloons-integrity-results.json create mode 100644 docs/comparisons/bridge-balloons-integrity-results.md create mode 100644 docs/comparisons/domain-contract-validation.md create mode 100644 docs/comparisons/fixed-balloons-mf-notes.md create mode 100644 docs/comparisons/mf-integrity-audit-20260912.md diff --git a/docs/comparisons/bridge-balloons-integrity-results.json b/docs/comparisons/bridge-balloons-integrity-results.json new file mode 100644 index 000000000..31162a905 --- /dev/null +++ b/docs/comparisons/bridge-balloons-integrity-results.json @@ -0,0 +1,135 @@ +{ + "generated_by": "/home/ycliang/predicators/logs/bridge_balloons_integrity_20260912/report.py", + "updated_at": "2026-09-13T09:07:21.054343+00:00", + "source_commit": "cad1000f92c5e5e94714ae613426b89761823ee0", + "rows": [ + { + "domain": "Bridge", + "approach": "agent_continual", + "arm": "MB + uncertainty", + "seed": 0, + "job": "22646173_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_span_transfer_r1/seed0/run_20260912_165804/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2939, + "resets": 0, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 0, + "job": "22646174_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_span_transfer_r1/seed0/run_20260912_165804/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2634, + "resets": 0, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 0, + "job": "22649185_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 771, + "resets": 1, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 1, + "job": "22649185_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed1/run_20260912_180112/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 554, + "resets": 1, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 2, + "job": "22649185_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed2/run_20260912_180113/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 829, + "resets": 2, + "end_reason": "all_levels_won", + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "MB + uncertainty", + "finished": 1, + "expected": 1, + "successes": 1, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2939, + "steps_n": 1, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "MF direct coding agent", + "finished": 1, + "expected": 1, + "successes": 1, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2634, + "steps_n": 1, + "resets_mean_finished": 0 + }, + { + "domain": "Balloons", + "arm": "MF direct coding agent", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 718, + "steps_n": 3, + "resets_mean_finished": 1.3333333333333333 + } + ] +} diff --git a/docs/comparisons/bridge-balloons-integrity-results.md b/docs/comparisons/bridge-balloons-integrity-results.md new file mode 100644 index 000000000..c140dd41b --- /dev/null +++ b/docs/comparisons/bridge-balloons-integrity-results.md @@ -0,0 +1,25 @@ + +# Bridge transfer and sustained-hover Balloons pilots + +Updated 2026-09-13T09:07:21.054343+00:00. + +Bridge: one seed per MB/MF arm, three-block train span and four-block test span. Balloons: three MF seeds, original non-hatch task distribution, 25-step sustained hovering. Historical cohorts and other comparisons remain separate. +Whole-run success means winning every training and test level. +Solve rates and mean resets use finished agent seeds only. +Mean steps uses only whole-run successes, with the qualifying count shown. +Unfinished, setup, and infrastructure outcomes are excluded from averages. +These tables are refreshed from scorecards and Slurm accounting. + +| Domain | Arm | Finished | Whole-run solve | Level solve | Mean successful steps (n) | Mean resets | +|---|---|---:|---:|---:|---:|---:| +| Bridge | MB + uncertainty | 1/1 | 100.0% (1/1) | 100.0% | 2,939 (n=1) | 0 | +| Bridge | MF direct coding agent | 1/1 | 100.0% (1/1) | 100.0% | 2,634 (n=1) | 0 | +| Balloons | MF direct coding agent | 3/3 | 100.0% (3/3) | 100.0% | 718 (n=3) | 1.3 | + +| Domain | Arm | Seed | Status | Wins | Steps | Resets | Job | Scorecard | +|---|---|---:|---|---:|---:|---:|---|---| +| Bridge | MB + uncertainty | 0 | Finished | 2/2 | 2,939 | 0 | 22646173_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_span_transfer_r1/seed0/run_20260912_165804/scorecard.json) | +| Bridge | MF direct coding agent | 0 | Finished | 2/2 | 2,634 | 0 | 22646174_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_span_transfer_r1/seed0/run_20260912_165804/scorecard.json) | +| Balloons | MF direct coding agent | 0 | Finished | 3/3 | 771 | 1 | 22649185_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json) | +| Balloons | MF direct coding agent | 1 | Finished | 3/3 | 554 | 1 | 22649185_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed1/run_20260912_180112/scorecard.json) | +| Balloons | MF direct coding agent | 2 | Finished | 3/3 | 829 | 2 | 22649185_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed2/run_20260912_180113/scorecard.json) | diff --git a/docs/comparisons/domain-contract-validation.md b/docs/comparisons/domain-contract-validation.md new file mode 100644 index 000000000..4da8e2954 --- /dev/null +++ b/docs/comparisons/domain-contract-validation.md @@ -0,0 +1,55 @@ +# Cross-domain comparison tool validation + +The standalone and zero-shot action-boundary tests now use the real noisy cohort configurations in all five domains. +Their earlier tests exercised these contracts only in Boil. +The other four comparison methods already had five-domain play-tool coverage. + +All ten extended checks passed in array `22650730`. +Type checking and lint passed in job `22650731`. +The tests use scripted agent tool calls and do not call Claude or produce solve-rate outcomes. + +| Domain | Runtime | Check job | Tests passed | +|---|---|---|---:| +| Boil | `59336397069a` | 22650730_0 | 2 | +| Bridge, 3-to-4 span | `73b5e517bf18` | 22650730_1 | 2 | +| Fan | `59336397069a` | 22650730_2 | 2 | +| Domino | `59336397069a` | 22650730_3 | 2 | +| Original Balloons | `59336397069a` | 22650730_4 | 2 | + +The standalone check edits its program during a real continual play session and verifies that subsequent predictions load the changed program. +It checks the supplied model interface has no engine instance or base-simulator reference files, rejects engine-diagnostic rollout modes, and records a charged real action. +This is an interface check, not an exhaustive sandbox security audit. + +The zero-shot check refuses the first charged action without a model, seals a valid model before taking that action, and rejects later source edits and fitting routes. +It restores the sealed source from a saved approach state and verifies that the next action is charged correctly. +The separate-process standalone resume test remains covered by the earlier 25-test suite in job `22648953`. + +An immutable copy of the extended test module was run against each cohort's existing frozen production package. +The runner checks the imported package path and test-file digest before executing the tests. +No frozen experiment code was edited and no agent run was restarted for this validation. +The source paths, digests, job records, and verified pass summaries are recorded in `/home/ycliang/predicators/logs/comparison_domain_contracts_20260912/validated.json` and its adjacent validation manifest. + + +## Standalone package access + +The experiment environment includes PyBullet, NumPy, and SciPy. +The standalone approach omits base-simulator reference files, removes the engine-backed evaluator, and routes supplied predictions through the learned skill-transition program. +The original frozen experiment prompt forbids importing an environment or physics engine inside a prediction. +The import guard screens hidden `predicators.envs` and `predicators.ground_truth_models` modules; it does not prohibit the `pybullet` package itself. +Thus the implemented isolation covers the supplied prediction interface, not all physics packages accessible to arbitrary agent code. +A scan of the two started Boil standalone runs on 2026-09-13 found PyBullet import matches in copied skill-controller references and a transcript displaying one such reference. +That scan did not find an agent-written PyBullet simulator, but it is not an exhaustive proof of engine-free behavior or import isolation. +No experiment runtime or package installation was changed for this audit. + + +## Physics libraries permitted, 2026-09-13 + +The user clarified that the standalone agent may use PyBullet or other available simulation libraries. +The intended comparison withholds a prepared scene and base simulator, while allowing the agent to construct its own predictive world from public observations and recorded interactions. +The development prompt now states this explicitly and retains the prohibition on inspecting the task environment, ground-truth mechanisms, or live simulator state. +The original frozen experiments retain their stricter prompt and must not be presented as a cohort run under the revised instruction. +Replacement configurations cover twelve original non-Bridge seeds and three Bridge span-transfer seeds, keeping their respective domain runtimes and task settings. +Whether to replace the existing standalone cohort or retain it as the stricter comparison is pending the user's preference; no replacement experiment has been submitted. +Compute array 22672397 passed all ten domain checks for agent-owned PyBullet predictions, live-environment isolation, and ordinary program predictions. +Static job 22672446 passed mypy and pylint for both changed Python files after correcting a test type annotation reported by 22672398. +Their manifests and outputs are under `/home/ycliang/predicators/logs/standalone_engine_contract_20260913/`; these are mechanical checks, not agent solve-rate seeds. diff --git a/docs/comparisons/fixed-balloons-mf-notes.md b/docs/comparisons/fixed-balloons-mf-notes.md new file mode 100644 index 000000000..c9fc41303 --- /dev/null +++ b/docs/comparisons/fixed-balloons-mf-notes.md @@ -0,0 +1,46 @@ +# Fixed Balloons: MF results and seed-0 analysis + +All three intended MF seeds solved every level with the sustained-hover goal. +The mean charged cost is 718 steps over three whole-run successful seeds, and mean resets is 1.33. +There is no matched MB rerun under this changed goal. +The current cohort table is [Bridge/Balloons integrity results](bridge-balloons-integrity-results.md). +Its frozen runtime is `cad1000f92c5e5e94714ae613426b89761823ee0`, with the original task distribution, 1 cm position noise, 0.02 rad orientation noise, and zero scalar-reading noise. +The goal requires 25 complete consecutive environment intervals in the band below 0.01 m/s. +Earlier instantaneous-goal Balloons and all hatch runs remain separate comparisons. + +| Seed | Wins | Steps | Resets | +|---|---:|---:|---:| +| 0 | 3/3 | 771 | 1 | +| 1 | 3/3 | 554 | 1 | +| 2 | 3/3 | 829 | 2 | + +All three scorecards record `all_levels_won`, a completion time, and the expected frozen runtime. +Their totals agree with their per-level records. +The detailed behavior analysis below concerns seed 0. + +| Level | Role | Wins | Steps | Resets | +|---|---|---:|---:|---:| +| 1 | Training | 1/1 | 452 | 1 | +| 2 | Training | 1/1 | 217 | 0 | +| 3 | Test | 1/1 | 102 | 0 | + +The [final scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json) records `all_levels_won` and a completion time. +Totals agree with the per-level records. + +## How it solved the tasks + +The direct coding agent measured resting heights and oscillation decay after real balloon releases, used one training reset, and retained the measurements in its journal. +It wrote an empirical equilibrium helper, `lift_model.py`, from those measurements. +The inspected helper contains scalar arithmetic and calibrated constants, without engine imports or calls. +Its fitted power-law relation is the agent's approximation, not a claim that it recovered the environment's exact dynamics. + +On the test task, it recognized an oak box and used the gold balloon's measured height from the first training task. +Before the irreversible release, it spent 12 Wait steps averaging noisy resting-height observations. +It then released gold and won after another 49 Wait steps; the goal interrupted the Wait when the dwell completed. +This behavior is recorded in the [test-level agent log](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/agent/003_play_20260912_182006.md). +The helper's source is recorded in the [second training-level log](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/agent/002_play_20260912_181548.md). + +The baseline has no supplied simulator, but its coding and journal capabilities still let it construct small empirical models from public observations. +Calling it MF does not imply that its reasoning must avoid physical hypotheses or numerical analysis. +The visible box-speed reading has zero scalar noise in this configuration, so using it to detect rest is an allowed observation. +These inspected actions explain the successful transfer; they do not constitute an exhaustive sandbox security audit or establish an MB/MF gap under the changed goal. diff --git a/docs/comparisons/mf-integrity-audit-20260912.md b/docs/comparisons/mf-integrity-audit-20260912.md new file mode 100644 index 000000000..96677f0e3 --- /dev/null +++ b/docs/comparisons/mf-integrity-audit-20260912.md @@ -0,0 +1,100 @@ +# Bridge and Balloons MF integrity audit + +Audited on 2026-09-12 against the selected noisy five-domain sweep runs and their frozen runtime. +Scope: all three selected MF seeds in Bridge and original non-hatch Balloons, including their experiment transcripts, journals, agent-visible recordings, current scorecards, and relevant harness code. + +## Assessment + +No evidence was found in the reviewed experiment logs of loading the hidden environment simulator or reading answers from other runs. +This is a log and data audit, not a proof that every possible sandbox escape is prevented. +Bridge has confirmed information leakage through shared controller diagnostics, and task-specific hints in the exposed controller source. +Balloons has an instantaneous success check that agents knowingly exploit, plus a strong baseline that writes and fits its own analytical models. +These results should not be described as a clean demonstration that a purely model-free agent discovers hidden physics from scratch. + +## Verified results + +| Domain | Arm | Whole-run successes | Mean steps, successful runs | Qualifying n | Mean resets, all three runs | +|---|---|---:|---:|---:|---:| +| Bridge | MB | 3/3 | 2214.0 | 3 | 0.00 | +| Bridge | MF | 3/3 | 2269.7 | 3 | 0.00 | +| Balloons | MB | 3/3 | 365.7 | 3 | 0.33 | +| Balloons | MF | 3/3 | 417.0 | 3 | 1.00 | + +All these seeds completed every level. +Steps include all real interaction in each successful run, including earlier reset episodes; they are not limited to the final winning episode. +MF uses about 2.5% more steps in Bridge and 14.0% more in Balloons. +Three seeds are insufficient to establish a general equivalence claim. +Balloons MB seeds 0 and 1 are the explicitly reused subclass pilot; hatch and historical MF cohorts are excluded. +Task draws should be treated as distribution-matched, not assumed identical merely because the agent seed has the same number. + +| Domain | Arm | Seed | Level wins | Steps | Resets | Scorecard | +|---|---|---:|---:|---:|---:|---| +| Bridge | MB | 0 | 2/2 | 2306 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed0/run_20260910_052215/scorecard.json) | +| Bridge | MB | 1 | 2/2 | 1950 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed1/run_20260910_052213/scorecard.json) | +| Bridge | MB | 2 | 2/2 | 2386 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed2/run_20260910_052219/scorecard.json) | +| Bridge | MF | 0 | 2/2 | 2408 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | +| Bridge | MF | 1 | 2/2 | 2413 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/scorecard.json) | +| Bridge | MF | 2 | 2/2 | 1988 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | +| Balloons | MB | 0 | 3/3 | 466 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed0/run_20260909_171523/scorecard.json) | +| Balloons | MB | 1 | 3/3 | 338 | 1 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed1/run_20260909_171520/scorecard.json) | +| Balloons | MB | 2 | 3/3 | 293 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_noise_sweep_r1/seed2/run_20260910_043043/scorecard.json) | +| Balloons | MF | 0 | 3/3 | 441 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | +| Balloons | MF | 1 | 3/3 | 476 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | +| Balloons | MF | 2 | 3/3 | 334 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | + +Bridge MF seed 1 retains the first completed execution, run_20260910_144727. +The snapshot documents a later duplicate execution after a scheduler restart during video generation; it is excluded as a duplicate, not treated as a fourth seed. + +## Bridge: a hidden attachment is exposed through an error + +In [seed 1, play 003](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/agent/003_play_20260910_151152.md:1006), a failed Place returns `welded span1` and an exact table penetration of 0.0061 m. +The agent immediately writes: `The failure message is gold: "welded span1" - the bond did form on the previous place`. +It then uses the reported geometry to adjust its release height. +The [shared planner diagnostic](/home/ycliang/predicators-noisy-sweep-20260910/predicators/ground_truth_models/skill_factories/base.py:2134) constructs these labels from held attachments. +This is observed use of information about the hidden attachment, not a speculative exploit. +Internal attachment-aware collision planning may be necessary for the controller, but exposing that attachment identity and precise internal geometry is a separate observation-policy decision. + +Before interaction, the agent also [reads the supplied controller references](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/agent/001_play_20260910_144733.md:1687) and extracts placement heights, weld behavior, and the recipe of lifting a welded three-span row by its middle block. +The [reference exporter](/home/ycliang/predicators-noisy-sweep-20260910/predicators/approaches/agent_model_free_approach.py:345) supplies controller implementation files, including comments. +The public task already asks for a rigid three-block span, so the references do not reveal the entire goal for the first time; they provide additional implementation-specific solution guidance. +These references and controllers are shared with MB, so this is a shared benchmark issue rather than evidence of preferential MF access. + +## Balloons: legitimate self-modeling and an instantaneous win + +The [seed 1 journal](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/agent/sandbox/journal.md:230) compares competing dynamics hypotheses. +Its [uncertainty calculations](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/agent/sandbox/journal.md:273) use damped-sinusoid fits and residual bootstrap estimates. +Other seeds similarly fit oscillation or lift models, enumerate balloon subsets, and use real probes before committing to releases. +These calculations operate on the agent’s own recorded observations and are allowed by the baseline prompt. +The absence of supplied simulator calls does not mean an absence of learned dynamics or uncertainty reasoning. +A more accurate baseline name is direct coding agent without supplied simulator or model-learning tools. + +The [InBand classifier](/home/ycliang/predicators-noisy-sweep-20260910/predicators/envs/pybullet_balloons.py:283) requires the box centre to be in the band and its speed to be below 0.01 m/s in the current state. +It imposes no sustained dwell condition. +The [seed 2 journal](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/agent/sandbox/journal.md:494) explicitly recognizes that an oscillation turning point can certify success even when the equilibrium height is outside the band. +All nine recorded MF level wins have a final speed below 0.01 m/s and a preceding frame above that threshold. +For example, seed 0 level 1 wins at z=0.78166 m in the [0.75952, 0.80952] m band, with speed falling from 0.11596 to 0.00991 m/s. +Those are valid wins under the implemented predicate, but do not demonstrate sustained hovering. +The recordings terminate at success, so this audit does not establish which runs would eventually settle successfully or fail a longer dwell test. +The predicate also applies to MB; this audit does not establish that the issue favors MF more than MB. + +## Observation and tool checks + +The [recording audit](/home/ycliang/predicators/logs/mf_integrity_audit_20260912/data-audit.json) examined 8,075 MF frames across the six runs. +No frame carried a populated privileged or latent state channel, and no simulator-state dictionary keys were exposed. +However, serialized Object instances retain sim_data metadata. +Bridge objects expose cure and attachment field names, although their values remain the default zero or -1 throughout these recordings; Balloons metadata includes body and clip joint IDs. +This is unnecessary implementation metadata and should be removed from the public serialization contract, but it is not evidence that these agents read live hidden cure/attachment values. +The [extracted tool calls](/home/ycliang/predicators/logs/mf_integrity_audit_20260912/tool-calls.json) show analysis of recorded trajectories and supplied reference files, with no use of registered simulator/model tools. +Some attempts to repair sandbox working-directory errors involved broader tools or path changes; these do not justify claiming that the sandbox is adversarially secure. + +## Recommended next steps + +1. Keep and report the existing results as the current protocol cohort, with the above limitations. +2. Replace controller implementation references with a public skill API specification, removing task-specific recipes and hidden-mechanism comments for both arms. +3. Sanitize controller diagnostics so they do not name hidden attachments or expose privileged geometry, while preserving useful feedback based on public observations. +4. Remove simulator metadata from agent-visible serialized objects and validate the complete observation/tool boundary. +5. If the intended Balloons task is sustained hovering, specify and mechanically validate a dwell criterion, then rerun both arms under the changed task as a separate cohort. +6. Preserve ordinary Python analysis and learned journals in the direct coding baseline; banning its successful reasoning would weaken the comparison artificially. + +No experiments, controllers, or task definitions were changed by this audit. +Cleaning these issues may change either arm’s performance; it does not guarantee a larger MB advantage. From a6036bf94ba9bca71cdd675dc81b37cef6097a3e Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:40:19 -0400 Subject: [PATCH 90/94] Archive continual comparison scorecards --- .../bridge-three-span-results.json | 423 ++++ docs/comparisons/bridge-three-span-results.md | 43 + docs/comparisons/continual-results.json | 2231 +++++++++++++++++ 3 files changed, 2697 insertions(+) create mode 100644 docs/comparisons/bridge-three-span-results.json create mode 100644 docs/comparisons/bridge-three-span-results.md create mode 100644 docs/comparisons/continual-results.json diff --git a/docs/comparisons/bridge-three-span-results.json b/docs/comparisons/bridge-three-span-results.json new file mode 100644 index 000000000..91529fafc --- /dev/null +++ b/docs/comparisons/bridge-three-span-results.json @@ -0,0 +1,423 @@ +{ + "generated_by": "/home/ycliang/predicators/logs/bridge_three_span_20260914/report.py", + "updated_at": "2026-09-14T18:09:00.815253+00:00", + "source_commit": "091d8c5db11f8944662e7a8af28a070905886604", + "rows": [ + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22717645_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed0/run_20260914_032501/scorecard.json", + "finished": true, + "success": false, + "wins": 0, + "levels": 2, + "steps": 12964, + "resets": 17, + "end_reason": "agent_ended", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22717645_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed1/run_20260914_032851/scorecard.json", + "finished": true, + "success": false, + "wins": 0, + "levels": 2, + "steps": 19324, + "resets": 16, + "end_reason": "agent_ended", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22717645_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed2/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2724, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22717646_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed0/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1921, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22717646_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed1/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1706, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22717646_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed2/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1851, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22717647_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed0/run_20260914_094920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2519, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22717647_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed1/run_20260914_094923/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 3158, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22717647_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed2/run_20260914_094920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2505, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22717650_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed0/run_20260914_041919/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2307, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22717650_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed1/run_20260914_041922/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2843, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22717650_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed2/run_20260914_041928/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2210, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22717651_0", + "job_state": "CANCELLED by 215151", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed0/run_20260914_110412/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2000, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22717651_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed1/run_20260914_110412/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 5766, + "resets": 2, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22717651_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed2/run_20260914_110417/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 4167, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22717652_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed0/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2327, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22717652_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed1/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2252, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22717652_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed2/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2482, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.3333333333333333, + "steps_mean_success": 2724, + "steps_n": 1, + "resets_mean_finished": 11 + }, + { + "domain": "Bridge", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 1826, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2727.3333333333335, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + }, + { + "domain": "Bridge", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2453.3333333333335, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 3977.6666666666665, + "steps_n": 3, + "resets_mean_finished": 1 + }, + { + "domain": "Bridge", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2353.6666666666665, + "steps_n": 3, + "resets_mean_finished": 0 + } + ] +} diff --git a/docs/comparisons/bridge-three-span-results.md b/docs/comparisons/bridge-three-span-results.md new file mode 100644 index 000000000..1626a0748 --- /dev/null +++ b/docs/comparisons/bridge-three-span-results.md @@ -0,0 +1,43 @@ + +# Three-span Bridge comparisons + +Updated 2026-09-14T18:09:00.815253+00:00. + +Three span blocks at training and test; seeds 0, 1, 2 per arm; 5 mm position noise and 0.02 rad orientation noise. +Shared information-integrity fixes retained; experimental rigid assemblies and transport previews absent. +Separate cohort from four-span results and historical results before information-integrity fixes. +Standalone agents may build their own physics simulators, including PyBullet; no prepared scene simulator is supplied. +No harness fitting permits agent-written fitting. No-explicit-uncertainty results require review for custom agent uncertainty checks. +Whole-run solve rate and mean resets use finished agent seeds; mean steps uses only whole-run successes, with n. +No-fitting seed 0 completed in 2000 steps before a scheduler restart repeated it (2138 steps, 2/2 wins, 0 resets); retain the original run and exclude the repeat. The pending further restart was cancelled. +Unfinished and infrastructure outcomes are excluded from averages. + +| Domain | Arm | Finished | Whole-run solve | Level solve | Mean successful steps (n) | Mean resets | +|---|---|---:|---:|---:|---:|---:| +| Bridge | 3. Standalone program | 3/3 | 33.3% (1/3) | 33.3% | 2,724 (n=1) | 11 | +| Bridge | 4. Oracle dynamics | 3/3 | 100.0% (3/3) | 100.0% | 1,826 (n=3) | 0 | +| Bridge | 5. Oracle scene | 3/3 | 100.0% (3/3) | 100.0% | 2,727.3 (n=3) | 0.3 | +| Bridge | 6. Zero-shot model | 3/3 | 100.0% (3/3) | 100.0% | 2,453.3 (n=3) | 0 | +| Bridge | 7. No harness fitting | 3/3 | 100.0% (3/3) | 100.0% | 3,977.7 (n=3) | 1 | +| Bridge | 8. No explicit uncertainty | 3/3 | 100.0% (3/3) | 100.0% | 2,353.7 (n=3) | 0 | + +| Domain | Arm | Seed | Status | Wins | Steps | Resets | Job | Scorecard | +|---|---|---:|---|---:|---:|---:|---|---| +| Bridge | 3. Standalone program | 0 | Finished | 0/2 | 12,964 | 17 | 22717645_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed0/run_20260914_032501/scorecard.json) | +| Bridge | 3. Standalone program | 1 | Finished | 0/2 | 19,324 | 16 | 22717645_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed1/run_20260914_032851/scorecard.json) | +| Bridge | 3. Standalone program | 2 | Finished | 2/2 | 2,724 | 0 | 22717645_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed2/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 0 | Finished | 2/2 | 1,921 | 0 | 22717646_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed0/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 1 | Finished | 2/2 | 1,706 | 0 | 22717646_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed1/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 2 | Finished | 2/2 | 1,851 | 0 | 22717646_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed2/run_20260914_032502/scorecard.json) | +| Bridge | 5. Oracle scene | 0 | Finished | 2/2 | 2,519 | 0 | 22717647_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed0/run_20260914_094920/scorecard.json) | +| Bridge | 5. Oracle scene | 1 | Finished | 2/2 | 3,158 | 1 | 22717647_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed1/run_20260914_094923/scorecard.json) | +| Bridge | 5. Oracle scene | 2 | Finished | 2/2 | 2,505 | 0 | 22717647_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed2/run_20260914_094920/scorecard.json) | +| Bridge | 6. Zero-shot model | 0 | Finished | 2/2 | 2,307 | 0 | 22717650_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed0/run_20260914_041919/scorecard.json) | +| Bridge | 6. Zero-shot model | 1 | Finished | 2/2 | 2,843 | 0 | 22717650_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed1/run_20260914_041922/scorecard.json) | +| Bridge | 6. Zero-shot model | 2 | Finished | 2/2 | 2,210 | 0 | 22717650_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed2/run_20260914_041928/scorecard.json) | +| Bridge | 7. No harness fitting | 0 | Finished | 2/2 | 2,000 | 0 | 22717651_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed0/run_20260914_110412/scorecard.json) | +| Bridge | 7. No harness fitting | 1 | Finished | 2/2 | 5,766 | 2 | 22717651_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed1/run_20260914_110412/scorecard.json) | +| Bridge | 7. No harness fitting | 2 | Finished | 2/2 | 4,167 | 1 | 22717651_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed2/run_20260914_110417/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 0 | Finished | 2/2 | 2,327 | 0 | 22717652_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed0/run_20260914_052356/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 1 | Finished | 2/2 | 2,252 | 0 | 22717652_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed1/run_20260914_052356/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 2 | Finished | 2/2 | 2,482 | 0 | 22717652_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed2/run_20260914_052356/scorecard.json) | diff --git a/docs/comparisons/continual-results.json b/docs/comparisons/continual-results.json new file mode 100644 index 000000000..7c3834308 --- /dev/null +++ b/docs/comparisons/continual-results.json @@ -0,0 +1,2231 @@ +{ + "generated_by": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/report.py", + "updated_at": "2026-09-13T13:48:42.462889+00:00", + "source_commit": "59336397069afd098f132aac1deaca5c9cba73e9", + "rows": [ + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642703_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 540, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642703_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_160623/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 535, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642703_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 573, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642704_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 350, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642704_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 174, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642704_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_160637/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 613, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642705_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_170526/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 622, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642705_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_170523/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 276, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642705_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_170525/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 328, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642706_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642706_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642706_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642707_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_173725/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 362, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642707_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_173956/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 307, + "resets": 0, + "end_reason": "level_lost", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642707_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_173920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 328, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642708_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed0/run_20260912_165320/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 758, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642708_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed1/run_20260912_165600/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 802, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642708_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed2/run_20260912_165331/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1277, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642709_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed0/run_20260912_190626/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 1594, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642709_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed1/run_20260912_203736/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 335, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642709_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed2/run_20260912_205841/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 470, + "resets": 0, + "end_reason": "level_lost", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642710_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed0/run_20260912_173627/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 715, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642710_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed1/run_20260912_173629/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 281, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642710_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed2/run_20260912_173623/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 545, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642711_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642711_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642711_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642712_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed0/run_20260912_183323/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 262, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642712_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed1/run_20260912_183337/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 300, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642712_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed2/run_20260912_183337/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 395, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642713_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed0/run_20260912_215028/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 517, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642713_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed1/run_20260912_211305/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 633, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642713_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed2/run_20260912_213833/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1022, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642714_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed0/run_20260912_205352/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 400, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642714_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed1/run_20260912_210259/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 436, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642714_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed2/run_20260912_210028/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 377, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642715_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed0/run_20260912_221531/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 374, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642715_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed1/run_20260912_214954/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 446, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642715_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed2/run_20260912_223336/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 533, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642716_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642716_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642716_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642717_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed0/run_20260912_223957/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 417, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642717_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed1/run_20260912_223139/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 526, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642717_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed2/run_20260913_012942/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 334, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642718_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed0/run_20260912_215805/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 542, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642718_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed1/run_20260912_215733/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 761, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642718_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed2/run_20260912_214921/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1071, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642719_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed0/run_20260913_024607/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 877, + "resets": 3, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642719_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed1/run_20260913_032344/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 631, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642719_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed2/run_20260913_025446/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 342, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642720_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed0/run_20260912_223138/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 330, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642720_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed1/run_20260912_223130/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 420, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642720_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed2/run_20260912_223933/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 282, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642721_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642721_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642721_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642722_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed0/run_20260913_012856/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 400, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642722_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed1/run_20260912_225924/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 394, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642722_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed2/run_20260912_230435/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 511, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642723_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed0/run_20260913_040903/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1093, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642723_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed1/run_20260913_042604/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 547, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642723_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed2/run_20260913_040746/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 565, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642724_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed0/run_20260913_035506/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 243, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642724_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed1/run_20260913_030534/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 271, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642724_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed2/run_20260913_033451/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 268, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642725_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed0/run_20260913_045206/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 346, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642725_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed1/run_20260913_045849/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 289, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642725_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed2/run_20260913_044710/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 302, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642726_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642726_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642726_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642727_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed0/run_20260913_051311/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 559, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642727_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed1/run_20260913_051832/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 272, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642727_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed2/run_20260913_051638/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 384, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642728_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed0/run_20260913_050000/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2846, + "resets": 7, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642728_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed1/run_20260913_034701/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1782, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642728_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed2/run_20260913_040647/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 886, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642729_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed0/run_20260913_075911/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 403, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642729_1", + "job_state": "RUNNING", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed1/run_20260913_083354/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 558, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "None", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642729_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed2/run_20260913_082421/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 788, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642730_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed0/run_20260913_074102/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 735, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642730_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed1/run_20260913_065042/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 890, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642730_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed2/run_20260913_050240/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 547, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642731_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642731_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642731_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642732_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed0/run_20260913_080352/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 702, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642732_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed1/run_20260913_073131/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 484, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642732_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed2/run_20260913_062933/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 459, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "3. Standalone program", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "4. Oracle dynamics", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "5. Oracle scene", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "6. Zero-shot model", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "7. No harness fitting", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Bridge", + "arm": "8. No explicit uncertainty", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Fan", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 724, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 408.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 513.6666666666666, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 451, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 344, + "steps_n": 3, + "resets_mean_finished": 0, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Fan", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 312.3333333333333, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 0, + "solve_rate": 0.0, + "level_solve_rate": 0.5, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 345, + "steps_n": 2, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 319, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 425.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 455.5, + "steps_n": 2, + "resets_mean_finished": 0, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Domino", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 328, + "steps_n": 2, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 1838, + "steps_n": 3, + "resets_mean_finished": 2.3333333333333335 + }, + { + "domain": "Boil", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 549.3333333333334, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 945.6666666666666, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 724, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 791.3333333333334, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Boil", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 735, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + }, + { + "domain": "Balloons", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.7777777777777778, + "steps_mean_success": 403, + "steps_n": 1, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 379, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Balloons", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.7777777777777778, + "steps_mean_success": 335, + "steps_n": 1, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8888888888888888, + "steps_mean_success": 388.5, + "steps_n": 2, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8888888888888888, + "steps_mean_success": 609.5, + "steps_n": 2, + "resets_mean_finished": 1.6666666666666667, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Balloons", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 260.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + } + ], + "active_expected": 72, + "superseded_seeds": 18, + "protocol_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/no-fitting-protocol-audit.json", + "uncertainty_protocol_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/no-uncertainty-protocol-audit.json", + "standalone_model_use_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/standalone-model-use-audit-20260913.json" +} From 8317fd1aa6045c553e62b476b8d515e6c58133e4 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:40:36 -0400 Subject: [PATCH 91/94] Add continual comparison figure renderer --- .../plotting/plot_continual_comparisons.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 scripts/plotting/plot_continual_comparisons.py diff --git a/scripts/plotting/plot_continual_comparisons.py b/scripts/plotting/plot_continual_comparisons.py new file mode 100644 index 000000000..21c45a72f --- /dev/null +++ b/scripts/plotting/plot_continual_comparisons.py @@ -0,0 +1,249 @@ +"""Capture verified continual cohorts and render the paper's Figure 4. + +Capture once with --capture --paper-root PATH; replot the archived +--snapshot without reading live experiment directories. Costs use +successful runs only. +""" +import argparse +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.ticker import FuncFormatter, MaxNLocator + +ROOT = Path(__file__).resolve().parents[2] +DOMAINS = ['Boil', 'Domino', 'Fan', 'Bridge', 'Balloons'] +ARMS = [ + 'MB', 'MF', 'agent_continual_program_world_model', + 'agent_continual_oracle_dynamics', 'agent_continual_oracle_scene', + 'agent_continual_zero_shot', 'agent_continual_no_fitting', + 'agent_continual_no_uncertainty' +] +LABELS = [ + 'EMPIRIC', 'Direct agent', 'Standalone sim.', 'Oracle dynamics', + 'Oracle scene', 'Zero-shot model', 'No harness fitting', + 'No explicit uncert.' +] +COLORS = [ + '#087f8c', '#bd5929', '#7467a6', '#397957', '#6b9483', '#5588ad', + '#b19658', '#88929d' +] + + +def capture(paper, target): + """Verify every selected final scorecard before freezing plot inputs.""" + sys.path.insert(0, str(paper / 'scripts')) + spec = importlib.util.spec_from_file_location( + 'paper_artifacts', paper / 'scripts/build_artifacts.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + rows, _ = module.verified_reported_rows() + reports = [ + ROOT / 'docs/comparisons/continual-results.json', + ROOT / 'docs/comparisons/bridge-three-span-results.json' + ] + sources = [] + for index, report in enumerate(reports): + document = json.loads(report.read_text()) + for row in document['rows']: + if (row['domain'] == 'Bridge') != (index == 1): + continue + assert row['finished'] and row['approach'] in ARMS + path = Path(row['scorecard']) + raw = path.read_bytes() + card = json.loads(raw) + totals = card['totals'] + assert card['finished_at'] and card['end_reason'] in { + 'all_levels_won', 'level_lost', 'level_not_won', 'agent_ended', + 'step_cap', 'wall_clock_cap' + } + assert card['seed'] == row['seed'] and card['arm'] == row[ + 'approach'] + assert document['source_commit'].startswith(card['git_sha']) + assert totals['total_steps'] == row['steps'] == sum( + l['steps'] for l in card['levels']) + assert totals['total_resets'] == row['resets'] == sum( + l['resets'] for l in card['levels']) + assert totals['levels_completed'] == row['wins'] == sum( + l['won'] for l in card['levels']) + assert totals['levels_total'] == row['levels'] + rows.append( + dict(domain=row['domain'], + arm=row['approach'], + seed=row['seed'], + won=row['wins'], + levels=row['levels'], + steps=row['steps'], + resets=row['resets'], + source=str(path), + sha256=hashlib.sha256(raw).hexdigest(), + git_sha=card['git_sha'])) + sources.append({ + 'path': + str(report), + 'sha256': + hashlib.sha256(report.read_bytes()).hexdigest() + }) + assert len(rows) == 115 + assert len({(r['domain'], r['arm'], r['seed']) for r in rows}) == 115 + for domain in DOMAINS: + for arm in ARMS: + assert sum(r['domain'] == domain and r['arm'] == arm + for r in rows) == (2 if arm == 'MF' else 3) + payload = { + 'generated_by': + 'scripts/plotting/plot_continual_comparisons.py', + 'policy': + 'Preserve paper MB (3) and historical MF (2); add six comparison arms (3 each). Bridge uses three-span integrity-fixed cohort. Steps: whole-run successes only; solve: all levels; resets: all finished runs.', + 'sources': + sources, + 'records': + rows, + 'caveats': [ + 'Cohorts differ in observation handling and agent runtime; not a matched causal ablation.', + 'Only new Bridge standalone permits engine imports; other domains retain stricter historical prompt.', + 'Some historical standalone agents did not use a model; uncertainty arms include known custom uncertainty checks.', + 'Original no-fitting Bridge seed 0 retained; post-completion scheduler repeat excluded.', + 'Balloons is original non-hatch with historical instantaneous goal.' + ] + } + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, indent=2) + '\n') + + +def render(snapshot, output): + """Means plus individual seed values, without small-sample CI claims.""" + data = json.loads(snapshot.read_text()) + rows = data['records'] + plt.rcParams.update({ + 'font.family': 'DejaVu Sans', + 'font.size': 9, + 'svg.fonttype': 'none', + 'pdf.fonttype': 42, + 'svg.hashsalt': 'continual-comparisons' + }) + fig, axes = plt.subplots(3, 5, figsize=(12.8, 8.7), sharey=True) + summary = [] + for col, domain in enumerate(DOMAINS): + for metric, field in enumerate(['solve', 'steps', 'resets']): + ax = axes[metric, col] + for i, arm in enumerate(ARMS): + group = [ + r for r in rows + if r['domain'] == domain and r['arm'] == arm + ] + eligible = [r for r in group if r['won'] == r['levels'] + ] if field == 'steps' else group + vals = [ + 100 * r['won'] / + r['levels'] if field == 'solve' else r[field] + for r in eligible + ] + avg = float(np.mean(vals)) if vals else None + summary.append( + dict(domain=domain, + arm=arm, + metric=field, + mean=avg, + n=len(vals), + values=vals)) + if i % 2 == 0: + ax.axhspan(i - .48, i + .48, color='#f3f6f7', zorder=0) + if vals: + ax.barh(i, avg, height=.56, color=COLORS[i], zorder=2) + jitter = np.linspace(-.15, .15, + len(vals)) if len(vals) > 1 else [0] + ax.scatter(vals, + i + np.asarray(jitter), + s=12, + facecolors='white', + edgecolors='#263c46', + linewidths=.6, + zorder=3, + clip_on=False) + if field == 'steps': + ax.text(.98, + i, + f'n={len(vals)}' if vals else 'no success', + transform=ax.get_yaxis_transform(), + ha='right', + va='center', + fontsize=7, + color='#344a55', + bbox=dict(facecolor='white', + edgecolor='none', + pad=.6)) + ax.set_ylim(7.6, -.7) + ax.set_yticks(range(8), LABELS, fontsize=10) + ax.spines[['top', 'right', 'left']].set_visible(False) + ax.spines['bottom'].set_color('#aebec5') + ax.tick_params(length=0, pad=4, labelsize=10) + ax.set_axisbelow(True) + ax.grid(axis='x', color='#dce4e7', lw=.6) + if field == 'solve': + ax.set_xlim(-3, 108) + ax.set_xticks([0, 50, 100]) + ax.set_title(domain, + fontsize=12, + fontweight='bold', + color='#203744', + pad=12) + ax.set_xlabel('Levels solved (%)', fontsize=11) + elif field == 'steps': + maximum = max( + (r['steps'] for r in rows + if r['domain'] == domain and r['won'] == r['levels']), + default=1) + ax.set_xlim(0, maximum * 1.30) + ax.xaxis.set_major_locator(MaxNLocator(3)) + ax.xaxis.set_major_formatter( + FuncFormatter(lambda x, _: f'{x/1000:g}k' + if x >= 1000 else f'{x:g}')) + ax.set_xlabel('Successful-run steps', fontsize=11) + else: + ax.set_xlim(left=-.05, right=max(1, ax.get_xlim()[1])) + ax.xaxis.set_major_locator(MaxNLocator(3, integer=True)) + ax.set_xlabel('Resets (all runs)', fontsize=11) + fig.subplots_adjust(left=.095, + right=.99, + top=.94, + bottom=.145, + wspace=.23, + hspace=.30) + output.parent.mkdir(parents=True, exist_ok=True) + for extension in ['pdf', 'svg', 'png']: + fig.savefig(output.with_suffix('.' + extension), + dpi=180, + bbox_inches='tight', + pad_inches=.06) + svg = output.with_suffix('.svg') + svg.write_text('\n'.join(line.rstrip() + for line in svg.read_text().splitlines()) + '\n') + plt.close(fig) + output.with_name(output.name + '-summary.json').write_text( + json.dumps(summary, indent=2) + '\n') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--paper-root', + type=Path, + default=ROOT.parent / 'sim-predicator-paper') + parser.add_argument('--snapshot', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--capture', action='store_true') + args = parser.parse_args() + if args.capture: + capture(args.paper_root, args.snapshot) + render(args.snapshot, args.output) + + +if __name__ == '__main__': + main() From 7c47be3c576186e960c4d9a89f38e68d09337e76 Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:43:26 -0400 Subject: [PATCH 92/94] Clarify replay and simulator discrepancy migration requirements --- docs/uncertainty/simplification-proposal.md | 34 +++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/uncertainty/simplification-proposal.md b/docs/uncertainty/simplification-proposal.md index 4ce57d8bf..269bc51e2 100644 --- a/docs/uncertainty/simplification-proposal.md +++ b/docs/uncertainty/simplification-proposal.md @@ -2,6 +2,7 @@ September 11, 2026. Revised September 12, 2026 to clarify exact conditioning, inference availability, and the optional execution-filter extension. +Implementation decisions confirmed September 14, 2026: replay from candidate initialization is the required reference, and limited simulator discrepancy is an explicitly evaluated extension after replay and program errors are diagnosed. This is a design proposal, not a description of an implemented or validated replacement. The companion [implementation explanation](explained.md) documents the current behavior and source locations. Implementation of the staged migration is tracked in [implementation progress](implementation-progress.md). @@ -20,7 +21,7 @@ Keeping the existing observation estimator is a valid final choice if those exte This endpoint unifies parameter uncertainty; it does not claim a full joint Bayesian belief during execution. Start by preserving the current agent behind a versioned result interface, without changing its estimates, reports, or decisions. -Verify state restoration and recorded-action replay before implementing parameter inference with uncertain initial states. +Verify candidate initialization and recorded-action replay before implementing parameter inference with uncertain initial states. Evaluate that inference offline using a fixed prior and all available fitting recordings, with intervals and parameter ensembles derived from the same posterior approximation. Introduce it into planning only after it passes prediction checks, while preserving the existing execution estimator and decision rules. Replace execution state estimation last, if it independently demonstrates a benefit. @@ -95,6 +96,10 @@ Only evaluator-only mechanical audits may use evaluator reset state or private d A portable candidate must include full body orientations, original command-weld frames, and commands queued for the next action, even when these quantities are absent from public observations. Their values must come from the candidate prior or simulated history, not privileged recording metadata. Arbitrary mid-trajectory restoration remains an approximation until separately validated against uninterrupted replay. +It is not required for this migration and must not block inference or planning integration that uses full-prefix replay. +Without a validated checkpoint, evaluate each alternative future by reconstructing its candidate and replaying the recorded low-level actions from initialization. +For a stochastic discrepancy model, preserve or explicitly resample the candidate's latent history under the declared conditional law; an unrelated random replay does not reconstruct the same candidate. +Checkpointing and prefix caching are optional optimizations whose identity must include the program, parameters, initial state, actions, and any latent random history. Numerically repeatable candidate replay, faithful evaluator reconstruction, and predictive accuracy of a learned program are three distinct acceptance claims. ### Required initial-state inventory @@ -180,6 +185,30 @@ Evaluate any transition-discrepancy model on held-out development interactions a For a stochastic discrepancy extension, infer the intermediate states under its declared transition distribution; the deterministic initial-state formula above no longer suffices. Persistent mismatch should still inform program revision. +### Discrepancy implementation decision + +Diagnose reconstruction bugs, parameter error, missing program logic, and residual approximation error separately before extending the probability model. +Fix reproducible initialization or replay defects directly. +Expose missing mechanisms and incompatible exact predictions to simulator-program revision; a discrepancy term must not silently excuse them. +Lack of informative observations, such as an all-burners-off fitting prefix, is a separate cause of broad uncertainty and does not by itself justify discrepancy. + +A limited discrepancy extension may proceed when a repeatable residual pattern remains after those checks. +Declare the affected quantities, temporal law, fixed hyperparameters or original hyperprior, and physical or output-level interpretation before running the comparison. +An output discrepancy changes the distribution of observations around a simulated trajectory; a transition discrepancy changes physical histories and therefore can change contacts and events. +These are distinct model changes and must not be substituted for one another. +Keep declared sensor noise unchanged and preserve exact observations through the appropriate conditional construction. + +Use the same discrepancy law in fitting, future generation, and future-density evaluation. +Select a law using fitting or designated development data, freeze it, and evaluate causal predictions on a separate suffix or recording. +If a previously held-out suffix motivates a new law, it becomes development evidence and a new untouched evaluation is required for acceptance. +Compare with the model without that extension where its conditional target is supported, retaining unsupported cases explicitly rather than manufacturing a posterior. +Report numerical repeatability, prediction error, consequential event probabilities, and compute cost; better training likelihood alone is insufficient. +Keep diagnostic interventions that suppress future noise separate from a consistently refitted model. + +The production implementation should not accumulate domain-specific corrections selected to make these recordings pass. +Any domain knowledge needed for a prior or model must have an explicit source available to the agent through the task interface or learned simulator. +Retain the existing execution estimator and decision rules during this evaluation. + ## 3. Standardize the inference result, evaluate the approximation Evaluate a batch sampler over joint dynamics parameters and uncertain episode initial states as the first candidate implementation. @@ -376,7 +405,8 @@ Record later runtime changes separately instead of attributing every difference ### Stage A: define and verify the probability model -Implement and verify the state/restoration contract from section 1 before the new fitter consumes real recordings. +Implement and verify the candidate-initialization and full-prefix replay contract from section 1 before the new fitter consumes real recordings. +Arbitrary mid-trajectory restoration is not an advancement requirement. Complete the five domain inventories and derive the reduced conditional targets before advancing to real-recording posterior comparisons. Implement the observation likelihood, initial-state prior, immutable data identity, and posterior result format beside the existing fitter. Verify the likelihood against the noise injector, including angle handling, missing measurements, and cached observations. From c93fe80fadee79ed6e79b6babafcfac1376cd77d Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 15:53:59 -0400 Subject: [PATCH 93/94] Validate conditional Boil heating inference and record completed comparisons --- .../balloons-selective-guidance.md | 14 ++- docs/uncertainty/boil-informative-heating.md | 85 ++++++++++++++ docs/uncertainty/bridge-memory-recovery.md | 12 +- docs/uncertainty/implementation-progress.md | 28 +++-- docs/uncertainty/reduced-boil-forecasts.md | 12 +- .../uncertainty/validation-status-20260914.md | 104 ++++++++++++++++++ 6 files changed, 236 insertions(+), 19 deletions(-) create mode 100644 docs/uncertainty/boil-informative-heating.md create mode 100644 docs/uncertainty/validation-status-20260914.md diff --git a/docs/uncertainty/balloons-selective-guidance.md b/docs/uncertainty/balloons-selective-guidance.md index 0460fe405..b699215a0 100644 --- a/docs/uncertainty/balloons-selective-guidance.md +++ b/docs/uncertainty/balloons-selective-guidance.md @@ -64,9 +64,9 @@ An additional likelihood that depends on a selected uniform coordinate preserves The existing exact prefix replays pass, 14 of 16 new guide candidates have finite support, serial and parallel targets agree exactly, and future-data corruption leaves the fitting target unchanged. The maximum target-factorization discrepancy is 9.09e-13. -Array `22715405` runs paired fitting seeds 620 and 621 on `mit_preemptable`, each with four CPUs and 20 GiB. +Array `22715405` completed paired fitting seeds 620 and 621 on `mit_preemptable`, each with four CPUs and 20 GiB. It retains 64 particles, 32 cubic temperatures, eight moves, the original blocks and a 16,448-evaluation cap. -Final-target readers `22715406` and `22715407` are dependency-queued to recover each complete checkpoint and freshly evaluate every retained particle. +Final-target readers `22715406` and `22715407` completed, recovering each complete checkpoint and freshly evaluating every retained particle. Their frozen bundle is `logs/uncertainty_balloons_selective_verification_20260914`. The selective-proposal weighted future adapter in `logs/uncertainty_balloons_selective_forecasts_20260914` has passed native validation and independent verification. Its particle decoder binds the new guide and all five uniform-coordinate selections; the default-guide decoder cannot be substituted for it. @@ -94,8 +94,8 @@ Both completed reports match the checksums of their frozen inputs and source art The reader successfully rejects both the old proposal center and the correct center with its uniform-coordinate selection omitted. This distinction matters because omitting the selection changes the physical state represented by the same saved proposal coordinates. -Full forecasts `22715569` and `22715571` are queued behind the completed-fit readers and successful adapter verification. -Independent forecast readers `22715570` and `22715572` follow them. +Full forecasts `22715569` and `22715571` completed after the completed-fit readers and successful adapter verification. +Independent forecast readers `22715570` and `22715572` failed before verification because their launcher invocation omitted the required numerical index. Comparison fixture `22715573` and final comparison `22715574` cover the original, default-centered and selective treatments, retaining the incumbent estimator as a separate reference. The full comparison also waits for the earlier default-guide forecast readers, so unfinished treatments cannot be silently omitted. @@ -106,3 +106,9 @@ Its synthetic weighted-distribution check and six deliberate invalid-budget, inv It independently reproduces the original two fits' parameter summaries and retains all four new forecasts as pending. An incomplete comparison fixture is expected while new forecasts are unavailable; full comparison requires all six verified forecasts. These reports remain offline evidence and do not establish agent solve rates. + +The resulting dependent comparison `22715574` was cancelled. +The exact shell failure was reproduced, and replacement reader array `22755276` supplies both required arguments to the unchanged frozen reader. +Comparison `22755277` follows successful completion of both readers. +No fit or forecast is regenerated for this invocation-only recovery. +The recovery manifest and outputs are in `logs/uncertainty_balloons_selective_reader_recovery_20260914`. diff --git a/docs/uncertainty/boil-informative-heating.md b/docs/uncertainty/boil-informative-heating.md new file mode 100644 index 000000000..edba7f497 --- /dev/null +++ b/docs/uncertainty/boil-informative-heating.md @@ -0,0 +1,85 @@ +# Boil heating: information and conditional numerical accuracy + +September 14, 2026. +This diagnostic follows the [completed reduced-target comparison](validation-status-20260914.md#boil-reduced-target-outcome). +It asks whether informative heating observations constrain the thermal parameters under the existing simulator program and declared sensor noise. +It does not change the production agent or establish an adequate joint posterior over the scene and all parameters. + +## Fixed conditional problem + +The physical histories come from the independently verified density trajectories of the two completed reduced fits, numerical seeds 410 and 411. +Selection uses the highest fitting weight with the smallest particle index as a tie-break, independently of future prediction errors. +All nonthermal parameters and each candidate physical history remain fixed. +Only burner radius, heating onset and heating width are inferred. + +The original independent uniform priors remain radius `[0.04, 0.25]`, onset `[5, 80]`, and width `[1, 40]`. +The bubbling sensor standard deviation remains 0.07. +The existing output model has no discrepancy process on bubbling, and none is added here. +The likelihood multiplies the Gaussian bubbling-reading factors through the designated prefix. +The initial reading supplies a parameter-independent constant and is explicitly omitted from the reported conditional normalizer. + +The two prefixes contain 132 and 224 actions. +The first never turns the burner on and retains the exact prior-retention control. +The second includes heating observations and is a separate development case. +Clean evaluator observations and noisy observations after action 224 do not enter this conditional inference. +The source density histories condition on observed robot motion causally; the native audit changes later joint observations and verifies that every prediction through action 224 remains unchanged. +This diagnostic conditions on a fixed physical prefix, and therefore does not integrate uncertainty about that prefix or update the other fitted quantities with the additional observations. + +## Integration and verification + +For a fixed physical prefix, distances to the burner partition the radius prior into intervals with identical heating decisions. +The radius integral is exact on this partition. +Onset and width use product Gauss-Legendre quadrature under their original normalized priors. +The producer groups reading residuals by accumulated integer heat, while the independent reader sums each Gaussian observation residual directly. +Both methods preserve the joint onset-width likelihood; marginal summaries do not imply independent posterior parameters. + +Native job `22755839` completed in 1:20, with 1,320 simulator actions. +It reproduces both complete archived native histories, predictions and carried memory in fresh worlds, and verifies prefix independence from altered later observations. +Reader `22755840` completed in six seconds, checking both prefixes for both selected candidates, the original observations, radius partitions, Gaussian likelihood probes and numerical moments. +The verified result checksum is `43edab5af557698d42eb65ff3a653484ab6b2dc4a543056e38966e4bf38ee810`. + +The initial 128- and 256-node rules disagree by up to 0.174 in parameter means on the informative prefix. +That motivates a separate precision experiment at 512, 1,024 and 2,048 nodes, with absolute tolerances of 0.005 for parameter means, standard deviations and the log normalizer. +It reuses the same verified conditional targets and performs no additional native simulation. +Jobs `22756314` and `22756315` completed in 15 and 12 seconds, respectively. +The independent reader confirms the higher-order integrals through direct residual sums. +Its verified result checksum is `94e57f245aa7d86cb49ee5c071d63b6afbe0a405f833fd256137dd32d2bb79a1`. + +| Comparison of quadrature orders | Maximum mean difference | Maximum standard-deviation difference | Log-normalizer difference | Declared precision screen | +|---|---:|---:|---:|---| +| 512 vs 1,024 | 0.001524 | 0.000992 | 0.008052 | Fails log-normalizer tolerance | +| 1,024 vs 2,048 | 0.000229 | 0.000163 | 0.001142 | Passes | + +This is empirical quadrature stability on this conditional problem, not a certified global integration bound or acceptance of the full-scene sampler. + +## Conditional outcome + +The two selected candidates induce the same heating-count history and scalar conditional likelihood. +They therefore produce identical conditional results; they are not two independent posterior-replication successes. +The higher-precision worker recognizes and reuses that identical calculation explicitly. + +| Parameter | 132-action mean | 132-action standard deviation | 224-action mean | 224-action standard deviation | +|---|---:|---:|---:|---:| +| Burner radius, metres | 0.14500 | 0.06062 | 0.14500 | 0.06062 | +| Heating onset, action steps | 42.50000 | 21.65064 | 30.12237 | 0.29841 | +| Heating ramp width, action steps | 20.50000 | 11.25833 | 5.71475 | 0.51485 | + +The all-off prefix retains all three normalized priors. +The longer prefix strongly constrains onset and width under the unchanged program and sensor likelihood. +In these selected histories, every radius in the original support induces the same heating decisions, so radius remains uniform even after heating. +This conditional invariance does not certify radius independence across all possible scenes. + +The result separates lack of informative observations from failure to compute a low-dimensional conditional distribution. +It supports testing heating-aware fitting without introducing a new bubbling-discrepancy term merely to match historical defaults. +It does not demonstrate held-out forecasting accuracy: no such score was computed here. + +## Next integration requirement + +A complete comparison on the longer prefix must define a new data identity and update the joint scene and parameter inference using all observations through that prefix. +The all-off independence certificate cannot be carried forward for onset and width. +Conditional thermal draws must preserve their dependence on each other and on the candidate scene and nonthermal parameters. +Do not substitute the conditional means in this table into every particle or independently sample these marginal standard deviations. +The original 132-action benchmark remains an extrapolation and prior-retention case. +Future prediction and live-agent acceptance still require their separate comparisons. + +Frozen scripts and reports are in `logs/uncertainty_boil_informative_heating_20260914` and `logs/uncertainty_boil_heating_quadrature_20260914`. diff --git a/docs/uncertainty/bridge-memory-recovery.md b/docs/uncertainty/bridge-memory-recovery.md index e9437d2fb..8cf632668 100644 --- a/docs/uncertainty/bridge-memory-recovery.md +++ b/docs/uncertainty/bridge-memory-recovery.md @@ -26,7 +26,7 @@ The failed first resume allocations and their cancelled dependent pipelines rema Array `22714037` resumes these checkpoints with the unchanged frozen fitter, probability model, sampler configuration, 16 ordered workers, and original evaluation budget. Each new allocation requests 128 GiB instead of 64 GiB on the original compute node. The underlying memory-growth cause has not been isolated; the larger allocation provides headroom for the remaining stages. -Readers `22714038` and `22714039` follow the fits and will independently replay the combined pre-interruption and resumed ledgers. +Readers `22714038` and `22714039` completed successfully, independently replaying the combined pre-interruption and resumed ledgers. The replacement forecast pipeline is frozen in `logs/uncertainty_bridge_resumed_forecasts_v2_20260914`. Its forecast and summary calculations are unchanged; source paths point to the resumed fits and new output directory. @@ -37,3 +37,13 @@ The old downstream jobs cannot proceed after their failed dependencies and are t The resumed fit report's elapsed-time field covers only its new allocation. Total inference cost must also include the interrupted allocation, initialization and recovery validation; do not quote the new elapsed-time field as total cost. The numerical and predictive gates remain open. + +## Completed forecasts and comparison + +Both resumed fits, forecasts `22714051` and `22714054`, readers `22714052` and `22714055`, and comparison `22714056` have completed. +All nine directly referenced comparison sources match their recorded hashes. +The two final populations retain one initial lineage each. +Both posterior forecasts and the incumbent selected point predict zero probability for the final clean geometric goal, which the assessment records as true. +The small aggregate goal Brier score of 1/586 reflects that this goal occurs at only one frame. +The maximum between-fit glue-probability gap is 0.1512; the comparison does not establish numerical or predictive acceptance. +See the [consolidated validation status](validation-status-20260914.md#bridge-completed-recovery) for the outcome and cumulative-cost caveat. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2dfbb0402..89ac3d102 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -6,6 +6,13 @@ The incumbent estimator remains the production default. ## Current stage +The [September 14 validation status](validation-status-20260914.md) consolidates the completed Boil and Bridge comparisons, Balloons reader recovery, and the independently verified informative-heating diagnostic. +Full-prefix replay is the required reference; arbitrary mid-trajectory restoration is not a migration gate. +The discrepancy policy now explicitly separates replay bugs, missing program mechanisms, uninformative data and residual model error. +The [informative-heating diagnostic](boil-informative-heating.md) preserves the thermal priors under the 132-action all-off prefix and constrains onset and width after 224 actions in two fixed candidate histories. +Native replay, later-observation isolation, independent direct Gaussian references and a higher-precision integration comparison pass. +This is conditional learnability evidence; a full-scene heating-aware target and held-out forecast remain separate requirements. + Stage 0 interface preservation is complete, with scripted behavior parity checked. Stage A has implemented probability and replay components, but physical support and numerical validation remain incomplete. The active work is Stage B offline comparison; Stage C live posterior use and Stage E retirement have not passed their evidence requirements. @@ -19,14 +26,15 @@ The reduced Boil target removes the three unobserved heating coordinates while p Native preflight `22717213` passes 56 assignments across eight source states, with 792 native actions and four contract guards. Independent reader `22717214` also passes, with 264 fresh native actions and a rejected placeholder coordinate. Both small fitting fixtures and readers now pass, including 4,224 fresh native actions per reader. -Full fits `22717256` and `22717258` are running with final-target verification queued. +Full fits `22717256` and `22717258` and their final-target verification have completed. The [reduced Boil forecast adapter](reduced-boil-forecasts.md) restores independently sampled thermal priors per complete trajectory while preserving retained joint rows and weights. Its twelve source guards, ten malformed-history checks and nonuniform density/variance reference pass in `22717364`. Native forecast fixture `22717373` completed 128 histories and 40,656 native actions. Independent reader `22717375` also passes, checking 304,128 joint factors and using 6,864 additional native actions, including two fresh complete histories. -Both full forecasts, their independent readers and the final comparison are dependency-queued behind successful fit and fixture verification. +Both full forecasts, their independent readers and final comparison `22717441` have completed. Comparison fixture `22717399` preserves all seven earlier controls while explicitly retaining both pending new outcomes. -No full reduced fit or new agent result is available yet. +The reduced fits retain one initial lineage each; bubbling RMSE is 0.30445 and 0.37039 versus the incumbent's 0.01915. +No new agent result or numerical/predictive acceptance follows from this completed comparison. The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. At one selected fitted state, removing future joint and velocity noise eliminates bursts in four continuations, but does not recover the goal; the other selected state does not improve. @@ -39,24 +47,28 @@ Array `22714577` now runs the two original numerical seeds with only the paramet The default-centered seed 621 fit `22714577_1` and its reader `22714629` have completed; all 64 final native targets and the checkpoint reproduce, with one surviving original lineage. This does not establish mixing or prediction quality. The paired seed 620 fit `22714577_0` and reader `22714628` have also completed, with the same final-target reproducibility and one surviving original lineage. -Both default-centered posterior forecasts remain pending or running; their comparison is incomplete. +Both default-centered posterior forecasts and comparison `22715120` have completed. +The changed guide improves box-speed RMSE but worsens box-height RMSE in both numerical fits; both retain one original lineage. The new weighted forecast adapter and independent reader `22715007`/`22715008` have passed, checking both broad and local proposal mappings, complete native prefixes, generation and density histories, and rejection of the old guide when decoding new particles. -Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` are dependency-queued behind the verified completed fits. +Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` completed after the verified fits. The comparison retains both earlier fits and the incumbent, including prediction metrics and computation costs; its fixture `22715119` passes while correctly leaving the new outcomes pending. The [parameter-dependence audit](balloons-selective-guidance.md) and independent reader `22715205`/`22715206` have completed all 46 native cases and twenty marginal checks. Changing five parameters separately or together leaves both tested fitting histories and scores exactly unchanged, while the original fits retain very narrow marginals for them. A selective guide with uniform proposals on those coordinates and 20% block refresh moves now passes native validation `22715307`, including 32 default-parity and 32 independent mixture-density checks. -Paired fits `22715405` are running with final-target readers `22715406`/`22715407` dependent on completion. +Paired fits `22715405` and final-target readers `22715406`/`22715407` have completed. The selective proposal's weighted future adapter and independent reader `22715541`/`22715542` now pass, including rejection of a decoder that omits the selective mask. -Forecasts `22715569`/`22715571`, independent readers and the six-population comparison `22715574` are dependency-queued. +Forecasts `22715569`/`22715571` completed, but their reader invocations omitted the required numerical index and failed before verification. +The cancelled comparison `22715574` is replaced by `22755277`, after invocation-recovery reader array `22755276`. The comparison fixture `22715573` passes its marginal and configuration guards, retaining the four pending forecasts explicitly. The comparison preserves the original and default-centered controls and reports parameter spread alongside future predictions. It retains the complete prior, likelihood and Metropolis correction and does not assume global parameter independence. The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. Both complete numerical prefixes and four fresh native targets reproduce exactly. -After fixing and checking a JSON-container comparison in the resume launcher, array `22714037` now resumes the same fits with 128 GiB per allocation, with replacement fit readers and the complete forecast/verification pipeline queued. +After fixing and checking a JSON-container comparison in the resume launcher, array `22714037` resumed the same fits with 128 GiB per allocation, followed by replacement fit readers and the complete forecast/verification pipeline. This is recovery from an infrastructure interruption; the incomplete fits are not model or agent outcomes. +The resumed fits and their full verification/forecast/comparison pipeline have now completed. +Both fits retain one initial lineage, and both posterior forecasts and the incumbent miss the final clean geometric goal. The [Fan fixture diagnostic](fan-static-fixtures.md) now isolates a practical forecast sensitivity after the speed-reflection result. Two verified scene-exchange audits use 4,884 native actions and retain three geometry-rejected candidates. diff --git a/docs/uncertainty/reduced-boil-forecasts.md b/docs/uncertainty/reduced-boil-forecasts.md index c4379cc18..e774add4f 100644 --- a/docs/uncertainty/reduced-boil-forecasts.md +++ b/docs/uncertainty/reduced-boil-forecasts.md @@ -11,9 +11,9 @@ Independent readers `22717219` and `22717221` replay the complete numerical ledg The verified source checksums are `8490860cad3bb895298296e523cb171ac3f737b56d931679107306bb322fc23f` and `e4b1360238f326cb6c990ceab3f3ac1c79d665a770da0d6739310b0ad6c826b7`. These deliberately small two-temperature fixtures establish implementation consistency, not posterior adequacy. -Full fits `22717256` and `22717258` are running on compute node `node1411` in `mit_preemptable`. +Full fits `22717256` and `22717258` completed on compute node `node1411` in `mit_preemptable`. They retain the original 32 particles, 64 temperatures, eight moves and 20,000-evaluation cap, with the three independent singleton blocks removed. -Their independent readers `22717257` and `22717259` are dependency-queued. +Their independent readers `22717257` and `22717259` completed successfully. The reports retain an unavailable numerical assessment until replication and budget-stability requirements are met. ## Restoring the independent priors in forecasts @@ -53,9 +53,8 @@ It uses 6,864 additional native actions. The verified forecast source checksum is `bbbb8324640aaf3a348a6888e4e21e6de44a539c8698477b2e489c023192bd35`. The reader reconstructs the 81-to-84 coordinate mapping separately, regenerates thermal draws from their recorded seeds, checks every native history and readout, and repeats complete generation and density histories. It independently computes the thermal density average and weighted prediction summaries. -Full forecasts `22717437` and `22717439` are dependency-queued behind the successful fixture reader and the corresponding full-fit reader. -Their independent readers are `22717438` and `22717440`, followed by final comparison `22717441`. -These gates precede full forecast execution. +Full forecasts `22717437` and `22717439`, independent readers `22717438` and `22717440`, and final comparison `22717441` completed successfully. +The [completed comparison](validation-status-20260914.md#boil-reduced-target-outcome) records the results and remaining numerical and predictive failures. ## Comparison and remaining acceptance @@ -68,4 +67,5 @@ The comparison reports fitting and forecast costs, between-fit prediction differ Comparison fixture `22717399` completed its checks and retained all seven original rows while correctly reporting both new full comparisons as pending. Missing results remain pending and cannot support an improvement claim. A completed mechanical comparison would still require numerical and predictive adequacy before live planning integration or retirement of the incumbent. -A separate fitting prefix containing heating remains necessary to test learning from informative data; it will not replace the original all-off extrapolation comparison. +A separate [conditional diagnostic](boil-informative-heating.md) using a 224-action prefix containing heating completed as `22755839`, with independent reader `22755840` and higher-precision follow-up `22756314`/`22756315`. +It holds selected physical histories and nonthermal parameters fixed and does not replace either the full-scene inference comparison or the original all-off extrapolation benchmark. diff --git a/docs/uncertainty/validation-status-20260914.md b/docs/uncertainty/validation-status-20260914.md new file mode 100644 index 000000000..1b9db9091 --- /dev/null +++ b/docs/uncertainty/validation-status-20260914.md @@ -0,0 +1,104 @@ +# Uncertainty replacement: validation status on September 14 + +The replacement remains in offline Stage B validation. +The production MB parameter fitter, execution estimator and decision rules are unchanged. +These experiments use fixed development programs and recorded interactions; numerical fitting seeds are not agent seeds. +No result below establishes a new solve rate or permission to retire the current estimator. + +## Agreed implementation scope + +Replay each candidate from initialization using the recorded low-level actions. +Arbitrary mid-trajectory restoration is an optional optimization and is not a migration requirement. +Initial-state uncertainty and candidate-specific simulator memory remain part of the inference problem. +Fix reproducible replay defects and missing program mechanisms before attributing residual mismatch to an explicitly declared discrepancy law. +Keep sensor noise fixed and test any discrepancy extension consistently in fitting and future generation. +The [proposal](simplification-proposal.md) records the full implementation decisions. + +## Completed comparisons + +| Domain | Latest evidence | Remaining difficulty | +|---|---|---| +| Domino | Both 64- and 128-particle comparisons are complete. | Agreement does not persist across numerical budgets; fixing the initial state is not a validated shortcut. | +| Fan | Fixed-fixture treatment reduces between-fit positional disagreement from 6.69 mm to 0.619 mm, with improved goal Brier scores on this recording. | Maximum future goal-probability disagreement remains 0.367; most sampled histories assign zero density to the recorded future. | +| Boil | Full reduced fits, forecasts, independent readers and comparison are complete. | The thermal prior is now represented correctly, but both fits retain one initial lineage and predictions remain worse than the incumbent. | +| Bridge | Both resumed fits, complete forecasts and independent readers are complete. | Both fits retain one initial lineage; all methods miss the final clean geometric goal, and the two posterior fits disagree on glue events. | +| Balloons | Both default-guided fits, forecasts and comparison are complete; selective fits and forecasts are complete. | Default-guided predictions remain inconsistent; selective forecast readers require invocation recovery before the full comparison can be assessed. | + +The earlier source-program audit found no learnable parameters in the saved Boil and Bridge programs from the original noisy sweep. +Those remain incomplete-model controls. +The positive Boil and Bridge comparisons instead use separately identified historical programs with learnable dynamics, including their learned defaults. +Their selected-point incumbent rows therefore carry historical information that an uninformative fitting prefix cannot recreate from a broad parameter prior. +This is a known comparison difference, not evidence that sensor noise should be enlarged or priors narrowed after seeing future scores. +See the [historical model controls](historical-model-controls.md) and [Boil incumbent control](boil-incumbent-control.md). + +## Boil reduced-target outcome + +The completed source is `logs/uncertainty_boil_reduced_comparison_20260914/full.json`. +All 42 referenced source hashes were checked after completion. +Both full fitting readers and both complete forecast readers passed. + +| Method | Numerical seed | Bubbling RMSE against clean readings | Final goal probability | Surviving initial lineages | +|---|---:|---:|---:|---:| +| Incumbent selected point | N/A | 0.01915 | 1.0000 | N/A | +| Original supported full target | 410 | 0.33856 | 0.5637 | 1 | +| Original supported full target | 411 | 0.31636 | 0.7691 | 1 | +| Reduced target with independent thermal priors | 410 | 0.30445 | 0.6836 | 1 | +| Reduced target with independent thermal priors | 411 | 0.37039 | 0.5156 | 1 | + +The dimension reduction preserves the original probability model but does not solve exploration of the retained scene and parameters. +It improves bubbling error in one fit and worsens it in the other. +The maximum future goal-probability gap between reduced fits is 0.1680. +One initial lineage is a warning about exploration, not by itself proof that every reported posterior moment is wrong. +The independent-replica and budget evidence remains necessary. + +The original 132-action prefix never activates the burner. +The three heating parameters should retain their prior under that prefix, even when a historically learned default predicts the future better. +New study `logs/uncertainty_boil_informative_heating_20260914` compares conditional heating inference at 132 and 224 actions using the same declared sensor variance and original thermal priors. +It selects two candidate scenes using fitting weight and a fixed index tie-break, holds nonthermal quantities fixed, and checks two quadrature budgets. +The longer prefix is a new development case; observations after action 224 and clean evaluator readings are excluded from its inference. +This study can establish conditional learnability, but cannot establish an adequate full-scene posterior or a live-agent improvement. +Native job `22755839` and independent reader `22755840` completed successfully on `mit_preemptable`. +The follow-up precision check `22756314` and independent reader `22756315` also completed; the 1,024/2,048-node comparison passes its declared moment and log-normalizer tolerances. +Conditional onset and width become concentrated at means 30.12237 and 5.71475, while radius remains uniform in these two selected histories. +The two histories give the same conditional heating target, not independent evidence of full-scene posterior agreement. +See the [informative-heating diagnostic](boil-informative-heating.md) for the original failed precision comparison, prior-retention control and remaining integration requirements. + +## Bridge completed recovery + +The completed source is `logs/uncertainty_bridge_resumed_forecasts_v2_20260914/paired-comparison.json`. +All nine directly referenced sources match their recorded hashes. +The original 64 GiB allocations failed with an out-of-memory condition; the checked 128 GiB continuations completed without changing the numerical target or resetting its budget. + +Numerical seeds 810 and 811 used 10,834 and 10,825 fitting evaluations, respectively, and each ends with one initial lineage. +All three methods, including the incumbent selected point, predict final goal probability zero. +The clean assessment contains the goal at the final frame, while evaluating the same geometric predicate on noisy poses misses it. +Their identical goal Brier score of 1/586 therefore conceals a consequential final-frame error rather than demonstrating success. +The maximum between-fit glue-probability gap is 0.1512, and maximum per-coordinate positional RMS disagreement is 8.49 mm. +The future-density calculation has 26/32 zero-density particles in seed 810 and none in seed 811. +These findings keep predictive and numerical acceptance open. + +The final report's fitting time describes the resumed allocation. +Compute accounting must also retain each original 2:51:51 allocation, recovery checks, initialization, forecast generation and independent verification. +Do not interpret resumed elapsed time as total fitting cost. + +## Balloons invocation recovery + +Selective forecast jobs `22715569` and `22715571` completed, but readers `22715570` and `22715572` failed before their Python verifier started. +Their launcher requires both the report path and numerical index; its submission omitted the second argument and shell expansion failed with `$2: unbound variable`. +The dependent comparison `22715574` was cancelled. +This is a verification-launch failure, not a failed fit, prediction, or agent seed. + +The exact failing expansion was reproduced in `logs/uncertainty_balloons_selective_reader_recovery_20260914/reproduction.json`. +Recovery array `22755276` explicitly supplies report and index to the unchanged frozen launcher and verifier. +Comparison `22755277` depends on successful completion of both readers. +No completed fit or forecast is regenerated, and old frozen source files are not edited. +The recovery manifest pins the original reports, scripts and invocation wrapper. + +## Next acceptance work + +Read and verify the recovered Balloons comparison when it completes. +Use the completed informative-heating diagnostic to construct a separately identified heating-aware joint target, retaining correlations and the original all-off control. +Use their results to choose a numerical or model change rather than repeat the same unstable fits at a larger budget without a specific hypothesis. +Existing conditional scalar-discrepancy, joint-variance and physical-transition experiments remain distinct model choices. +Any new law motivated by these already-inspected future recordings needs a new untouched evaluation before acceptance. +Live posterior integration, matched five-domain continual evaluation, and retirement of the current parameter fitter remain required later stages. From dbc1a3a65db4a6159a1b2cf9ab647bdfb05ceaed Mon Sep 17 00:00:00 2001 From: Yichao Liang Date: Mon, 14 Sep 2026 16:04:06 -0400 Subject: [PATCH 94/94] Type and lint the continual comparison figure renderer --- .../plotting/plot_continual_comparisons.py | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/scripts/plotting/plot_continual_comparisons.py b/scripts/plotting/plot_continual_comparisons.py index 21c45a72f..f3325db3c 100644 --- a/scripts/plotting/plot_continual_comparisons.py +++ b/scripts/plotting/plot_continual_comparisons.py @@ -10,15 +10,19 @@ import json import sys from pathlib import Path +from typing import Any, Dict, List import matplotlib matplotlib.use('Agg') +# pylint: disable=wrong-import-position import matplotlib.pyplot as plt import numpy as np from matplotlib.ticker import FuncFormatter, MaxNLocator -ROOT = Path(__file__).resolve().parents[2] +# pylint: enable=wrong-import-position + +ROOT = Path(__file__).resolve().parent.parent.parent DOMAINS = ['Boil', 'Domino', 'Fan', 'Bridge', 'Balloons'] ARMS = [ 'MB', 'MF', 'agent_continual_program_world_model', @@ -37,11 +41,12 @@ ] -def capture(paper, target): +def capture(paper: Path, target: Path) -> None: """Verify every selected final scorecard before freezing plot inputs.""" sys.path.insert(0, str(paper / 'scripts')) spec = importlib.util.spec_from_file_location( 'paper_artifacts', paper / 'scripts/build_artifacts.py') + assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) rows, _ = module.verified_reported_rows() @@ -49,7 +54,7 @@ def capture(paper, target): ROOT / 'docs/comparisons/continual-results.json', ROOT / 'docs/comparisons/bridge-three-span-results.json' ] - sources = [] + sources: List[Dict[str, str]] = [] for index, report in enumerate(reports): document = json.loads(report.read_text()) for row in document['rows']: @@ -101,24 +106,30 @@ def capture(paper, target): 'generated_by': 'scripts/plotting/plot_continual_comparisons.py', 'policy': - 'Preserve paper MB (3) and historical MF (2); add six comparison arms (3 each). Bridge uses three-span integrity-fixed cohort. Steps: whole-run successes only; solve: all levels; resets: all finished runs.', + ('Preserve paper MB (3) and historical MF (2); add six comparison ' + 'arms (3 each). Bridge uses three-span integrity-fixed cohort. ' + 'Steps: whole-run successes only; solve: all levels; resets: all ' + 'finished runs.'), 'sources': sources, 'records': rows, - 'caveats': [ - 'Cohorts differ in observation handling and agent runtime; not a matched causal ablation.', - 'Only new Bridge standalone permits engine imports; other domains retain stricter historical prompt.', - 'Some historical standalone agents did not use a model; uncertainty arms include known custom uncertainty checks.', - 'Original no-fitting Bridge seed 0 retained; post-completion scheduler repeat excluded.', - 'Balloons is original non-hatch with historical instantaneous goal.' - ] + 'caveats': + [('Cohorts differ in observation handling and agent runtime; not ' + 'a matched causal ablation.'), + ('Only new Bridge standalone permits engine imports; other ' + 'domains retain stricter historical prompt.'), + ('Some historical standalone agents did not use a model; ' + 'uncertainty arms include known custom uncertainty checks.'), + ('Original no-fitting Bridge seed 0 retained; post-completion ' + 'scheduler repeat excluded.'), + 'Balloons is original non-hatch with historical instantaneous goal.'] } target.parent.mkdir(parents=True, exist_ok=True) target.write_text(json.dumps(payload, indent=2) + '\n') -def render(snapshot, output): +def render(snapshot: Path, output: Path) -> None: """Means plus individual seed values, without small-sample CI claims.""" data = json.loads(snapshot.read_text()) rows = data['records'] @@ -130,7 +141,7 @@ def render(snapshot, output): 'svg.hashsalt': 'continual-comparisons' }) fig, axes = plt.subplots(3, 5, figsize=(12.8, 8.7), sharey=True) - summary = [] + summary: List[Dict[str, Any]] = [] for col, domain in enumerate(DOMAINS): for metric, field in enumerate(['solve', 'steps', 'resets']): ax = axes[metric, col] @@ -231,7 +242,8 @@ def render(snapshot, output): json.dumps(summary, indent=2) + '\n') -def main(): +def main() -> None: + """Optionally capture a snapshot, then render the figure.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--paper-root', type=Path,