diff --git a/docs/comparisons/bridge-balloons-integrity-results.json b/docs/comparisons/bridge-balloons-integrity-results.json new file mode 100644 index 0000000000..31162a9050 --- /dev/null +++ b/docs/comparisons/bridge-balloons-integrity-results.json @@ -0,0 +1,135 @@ +{ + "generated_by": "/home/ycliang/predicators/logs/bridge_balloons_integrity_20260912/report.py", + "updated_at": "2026-09-13T09:07:21.054343+00:00", + "source_commit": "cad1000f92c5e5e94714ae613426b89761823ee0", + "rows": [ + { + "domain": "Bridge", + "approach": "agent_continual", + "arm": "MB + uncertainty", + "seed": 0, + "job": "22646173_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_span_transfer_r1/seed0/run_20260912_165804/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2939, + "resets": 0, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 0, + "job": "22646174_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_span_transfer_r1/seed0/run_20260912_165804/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2634, + "resets": 0, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 0, + "job": "22649185_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 771, + "resets": 1, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 1, + "job": "22649185_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed1/run_20260912_180112/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 554, + "resets": 1, + "end_reason": "all_levels_won", + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_model_free", + "arm": "MF direct coding agent", + "seed": 2, + "job": "22649185_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed2/run_20260912_180113/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 829, + "resets": 2, + "end_reason": "all_levels_won", + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "MB + uncertainty", + "finished": 1, + "expected": 1, + "successes": 1, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2939, + "steps_n": 1, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "MF direct coding agent", + "finished": 1, + "expected": 1, + "successes": 1, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2634, + "steps_n": 1, + "resets_mean_finished": 0 + }, + { + "domain": "Balloons", + "arm": "MF direct coding agent", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 718, + "steps_n": 3, + "resets_mean_finished": 1.3333333333333333 + } + ] +} diff --git a/docs/comparisons/bridge-balloons-integrity-results.md b/docs/comparisons/bridge-balloons-integrity-results.md new file mode 100644 index 0000000000..c140dd41b9 --- /dev/null +++ b/docs/comparisons/bridge-balloons-integrity-results.md @@ -0,0 +1,25 @@ + +# Bridge transfer and sustained-hover Balloons pilots + +Updated 2026-09-13T09:07:21.054343+00:00. + +Bridge: one seed per MB/MF arm, three-block train span and four-block test span. Balloons: three MF seeds, original non-hatch task distribution, 25-step sustained hovering. Historical cohorts and other comparisons remain separate. +Whole-run success means winning every training and test level. +Solve rates and mean resets use finished agent seeds only. +Mean steps uses only whole-run successes, with the qualifying count shown. +Unfinished, setup, and infrastructure outcomes are excluded from averages. +These tables are refreshed from scorecards and Slurm accounting. + +| Domain | Arm | Finished | Whole-run solve | Level solve | Mean successful steps (n) | Mean resets | +|---|---|---:|---:|---:|---:|---:| +| Bridge | MB + uncertainty | 1/1 | 100.0% (1/1) | 100.0% | 2,939 (n=1) | 0 | +| Bridge | MF direct coding agent | 1/1 | 100.0% (1/1) | 100.0% | 2,634 (n=1) | 0 | +| Balloons | MF direct coding agent | 3/3 | 100.0% (3/3) | 100.0% | 718 (n=3) | 1.3 | + +| Domain | Arm | Seed | Status | Wins | Steps | Resets | Job | Scorecard | +|---|---|---:|---|---:|---:|---:|---|---| +| Bridge | MB + uncertainty | 0 | Finished | 2/2 | 2,939 | 0 | 22646173_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_span_transfer_r1/seed0/run_20260912_165804/scorecard.json) | +| Bridge | MF direct coding agent | 0 | Finished | 2/2 | 2,634 | 0 | 22646174_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_span_transfer_r1/seed0/run_20260912_165804/scorecard.json) | +| Balloons | MF direct coding agent | 0 | Finished | 3/3 | 771 | 1 | 22649185_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json) | +| Balloons | MF direct coding agent | 1 | Finished | 3/3 | 554 | 1 | 22649185_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed1/run_20260912_180112/scorecard.json) | +| Balloons | MF direct coding agent | 2 | Finished | 3/3 | 829 | 2 | 22649185_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed2/run_20260912_180113/scorecard.json) | diff --git a/docs/comparisons/bridge-three-span-results.json b/docs/comparisons/bridge-three-span-results.json new file mode 100644 index 0000000000..91529fafc6 --- /dev/null +++ b/docs/comparisons/bridge-three-span-results.json @@ -0,0 +1,423 @@ +{ + "generated_by": "/home/ycliang/predicators/logs/bridge_three_span_20260914/report.py", + "updated_at": "2026-09-14T18:09:00.815253+00:00", + "source_commit": "091d8c5db11f8944662e7a8af28a070905886604", + "rows": [ + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22717645_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed0/run_20260914_032501/scorecard.json", + "finished": true, + "success": false, + "wins": 0, + "levels": 2, + "steps": 12964, + "resets": 17, + "end_reason": "agent_ended", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22717645_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed1/run_20260914_032851/scorecard.json", + "finished": true, + "success": false, + "wins": 0, + "levels": 2, + "steps": 19324, + "resets": 16, + "end_reason": "agent_ended", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22717645_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed2/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2724, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22717646_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed0/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1921, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22717646_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed1/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1706, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22717646_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed2/run_20260914_032502/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1851, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22717647_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed0/run_20260914_094920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2519, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22717647_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed1/run_20260914_094923/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 3158, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22717647_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed2/run_20260914_094920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2505, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22717650_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed0/run_20260914_041919/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2307, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22717650_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed1/run_20260914_041922/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2843, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22717650_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed2/run_20260914_041928/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2210, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22717651_0", + "job_state": "CANCELLED by 215151", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed0/run_20260914_110412/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2000, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22717651_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed1/run_20260914_110412/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 5766, + "resets": 2, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22717651_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed2/run_20260914_110417/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 4167, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22717652_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed0/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2327, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22717652_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed1/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2252, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22717652_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed2/run_20260914_052356/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2482, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.3333333333333333, + "steps_mean_success": 2724, + "steps_n": 1, + "resets_mean_finished": 11 + }, + { + "domain": "Bridge", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 1826, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2727.3333333333335, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + }, + { + "domain": "Bridge", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2453.3333333333335, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Bridge", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 3977.6666666666665, + "steps_n": 3, + "resets_mean_finished": 1 + }, + { + "domain": "Bridge", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 2353.6666666666665, + "steps_n": 3, + "resets_mean_finished": 0 + } + ] +} diff --git a/docs/comparisons/bridge-three-span-results.md b/docs/comparisons/bridge-three-span-results.md new file mode 100644 index 0000000000..1626a0748e --- /dev/null +++ b/docs/comparisons/bridge-three-span-results.md @@ -0,0 +1,43 @@ + +# Three-span Bridge comparisons + +Updated 2026-09-14T18:09:00.815253+00:00. + +Three span blocks at training and test; seeds 0, 1, 2 per arm; 5 mm position noise and 0.02 rad orientation noise. +Shared information-integrity fixes retained; experimental rigid assemblies and transport previews absent. +Separate cohort from four-span results and historical results before information-integrity fixes. +Standalone agents may build their own physics simulators, including PyBullet; no prepared scene simulator is supplied. +No harness fitting permits agent-written fitting. No-explicit-uncertainty results require review for custom agent uncertainty checks. +Whole-run solve rate and mean resets use finished agent seeds; mean steps uses only whole-run successes, with n. +No-fitting seed 0 completed in 2000 steps before a scheduler restart repeated it (2138 steps, 2/2 wins, 0 resets); retain the original run and exclude the repeat. The pending further restart was cancelled. +Unfinished and infrastructure outcomes are excluded from averages. + +| Domain | Arm | Finished | Whole-run solve | Level solve | Mean successful steps (n) | Mean resets | +|---|---|---:|---:|---:|---:|---:| +| Bridge | 3. Standalone program | 3/3 | 33.3% (1/3) | 33.3% | 2,724 (n=1) | 11 | +| Bridge | 4. Oracle dynamics | 3/3 | 100.0% (3/3) | 100.0% | 1,826 (n=3) | 0 | +| Bridge | 5. Oracle scene | 3/3 | 100.0% (3/3) | 100.0% | 2,727.3 (n=3) | 0.3 | +| Bridge | 6. Zero-shot model | 3/3 | 100.0% (3/3) | 100.0% | 2,453.3 (n=3) | 0 | +| Bridge | 7. No harness fitting | 3/3 | 100.0% (3/3) | 100.0% | 3,977.7 (n=3) | 1 | +| Bridge | 8. No explicit uncertainty | 3/3 | 100.0% (3/3) | 100.0% | 2,353.7 (n=3) | 0 | + +| Domain | Arm | Seed | Status | Wins | Steps | Resets | Job | Scorecard | +|---|---|---:|---|---:|---:|---:|---|---| +| Bridge | 3. Standalone program | 0 | Finished | 0/2 | 12,964 | 17 | 22717645_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed0/run_20260914_032501/scorecard.json) | +| Bridge | 3. Standalone program | 1 | Finished | 0/2 | 19,324 | 16 | 22717645_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed1/run_20260914_032851/scorecard.json) | +| Bridge | 3. Standalone program | 2 | Finished | 2/2 | 2,724 | 0 | 22717645_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_program_world_model/bridge-continual_standalone_three_span_r1/seed2/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 0 | Finished | 2/2 | 1,921 | 0 | 22717646_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed0/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 1 | Finished | 2/2 | 1,706 | 0 | 22717646_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed1/run_20260914_032502/scorecard.json) | +| Bridge | 4. Oracle dynamics | 2 | Finished | 2/2 | 1,851 | 0 | 22717646_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/bridge-continual_oracle_dynamics_three_span_r1/seed2/run_20260914_032502/scorecard.json) | +| Bridge | 5. Oracle scene | 0 | Finished | 2/2 | 2,519 | 0 | 22717647_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed0/run_20260914_094920/scorecard.json) | +| Bridge | 5. Oracle scene | 1 | Finished | 2/2 | 3,158 | 1 | 22717647_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed1/run_20260914_094923/scorecard.json) | +| Bridge | 5. Oracle scene | 2 | Finished | 2/2 | 2,505 | 0 | 22717647_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_oracle_scene/bridge-continual_oracle_scene_three_span_r1/seed2/run_20260914_094920/scorecard.json) | +| Bridge | 6. Zero-shot model | 0 | Finished | 2/2 | 2,307 | 0 | 22717650_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed0/run_20260914_041919/scorecard.json) | +| Bridge | 6. Zero-shot model | 1 | Finished | 2/2 | 2,843 | 0 | 22717650_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed1/run_20260914_041922/scorecard.json) | +| Bridge | 6. Zero-shot model | 2 | Finished | 2/2 | 2,210 | 0 | 22717650_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_zero_shot/bridge-continual_zero_shot_three_span_r1/seed2/run_20260914_041928/scorecard.json) | +| Bridge | 7. No harness fitting | 0 | Finished | 2/2 | 2,000 | 0 | 22717651_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed0/run_20260914_110412/scorecard.json) | +| Bridge | 7. No harness fitting | 1 | Finished | 2/2 | 5,766 | 2 | 22717651_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed1/run_20260914_110412/scorecard.json) | +| Bridge | 7. No harness fitting | 2 | Finished | 2/2 | 4,167 | 1 | 22717651_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_fitting/bridge-continual_no_fitting_three_span_r1/seed2/run_20260914_110417/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 0 | Finished | 2/2 | 2,327 | 0 | 22717652_0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed0/run_20260914_052356/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 1 | Finished | 2/2 | 2,252 | 0 | 22717652_1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed1/run_20260914_052356/scorecard.json) | +| Bridge | 8. No explicit uncertainty | 2 | Finished | 2/2 | 2,482 | 0 | 22717652_2 | [scorecard](/home/ycliang/predicators/logs/agent_continual_no_uncertainty/bridge-continual_no_uncertainty_three_span_r1/seed2/run_20260914_052356/scorecard.json) | diff --git a/docs/comparisons/continual-results.json b/docs/comparisons/continual-results.json new file mode 100644 index 0000000000..7c3834308c --- /dev/null +++ b/docs/comparisons/continual-results.json @@ -0,0 +1,2231 @@ +{ + "generated_by": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/report.py", + "updated_at": "2026-09-13T13:48:42.462889+00:00", + "source_commit": "59336397069afd098f132aac1deaca5c9cba73e9", + "rows": [ + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642703_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 540, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642703_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_160623/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 535, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642703_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/boil-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 573, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642704_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 350, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642704_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_160625/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 174, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642704_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/balloons-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_160637/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 613, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642705_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_170526/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 622, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642705_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_170523/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 276, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642705_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/fan-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_170525/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 328, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642706_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642706_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642706_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 0, + "job": "22642707_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed0/run_20260912_173725/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 362, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 1, + "job": "22642707_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed1/run_20260912_173956/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 307, + "resets": 0, + "end_reason": "level_lost", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_dynamics", + "arm": "4. Oracle dynamics", + "seed": 2, + "job": "22642707_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_dynamics/domino_high_friction_turn-continual_oracle_dynamics_noisy_r1/seed2/run_20260912_173920/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 328, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642708_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed0/run_20260912_165320/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 758, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642708_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed1/run_20260912_165600/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 802, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642708_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/boil-continual_oracle_scene_noisy_r1/seed2/run_20260912_165331/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1277, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642709_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed0/run_20260912_190626/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 1594, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642709_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed1/run_20260912_203736/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 335, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642709_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/balloons-continual_oracle_scene_noisy_r1/seed2/run_20260912_205841/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 470, + "resets": 0, + "end_reason": "level_lost", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642710_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed0/run_20260912_173627/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 715, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642710_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed1/run_20260912_173629/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 281, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642710_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/fan-continual_oracle_scene_noisy_r1/seed2/run_20260912_173623/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 545, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642711_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642711_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642711_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 0, + "job": "22642712_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed0/run_20260912_183323/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 262, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 1, + "job": "22642712_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed1/run_20260912_183337/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 300, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_oracle_scene", + "arm": "5. Oracle scene", + "seed": 2, + "job": "22642712_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_oracle_scene/domino_high_friction_turn-continual_oracle_scene_noisy_r1/seed2/run_20260912_183337/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 395, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642713_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed0/run_20260912_215028/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 517, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642713_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed1/run_20260912_211305/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 633, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642713_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/boil-continual_zero_shot_noisy_r1/seed2/run_20260912_213833/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1022, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642714_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed0/run_20260912_205352/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 400, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642714_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed1/run_20260912_210259/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 436, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642714_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/balloons-continual_zero_shot_noisy_r1/seed2/run_20260912_210028/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 377, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642715_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed0/run_20260912_221531/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 374, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642715_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed1/run_20260912_214954/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 446, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642715_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/fan-continual_zero_shot_noisy_r1/seed2/run_20260912_223336/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 533, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642716_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642716_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642716_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 0, + "job": "22642717_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed0/run_20260912_223957/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 417, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 1, + "job": "22642717_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed1/run_20260912_223139/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 526, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_zero_shot", + "arm": "6. Zero-shot model", + "seed": 2, + "job": "22642717_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_zero_shot/domino_high_friction_turn-continual_zero_shot_noisy_r1/seed2/run_20260913_012942/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 334, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642718_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed0/run_20260912_215805/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 542, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642718_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed1/run_20260912_215733/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 761, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642718_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/boil-continual_no_fitting_noisy_r1/seed2/run_20260912_214921/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1071, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642719_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed0/run_20260913_024607/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 877, + "resets": 3, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642719_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed1/run_20260913_032344/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 631, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642719_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/balloons-continual_no_fitting_noisy_r1/seed2/run_20260913_025446/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 342, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written dynamics fitting observed" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642720_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed0/run_20260912_223138/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 330, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642720_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed1/run_20260912_223130/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 420, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Fan", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642720_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/fan-continual_no_fitting_noisy_r1/seed2/run_20260912_223933/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 282, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642721_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642721_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642721_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 0, + "job": "22642722_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed0/run_20260913_012856/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 400, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 1, + "job": "22642722_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed1/run_20260912_225924/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 394, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_fitting", + "arm": "7. No harness fitting", + "seed": 2, + "job": "22642722_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_fitting/domino_high_friction_turn-continual_no_fitting_noisy_r1/seed2/run_20260912_230435/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 511, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "protocol_status": "agent-written fitting not fully audited" + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642723_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed0/run_20260913_040903/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1093, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642723_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed1/run_20260913_042604/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 547, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642723_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/boil-continual_no_uncertainty_noisy_r1/seed2/run_20260913_040746/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 565, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642724_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed0/run_20260913_035506/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 243, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642724_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed1/run_20260913_030534/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 271, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Balloons", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642724_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/balloons-continual_no_uncertainty_noisy_r1/seed2/run_20260913_033451/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 268, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642725_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed0/run_20260913_045206/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 346, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642725_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed1/run_20260913_045849/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 289, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642725_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/fan-continual_no_uncertainty_noisy_r1/seed2/run_20260913_044710/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 302, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642726_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642726_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642726_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 0, + "job": "22642727_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed0/run_20260913_051311/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 559, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 1, + "job": "22642727_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed1/run_20260913_051832/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 272, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Domino", + "approach": "agent_continual_no_uncertainty", + "arm": "8. No explicit uncertainty", + "seed": 2, + "job": "22642727_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_no_uncertainty/domino_high_friction_turn-continual_no_uncertainty_noisy_r1/seed2/run_20260913_051638/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 384, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "uncertainty_protocol_status": "agent-written uncertainty sweep confirmed" + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642728_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed0/run_20260913_050000/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 2846, + "resets": 7, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642728_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed1/run_20260913_034701/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 1782, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Boil", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642728_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/boil-continual_standalone_noisy_r1/seed2/run_20260913_040647/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 886, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642729_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed0/run_20260913_075911/scorecard.json", + "finished": true, + "success": true, + "wins": 3, + "levels": 3, + "steps": 403, + "resets": 1, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642729_1", + "job_state": "RUNNING", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed1/run_20260913_083354/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 558, + "resets": 1, + "end_reason": "agent_ended", + "queue_reason": "None", + "superseded": false, + "quota_message": null + }, + { + "domain": "Balloons", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642729_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/balloons-continual_standalone_noisy_r1/seed2/run_20260913_082421/scorecard.json", + "finished": true, + "success": false, + "wins": 2, + "levels": 3, + "steps": 788, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642730_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed0/run_20260913_074102/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 735, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642730_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed1/run_20260913_065042/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 890, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Fan", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642730_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/fan-continual_standalone_noisy_r1/seed2/run_20260913_050240/scorecard.json", + "finished": true, + "success": true, + "wins": 2, + "levels": 2, + "steps": 547, + "resets": 0, + "end_reason": "all_levels_won", + "queue_reason": "", + "superseded": false, + "quota_message": null, + "standalone_model_use_status": "no saved world_model.py or supplied prediction-interface call mentions" + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642731_0", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642731_1", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Bridge", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642731_2", + "job_state": "CANCELLED by 215151", + "status": "Superseded by span-transfer cohort", + "scorecard": null, + "finished": false, + "success": false, + "wins": null, + "levels": 2, + "steps": null, + "resets": null, + "end_reason": null, + "queue_reason": "", + "superseded": true, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 0, + "job": "22642732_0", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed0/run_20260913_080352/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 702, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 1, + "job": "22642732_1", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed1/run_20260913_073131/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 484, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + }, + { + "domain": "Domino", + "approach": "agent_continual_program_world_model", + "arm": "3. Standalone program", + "seed": 2, + "job": "22642732_2", + "job_state": "COMPLETED", + "status": "Finished", + "scorecard": "/home/ycliang/predicators/logs/agent_continual_program_world_model/domino_high_friction_turn-continual_standalone_noisy_r1/seed2/run_20260913_062933/scorecard.json", + "finished": true, + "success": false, + "wins": 1, + "levels": 2, + "steps": 459, + "resets": 0, + "end_reason": "agent_ended", + "queue_reason": "", + "superseded": false, + "quota_message": null + } + ], + "aggregate": [ + { + "domain": "Bridge", + "arm": "3. Standalone program", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "4. Oracle dynamics", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "5. Oracle scene", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "6. Zero-shot model", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Bridge", + "arm": "7. No harness fitting", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Bridge", + "arm": "8. No explicit uncertainty", + "finished": 0, + "expected": 0, + "originally_planned": 3, + "successes": 0, + "solve_rate": null, + "level_solve_rate": null, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": null + }, + { + "domain": "Fan", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 724, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 408.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 513.6666666666666, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 451, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Fan", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 344, + "steps_n": 3, + "resets_mean_finished": 0, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Fan", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 312.3333333333333, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 0, + "solve_rate": 0.0, + "level_solve_rate": 0.5, + "steps_mean_success": null, + "steps_n": 0, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 345, + "steps_n": 2, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 319, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 425.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Domino", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 455.5, + "steps_n": 2, + "resets_mean_finished": 0, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Domino", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8333333333333334, + "steps_mean_success": 328, + "steps_n": 2, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 1838, + "steps_n": 3, + "resets_mean_finished": 2.3333333333333335 + }, + { + "domain": "Boil", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 549.3333333333334, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 945.6666666666666, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 724, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Boil", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 791.3333333333334, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Boil", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 735, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + }, + { + "domain": "Balloons", + "arm": "3. Standalone program", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.7777777777777778, + "steps_mean_success": 403, + "steps_n": 1, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "4. Oracle dynamics", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 379, + "steps_n": 3, + "resets_mean_finished": 0 + }, + { + "domain": "Balloons", + "arm": "5. Oracle scene", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 1, + "solve_rate": 0.3333333333333333, + "level_solve_rate": 0.7777777777777778, + "steps_mean_success": 335, + "steps_n": 1, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "6. Zero-shot model", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8888888888888888, + "steps_mean_success": 388.5, + "steps_n": 2, + "resets_mean_finished": 0.6666666666666666 + }, + { + "domain": "Balloons", + "arm": "7. No harness fitting", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 2, + "solve_rate": 0.6666666666666666, + "level_solve_rate": 0.8888888888888888, + "steps_mean_success": 609.5, + "steps_n": 2, + "resets_mean_finished": 1.6666666666666667, + "protocol_status": "No harness fitting; agent-written fitting may occur, so this does not isolate all numerical estimation" + }, + { + "domain": "Balloons", + "arm": "8. No explicit uncertainty", + "finished": 3, + "expected": 3, + "originally_planned": 3, + "successes": 3, + "solve_rate": 1.0, + "level_solve_rate": 1.0, + "steps_mean_success": 260.6666666666667, + "steps_n": 3, + "resets_mean_finished": 0.3333333333333333 + } + ], + "active_expected": 72, + "superseded_seeds": 18, + "protocol_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/no-fitting-protocol-audit.json", + "uncertainty_protocol_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/no-uncertainty-protocol-audit.json", + "standalone_model_use_audit": "/orcd/home/002/ycliang/predicators/logs/continual_comparisons_20260912/standalone-model-use-audit-20260913.json" +} diff --git a/docs/comparisons/domain-contract-validation.md b/docs/comparisons/domain-contract-validation.md new file mode 100644 index 0000000000..4da8e2954b --- /dev/null +++ b/docs/comparisons/domain-contract-validation.md @@ -0,0 +1,55 @@ +# Cross-domain comparison tool validation + +The standalone and zero-shot action-boundary tests now use the real noisy cohort configurations in all five domains. +Their earlier tests exercised these contracts only in Boil. +The other four comparison methods already had five-domain play-tool coverage. + +All ten extended checks passed in array `22650730`. +Type checking and lint passed in job `22650731`. +The tests use scripted agent tool calls and do not call Claude or produce solve-rate outcomes. + +| Domain | Runtime | Check job | Tests passed | +|---|---|---|---:| +| Boil | `59336397069a` | 22650730_0 | 2 | +| Bridge, 3-to-4 span | `73b5e517bf18` | 22650730_1 | 2 | +| Fan | `59336397069a` | 22650730_2 | 2 | +| Domino | `59336397069a` | 22650730_3 | 2 | +| Original Balloons | `59336397069a` | 22650730_4 | 2 | + +The standalone check edits its program during a real continual play session and verifies that subsequent predictions load the changed program. +It checks the supplied model interface has no engine instance or base-simulator reference files, rejects engine-diagnostic rollout modes, and records a charged real action. +This is an interface check, not an exhaustive sandbox security audit. + +The zero-shot check refuses the first charged action without a model, seals a valid model before taking that action, and rejects later source edits and fitting routes. +It restores the sealed source from a saved approach state and verifies that the next action is charged correctly. +The separate-process standalone resume test remains covered by the earlier 25-test suite in job `22648953`. + +An immutable copy of the extended test module was run against each cohort's existing frozen production package. +The runner checks the imported package path and test-file digest before executing the tests. +No frozen experiment code was edited and no agent run was restarted for this validation. +The source paths, digests, job records, and verified pass summaries are recorded in `/home/ycliang/predicators/logs/comparison_domain_contracts_20260912/validated.json` and its adjacent validation manifest. + + +## Standalone package access + +The experiment environment includes PyBullet, NumPy, and SciPy. +The standalone approach omits base-simulator reference files, removes the engine-backed evaluator, and routes supplied predictions through the learned skill-transition program. +The original frozen experiment prompt forbids importing an environment or physics engine inside a prediction. +The import guard screens hidden `predicators.envs` and `predicators.ground_truth_models` modules; it does not prohibit the `pybullet` package itself. +Thus the implemented isolation covers the supplied prediction interface, not all physics packages accessible to arbitrary agent code. +A scan of the two started Boil standalone runs on 2026-09-13 found PyBullet import matches in copied skill-controller references and a transcript displaying one such reference. +That scan did not find an agent-written PyBullet simulator, but it is not an exhaustive proof of engine-free behavior or import isolation. +No experiment runtime or package installation was changed for this audit. + + +## Physics libraries permitted, 2026-09-13 + +The user clarified that the standalone agent may use PyBullet or other available simulation libraries. +The intended comparison withholds a prepared scene and base simulator, while allowing the agent to construct its own predictive world from public observations and recorded interactions. +The development prompt now states this explicitly and retains the prohibition on inspecting the task environment, ground-truth mechanisms, or live simulator state. +The original frozen experiments retain their stricter prompt and must not be presented as a cohort run under the revised instruction. +Replacement configurations cover twelve original non-Bridge seeds and three Bridge span-transfer seeds, keeping their respective domain runtimes and task settings. +Whether to replace the existing standalone cohort or retain it as the stricter comparison is pending the user's preference; no replacement experiment has been submitted. +Compute array 22672397 passed all ten domain checks for agent-owned PyBullet predictions, live-environment isolation, and ordinary program predictions. +Static job 22672446 passed mypy and pylint for both changed Python files after correcting a test type annotation reported by 22672398. +Their manifests and outputs are under `/home/ycliang/predicators/logs/standalone_engine_contract_20260913/`; these are mechanical checks, not agent solve-rate seeds. diff --git a/docs/comparisons/fixed-balloons-mf-notes.md b/docs/comparisons/fixed-balloons-mf-notes.md new file mode 100644 index 0000000000..c9fc413037 --- /dev/null +++ b/docs/comparisons/fixed-balloons-mf-notes.md @@ -0,0 +1,46 @@ +# Fixed Balloons: MF results and seed-0 analysis + +All three intended MF seeds solved every level with the sustained-hover goal. +The mean charged cost is 718 steps over three whole-run successful seeds, and mean resets is 1.33. +There is no matched MB rerun under this changed goal. +The current cohort table is [Bridge/Balloons integrity results](bridge-balloons-integrity-results.md). +Its frozen runtime is `cad1000f92c5e5e94714ae613426b89761823ee0`, with the original task distribution, 1 cm position noise, 0.02 rad orientation noise, and zero scalar-reading noise. +The goal requires 25 complete consecutive environment intervals in the band below 0.01 m/s. +Earlier instantaneous-goal Balloons and all hatch runs remain separate comparisons. + +| Seed | Wins | Steps | Resets | +|---|---:|---:|---:| +| 0 | 3/3 | 771 | 1 | +| 1 | 3/3 | 554 | 1 | +| 2 | 3/3 | 829 | 2 | + +All three scorecards record `all_levels_won`, a completion time, and the expected frozen runtime. +Their totals agree with their per-level records. +The detailed behavior analysis below concerns seed 0. + +| Level | Role | Wins | Steps | Resets | +|---|---|---:|---:|---:| +| 1 | Training | 1/1 | 452 | 1 | +| 2 | Training | 1/1 | 217 | 0 | +| 3 | Test | 1/1 | 102 | 0 | + +The [final scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/scorecard.json) records `all_levels_won` and a completion time. +Totals agree with the per-level records. + +## How it solved the tasks + +The direct coding agent measured resting heights and oscillation decay after real balloon releases, used one training reset, and retained the measurements in its journal. +It wrote an empirical equilibrium helper, `lift_model.py`, from those measurements. +The inspected helper contains scalar arithmetic and calibrated constants, without engine imports or calls. +Its fitted power-law relation is the agent's approximation, not a claim that it recovered the environment's exact dynamics. + +On the test task, it recognized an oak box and used the gold balloon's measured height from the first training task. +Before the irreversible release, it spent 12 Wait steps averaging noisy resting-height observations. +It then released gold and won after another 49 Wait steps; the goal interrupted the Wait when the dwell completed. +This behavior is recorded in the [test-level agent log](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/agent/003_play_20260912_182006.md). +The helper's source is recorded in the [second training-level log](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_dwell_r1/seed0/run_20260912_180112/agent/002_play_20260912_181548.md). + +The baseline has no supplied simulator, but its coding and journal capabilities still let it construct small empirical models from public observations. +Calling it MF does not imply that its reasoning must avoid physical hypotheses or numerical analysis. +The visible box-speed reading has zero scalar noise in this configuration, so using it to detect rest is an allowed observation. +These inspected actions explain the successful transfer; they do not constitute an exhaustive sandbox security audit or establish an MB/MF gap under the changed goal. diff --git a/docs/comparisons/mf-integrity-audit-20260912.md b/docs/comparisons/mf-integrity-audit-20260912.md new file mode 100644 index 0000000000..96677f0e33 --- /dev/null +++ b/docs/comparisons/mf-integrity-audit-20260912.md @@ -0,0 +1,100 @@ +# Bridge and Balloons MF integrity audit + +Audited on 2026-09-12 against the selected noisy five-domain sweep runs and their frozen runtime. +Scope: all three selected MF seeds in Bridge and original non-hatch Balloons, including their experiment transcripts, journals, agent-visible recordings, current scorecards, and relevant harness code. + +## Assessment + +No evidence was found in the reviewed experiment logs of loading the hidden environment simulator or reading answers from other runs. +This is a log and data audit, not a proof that every possible sandbox escape is prevented. +Bridge has confirmed information leakage through shared controller diagnostics, and task-specific hints in the exposed controller source. +Balloons has an instantaneous success check that agents knowingly exploit, plus a strong baseline that writes and fits its own analytical models. +These results should not be described as a clean demonstration that a purely model-free agent discovers hidden physics from scratch. + +## Verified results + +| Domain | Arm | Whole-run successes | Mean steps, successful runs | Qualifying n | Mean resets, all three runs | +|---|---|---:|---:|---:|---:| +| Bridge | MB | 3/3 | 2214.0 | 3 | 0.00 | +| Bridge | MF | 3/3 | 2269.7 | 3 | 0.00 | +| Balloons | MB | 3/3 | 365.7 | 3 | 0.33 | +| Balloons | MF | 3/3 | 417.0 | 3 | 1.00 | + +All these seeds completed every level. +Steps include all real interaction in each successful run, including earlier reset episodes; they are not limited to the final winning episode. +MF uses about 2.5% more steps in Bridge and 14.0% more in Balloons. +Three seeds are insufficient to establish a general equivalence claim. +Balloons MB seeds 0 and 1 are the explicitly reused subclass pilot; hatch and historical MF cohorts are excluded. +Task draws should be treated as distribution-matched, not assumed identical merely because the agent seed has the same number. + +| Domain | Arm | Seed | Level wins | Steps | Resets | Scorecard | +|---|---|---:|---:|---:|---:|---| +| Bridge | MB | 0 | 2/2 | 2306 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed0/run_20260910_052215/scorecard.json) | +| Bridge | MB | 1 | 2/2 | 1950 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed1/run_20260910_052213/scorecard.json) | +| Bridge | MB | 2 | 2/2 | 2386 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed2/run_20260910_052219/scorecard.json) | +| Bridge | MF | 0 | 2/2 | 2408 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | +| Bridge | MF | 1 | 2/2 | 2413 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/scorecard.json) | +| Bridge | MF | 2 | 2/2 | 1988 | 0 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | +| Balloons | MB | 0 | 3/3 | 466 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed0/run_20260909_171523/scorecard.json) | +| Balloons | MB | 1 | 3/3 | 338 | 1 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed1/run_20260909_171520/scorecard.json) | +| Balloons | MB | 2 | 3/3 | 293 | 0 | [record](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_noise_sweep_r1/seed2/run_20260910_043043/scorecard.json) | +| Balloons | MF | 0 | 3/3 | 441 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | +| Balloons | MF | 1 | 3/3 | 476 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | +| Balloons | MF | 2 | 3/3 | 334 | 1 | [record](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | + +Bridge MF seed 1 retains the first completed execution, run_20260910_144727. +The snapshot documents a later duplicate execution after a scheduler restart during video generation; it is excluded as a duplicate, not treated as a fourth seed. + +## Bridge: a hidden attachment is exposed through an error + +In [seed 1, play 003](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/agent/003_play_20260910_151152.md:1006), a failed Place returns `welded span1` and an exact table penetration of 0.0061 m. +The agent immediately writes: `The failure message is gold: "welded span1" - the bond did form on the previous place`. +It then uses the reported geometry to adjust its release height. +The [shared planner diagnostic](/home/ycliang/predicators-noisy-sweep-20260910/predicators/ground_truth_models/skill_factories/base.py:2134) constructs these labels from held attachments. +This is observed use of information about the hidden attachment, not a speculative exploit. +Internal attachment-aware collision planning may be necessary for the controller, but exposing that attachment identity and precise internal geometry is a separate observation-policy decision. + +Before interaction, the agent also [reads the supplied controller references](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/agent/001_play_20260910_144733.md:1687) and extracts placement heights, weld behavior, and the recipe of lifting a welded three-span row by its middle block. +The [reference exporter](/home/ycliang/predicators-noisy-sweep-20260910/predicators/approaches/agent_model_free_approach.py:345) supplies controller implementation files, including comments. +The public task already asks for a rigid three-block span, so the references do not reveal the entire goal for the first time; they provide additional implementation-specific solution guidance. +These references and controllers are shared with MB, so this is a shared benchmark issue rather than evidence of preferential MF access. + +## Balloons: legitimate self-modeling and an instantaneous win + +The [seed 1 journal](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/agent/sandbox/journal.md:230) compares competing dynamics hypotheses. +Its [uncertainty calculations](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/agent/sandbox/journal.md:273) use damped-sinusoid fits and residual bootstrap estimates. +Other seeds similarly fit oscillation or lift models, enumerate balloon subsets, and use real probes before committing to releases. +These calculations operate on the agent’s own recorded observations and are allowed by the baseline prompt. +The absence of supplied simulator calls does not mean an absence of learned dynamics or uncertainty reasoning. +A more accurate baseline name is direct coding agent without supplied simulator or model-learning tools. + +The [InBand classifier](/home/ycliang/predicators-noisy-sweep-20260910/predicators/envs/pybullet_balloons.py:283) requires the box centre to be in the band and its speed to be below 0.01 m/s in the current state. +It imposes no sustained dwell condition. +The [seed 2 journal](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/agent/sandbox/journal.md:494) explicitly recognizes that an oscillation turning point can certify success even when the equilibrium height is outside the band. +All nine recorded MF level wins have a final speed below 0.01 m/s and a preceding frame above that threshold. +For example, seed 0 level 1 wins at z=0.78166 m in the [0.75952, 0.80952] m band, with speed falling from 0.11596 to 0.00991 m/s. +Those are valid wins under the implemented predicate, but do not demonstrate sustained hovering. +The recordings terminate at success, so this audit does not establish which runs would eventually settle successfully or fail a longer dwell test. +The predicate also applies to MB; this audit does not establish that the issue favors MF more than MB. + +## Observation and tool checks + +The [recording audit](/home/ycliang/predicators/logs/mf_integrity_audit_20260912/data-audit.json) examined 8,075 MF frames across the six runs. +No frame carried a populated privileged or latent state channel, and no simulator-state dictionary keys were exposed. +However, serialized Object instances retain sim_data metadata. +Bridge objects expose cure and attachment field names, although their values remain the default zero or -1 throughout these recordings; Balloons metadata includes body and clip joint IDs. +This is unnecessary implementation metadata and should be removed from the public serialization contract, but it is not evidence that these agents read live hidden cure/attachment values. +The [extracted tool calls](/home/ycliang/predicators/logs/mf_integrity_audit_20260912/tool-calls.json) show analysis of recorded trajectories and supplied reference files, with no use of registered simulator/model tools. +Some attempts to repair sandbox working-directory errors involved broader tools or path changes; these do not justify claiming that the sandbox is adversarially secure. + +## Recommended next steps + +1. Keep and report the existing results as the current protocol cohort, with the above limitations. +2. Replace controller implementation references with a public skill API specification, removing task-specific recipes and hidden-mechanism comments for both arms. +3. Sanitize controller diagnostics so they do not name hidden attachments or expose privileged geometry, while preserving useful feedback based on public observations. +4. Remove simulator metadata from agent-visible serialized objects and validate the complete observation/tool boundary. +5. If the intended Balloons task is sustained hovering, specify and mechanically validate a dwell criterion, then rerun both arms under the changed task as a separate cohort. +6. Preserve ordinary Python analysis and learned journals in the direct coding baseline; banning its successful reasoning would weaken the comparison artificially. + +No experiments, controllers, or task definitions were changed by this audit. +Cleaning these issues may change either arm’s performance; it does not guarantee a larger MB advantage. diff --git a/docs/uncertainty-results/make_noisy_sweep_table.py b/docs/uncertainty-results/make_noisy_sweep_table.py index 96dceda487..c60862ea65 100644 --- a/docs/uncertainty-results/make_noisy_sweep_table.py +++ b/docs/uncertainty-results/make_noisy_sweep_table.py @@ -183,7 +183,9 @@ def refresh(manifest_path=MANIFEST): lines = [ "", "# MB versus MF: five-domain noisy sweep", "", f"Updated {stamp}.", "", - "The result watcher refreshes this table every two minutes while the sweep is active.", + "This table is regenerated from verified scorecards; the periodic watcher is disabled.", + "This is the original cohort, before the Bridge observation-boundary fixes and the Balloons sustained-hover rule.", + "The corrected pilots are reported separately in [Bridge/Balloons integrity results](../comparisons/bridge-balloons-integrity-results.md).", "Each domain and arm has three planned seeds; aggregates remain provisional until all three finish.", "Whole-run solve rate means winning every training and test level.", "Both solve rates and mean resets use finished agent runs, with their count shown.", diff --git a/docs/uncertainty-results/noisy-sweep-snapshot.json b/docs/uncertainty-results/noisy-sweep-snapshot.json index ee20dc245e..a84ff45e17 100644 --- a/docs/uncertainty-results/noisy-sweep-snapshot.json +++ b/docs/uncertainty-results/noisy-sweep-snapshot.json @@ -1,6 +1,6 @@ { "generated_by": "make_noisy_sweep_table.py; do not edit manually", - "updated_at": "2026-09-11T11:39:41.036197+00:00", + "updated_at": "2026-09-14T19:39:34.906642+00:00", "manifest": "/home/ycliang/predicators/logs/noisy_sweep_20260910/launch-manifest.json", "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", "rows": [ @@ -60,17 +60,17 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456627_0", + "job": "22625667_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 2408, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Bridge", @@ -101,17 +101,17 @@ "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456627_2", + "job": "22625597_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 1988, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Fan", @@ -186,17 +186,17 @@ "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456628_1", + "job": "22625598_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 842, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Fan", @@ -271,17 +271,17 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456629_0", + "job": "22625599_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 1, "levels": 2, - "steps": 0, + "steps": 712, "resets": 0, "success": false, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json", - "end_reason": null + "end_reason": "agent_ended" }, { "domain": "Domino", @@ -305,17 +305,17 @@ "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456629_2", + "job": "22639804_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 1, "levels": 2, - "steps": 0, + "steps": 821, "resets": 0, "success": false, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json", - "end_reason": null + "end_reason": "level_lost" }, { "domain": "Boil", @@ -373,51 +373,51 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_0", + "job": "22639805_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 1, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 593, + "steps": 738, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Boil", "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_1", + "job": "22639805_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 0, + "steps": 1076, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Boil", "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456631_2", + "job": "22625600_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 1, + "status": "Finished", + "finished": true, + "wins": 2, "levels": 2, - "steps": 570, + "steps": 988, "resets": 0, - "success": false, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", @@ -475,51 +475,51 @@ "arm": "MF", "seed": 0, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_0", + "job": "22639807_0", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 441, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", "arm": "MF", "seed": 1, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_1", + "job": "22625601_1", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 476, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" }, { "domain": "Balloons", "arm": "MF", "seed": 2, "source_commit": "b09217bb38f2c3082136994ae43fbef2eb590e82", - "job": "22456632_2", + "job": "22625601_2", "reused": false, - "status": "In progress", - "finished": false, - "wins": 0, + "status": "Finished", + "finished": true, + "wins": 3, "levels": 3, - "steps": 0, - "resets": 0, - "success": false, + "steps": 334, + "resets": 1, + "success": true, "scorecard": "/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json", - "end_reason": null + "end_reason": "all_levels_won" } ], "aggregate": [ @@ -538,13 +538,13 @@ { "domain": "Bridge", "arm": "MF", - "finished": 1, + "finished": 3, "expected": 3, - "successful": 1, + "successful": 3, "whole_run_solve_pct": 100.0, "level_solve_pct": 100.0, - "mean_steps_successful": 2413, - "steps_n": 1, + "mean_steps_successful": 2269.6666666666665, + "steps_n": 3, "mean_resets_finished": 0 }, { @@ -562,13 +562,13 @@ { "domain": "Fan", "arm": "MF", - "finished": 2, + "finished": 3, "expected": 3, - "successful": 2, + "successful": 3, "whole_run_solve_pct": 100.0, "level_solve_pct": 100.0, - "mean_steps_successful": 901, - "steps_n": 2, + "mean_steps_successful": 881.3333333333334, + "steps_n": 3, "mean_resets_finished": 0 }, { @@ -586,7 +586,7 @@ { "domain": "Domino", "arm": "MF", - "finished": 1, + "finished": 3, "expected": 3, "successful": 0, "whole_run_solve_pct": 0.0, @@ -610,14 +610,14 @@ { "domain": "Boil", "arm": "MF", - "finished": 0, + "finished": 3, "expected": 3, - "successful": 0, - "whole_run_solve_pct": null, - "level_solve_pct": null, - "mean_steps_successful": null, - "steps_n": 0, - "mean_resets_finished": null + "successful": 3, + "whole_run_solve_pct": 100.0, + "level_solve_pct": 100.0, + "mean_steps_successful": 934, + "steps_n": 3, + "mean_resets_finished": 0 }, { "domain": "Balloons", @@ -634,14 +634,14 @@ { "domain": "Balloons", "arm": "MF", - "finished": 0, + "finished": 3, "expected": 3, - "successful": 0, - "whole_run_solve_pct": null, - "level_solve_pct": null, - "mean_steps_successful": null, - "steps_n": 0, - "mean_resets_finished": null + "successful": 3, + "whole_run_solve_pct": 100.0, + "level_solve_pct": 100.0, + "mean_steps_successful": 417, + "steps_n": 3, + "mean_resets_finished": 1 } ] } diff --git a/docs/uncertainty-results/noisy-sweep-summary.tsv b/docs/uncertainty-results/noisy-sweep-summary.tsv index 50a5c53936..31839adcf1 100644 --- a/docs/uncertainty-results/noisy-sweep-summary.tsv +++ b/docs/uncertainty-results/noisy-sweep-summary.tsv @@ -1,11 +1,11 @@ domain arm finished expected successful whole_run_solve_pct level_solve_pct mean_steps_successful steps_n mean_resets_finished Bridge MB 3 3 3 100.0 100.0 2214 3 0 -Bridge MF 1 3 1 100.0 100.0 2413 1 0 +Bridge MF 3 3 3 100.0 100.0 2269.6666666666665 3 0 Fan MB 3 3 3 100.0 100.0 425 3 0 -Fan MF 2 3 2 100.0 100.0 901 2 0 +Fan MF 3 3 3 100.0 100.0 881.3333333333334 3 0 Domino MB 3 3 3 100.0 100.0 404.6666666666667 3 0 -Domino MF 1 3 0 0.0 50.0 NA 0 0 +Domino MF 3 3 0 0.0 50.0 NA 0 0 Boil MB 3 3 3 100.0 100.0 668.3333333333334 3 0 -Boil MF 0 3 0 NA NA NA 0 NA +Boil MF 3 3 3 100.0 100.0 934 3 0 Balloons MB 3 3 3 100.0 100.0 365.6666666666667 3 0.3333333333333333 -Balloons MF 0 3 0 NA NA NA 0 NA +Balloons MF 3 3 3 100.0 100.0 417 3 1 diff --git a/docs/uncertainty-results/noisy-sweep-table.md b/docs/uncertainty-results/noisy-sweep-table.md index 82fd7a2371..e41829624c 100644 --- a/docs/uncertainty-results/noisy-sweep-table.md +++ b/docs/uncertainty-results/noisy-sweep-table.md @@ -1,9 +1,11 @@ # MB versus MF: five-domain noisy sweep -Updated 2026-09-11T11:39:41.036197+00:00. +Updated 2026-09-14T19:39:34.906642+00:00. -The result watcher refreshes this table every two minutes while the sweep is active. +This table is regenerated from verified scorecards; the periodic watcher is disabled. +This is the original cohort, before the Bridge observation-boundary fixes and the Balloons sustained-hover rule. +The corrected pilots are reported separately in [Bridge/Balloons integrity results](../comparisons/bridge-balloons-integrity-results.md). Each domain and arm has three planned seeds; aggregates remain provisional until all three finish. Whole-run solve rate means winning every training and test level. Both solve rates and mean resets use finished agent runs, with their count shown. @@ -14,48 +16,48 @@ Balloons MB seeds 0 and 1 are reused from the identical agent runtime and flags; | Domain | Arm | Finished seeds | Whole-run solve rate | Level solve rate | Mean steps, successful runs (n) | Mean resets | |---|---|---:|---:|---:|---:|---:| | Bridge | MB | 3/3 | 100.0% (3/3) | 100.0% | 2,214 (n=3) | 0 | -| Bridge | MF | 1/3 | 100.0% (1/1) | 100.0% | 2,413 (n=1) | 0 | +| Bridge | MF | 3/3 | 100.0% (3/3) | 100.0% | 2,269.7 (n=3) | 0 | | Fan | MB | 3/3 | 100.0% (3/3) | 100.0% | 425 (n=3) | 0 | -| Fan | MF | 2/3 | 100.0% (2/2) | 100.0% | 901 (n=2) | 0 | +| Fan | MF | 3/3 | 100.0% (3/3) | 100.0% | 881.3 (n=3) | 0 | | Domino | MB | 3/3 | 100.0% (3/3) | 100.0% | 404.7 (n=3) | 0 | -| Domino | MF | 1/3 | 0.0% (0/1) | 50.0% | - (n=0) | 0 | +| Domino | MF | 3/3 | 0.0% (0/3) | 50.0% | - (n=0) | 0 | | Boil | MB | 3/3 | 100.0% (3/3) | 100.0% | 668.3 (n=3) | 0 | -| Boil | MF | 0/3 | Pending | Pending | - (n=0) | - | +| Boil | MF | 3/3 | 100.0% (3/3) | 100.0% | 934 (n=3) | 0 | | Balloons | MB | 3/3 | 100.0% (3/3) | 100.0% | 365.7 (n=3) | 0.33 | -| Balloons | MF | 0/3 | Pending | Pending | - (n=0) | - | +| Balloons | MF | 3/3 | 100.0% (3/3) | 100.0% | 417 (n=3) | 1 | | Domain | Arm | Seed | Status | Wins | Steps | Resets | Source | |---|---|---:|---|---:|---:|---:|---| | Bridge | MB | 0 | Finished | 2/2 | 2306 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed0/run_20260910_052215/scorecard.json) | | Bridge | MB | 1 | Finished | 2/2 | 1950 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed1/run_20260910_052213/scorecard.json) | | Bridge | MB | 2 | Finished | 2/2 | 2386 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/bridge-agent_continual_noise_sweep_r1/seed2/run_20260910_052219/scorecard.json) | -| Bridge | MF | 0 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | +| Bridge | MF | 0 | Finished | 2/2 | 2408 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_144716/scorecard.json) | | Bridge | MF | 1 | Finished | 2/2 | 2413 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_144727/scorecard.json) | -| Bridge | MF | 2 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | +| Bridge | MF | 2 | Finished | 2/2 | 1988 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/bridge-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_144730/scorecard.json) | | Fan | MB | 0 | Finished | 2/2 | 312 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed0/run_20260910_062635/scorecard.json) | | Fan | MB | 1 | Finished | 2/2 | 493 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed1/run_20260910_062648/scorecard.json) | | Fan | MB | 2 | Finished | 2/2 | 470 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/fan-agent_continual_noise_sweep_r1/seed2/run_20260910_062710/scorecard.json) | | Fan | MF | 0 | Finished | 2/2 | 1396 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_164313/scorecard.json) | -| Fan | MF | 1 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json) | +| Fan | MF | 1 | Finished | 2/2 | 842 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_164307/scorecard.json) | | Fan | MF | 2 | Finished | 2/2 | 406 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/fan-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_164315/scorecard.json) | | Domino | MB | 0 | Finished | 2/2 | 395 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed0/run_20260910_073549/scorecard.json) | | Domino | MB | 1 | Finished | 2/2 | 310 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed1/run_20260910_073544/scorecard.json) | | Domino | MB | 2 | Finished | 2/2 | 509 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/domino_high_friction_turn-agent_continual_noise_sweep_r1/seed2/run_20260910_073545/scorecard.json) | -| Domino | MF | 0 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json) | +| Domino | MF | 0 | Finished | 1/2 | 712 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_174127/scorecard.json) | | Domino | MF | 1 | Finished | 1/2 | 430 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_174127/scorecard.json) | -| Domino | MF | 2 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json) | +| Domino | MF | 2 | Finished | 1/2 | 821 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/domino_high_friction_turn-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_174127/scorecard.json) | | Boil | MB | 0 | Finished | 2/2 | 721 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed0/run_20260910_133006/scorecard.json) | | Boil | MB | 1 | Finished | 2/2 | 775 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed1/run_20260910_133006/scorecard.json) | | Boil | MB | 2 | Finished | 2/2 | 509 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/boil-agent_continual_noise_sweep_r1/seed2/run_20260910_132947/scorecard.json) | -| Boil | MF | 0 | In progress | 1/2 | 593 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json) | -| Boil | MF | 1 | In progress | 0/2 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json) | -| Boil | MF | 2 | In progress | 1/2 | 570 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json) | +| Boil | MF | 0 | Finished | 2/2 | 738 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_192219/scorecard.json) | +| Boil | MF | 1 | Finished | 2/2 | 1076 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_192202/scorecard.json) | +| Boil | MF | 2 | Finished | 2/2 | 988 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/boil-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_192203/scorecard.json) | | Balloons | MB | 0 | Finished | 3/3 | 466 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed0/run_20260909_171523/scorecard.json) (reused) | | Balloons | MB | 1 | Finished | 3/3 | 338 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_original_subclass_r1/seed1/run_20260909_171520/scorecard.json) (reused) | | Balloons | MB | 2 | Finished | 3/3 | 293 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual/balloons-agent_continual_noise_sweep_r1/seed2/run_20260910_043043/scorecard.json) | -| Balloons | MF | 0 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | -| Balloons | MF | 1 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | -| Balloons | MF | 2 | In progress | 0/3 | 0 | 0 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | +| Balloons | MF | 0 | Finished | 3/3 | 441 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed0/run_20260910_201514/scorecard.json) | +| Balloons | MF | 1 | Finished | 3/3 | 476 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed1/run_20260910_201533/scorecard.json) | +| Balloons | MF | 2 | Finished | 3/3 | 334 | 1 | [scorecard](/home/ycliang/predicators/logs/agent_continual_model_free/balloons-agent_continual_model_free_noise_sweep_r1/seed2/run_20260910_201534/scorecard.json) | ## Duplicate execution audit diff --git a/docs/uncertainty/balloons-selective-guidance.md b/docs/uncertainty/balloons-selective-guidance.md index 0460fe405b..b699215a09 100644 --- a/docs/uncertainty/balloons-selective-guidance.md +++ b/docs/uncertainty/balloons-selective-guidance.md @@ -64,9 +64,9 @@ An additional likelihood that depends on a selected uniform coordinate preserves The existing exact prefix replays pass, 14 of 16 new guide candidates have finite support, serial and parallel targets agree exactly, and future-data corruption leaves the fitting target unchanged. The maximum target-factorization discrepancy is 9.09e-13. -Array `22715405` runs paired fitting seeds 620 and 621 on `mit_preemptable`, each with four CPUs and 20 GiB. +Array `22715405` completed paired fitting seeds 620 and 621 on `mit_preemptable`, each with four CPUs and 20 GiB. It retains 64 particles, 32 cubic temperatures, eight moves, the original blocks and a 16,448-evaluation cap. -Final-target readers `22715406` and `22715407` are dependency-queued to recover each complete checkpoint and freshly evaluate every retained particle. +Final-target readers `22715406` and `22715407` completed, recovering each complete checkpoint and freshly evaluating every retained particle. Their frozen bundle is `logs/uncertainty_balloons_selective_verification_20260914`. The selective-proposal weighted future adapter in `logs/uncertainty_balloons_selective_forecasts_20260914` has passed native validation and independent verification. Its particle decoder binds the new guide and all five uniform-coordinate selections; the default-guide decoder cannot be substituted for it. @@ -94,8 +94,8 @@ Both completed reports match the checksums of their frozen inputs and source art The reader successfully rejects both the old proposal center and the correct center with its uniform-coordinate selection omitted. This distinction matters because omitting the selection changes the physical state represented by the same saved proposal coordinates. -Full forecasts `22715569` and `22715571` are queued behind the completed-fit readers and successful adapter verification. -Independent forecast readers `22715570` and `22715572` follow them. +Full forecasts `22715569` and `22715571` completed after the completed-fit readers and successful adapter verification. +Independent forecast readers `22715570` and `22715572` failed before verification because their launcher invocation omitted the required numerical index. Comparison fixture `22715573` and final comparison `22715574` cover the original, default-centered and selective treatments, retaining the incumbent estimator as a separate reference. The full comparison also waits for the earlier default-guide forecast readers, so unfinished treatments cannot be silently omitted. @@ -106,3 +106,9 @@ Its synthetic weighted-distribution check and six deliberate invalid-budget, inv It independently reproduces the original two fits' parameter summaries and retains all four new forecasts as pending. An incomplete comparison fixture is expected while new forecasts are unavailable; full comparison requires all six verified forecasts. These reports remain offline evidence and do not establish agent solve rates. + +The resulting dependent comparison `22715574` was cancelled. +The exact shell failure was reproduced, and replacement reader array `22755276` supplies both required arguments to the unchanged frozen reader. +Comparison `22755277` follows successful completion of both readers. +No fit or forecast is regenerated for this invocation-only recovery. +The recovery manifest and outputs are in `logs/uncertainty_balloons_selective_reader_recovery_20260914`. diff --git a/docs/uncertainty/boil-informative-heating.md b/docs/uncertainty/boil-informative-heating.md new file mode 100644 index 0000000000..edba7f497f --- /dev/null +++ b/docs/uncertainty/boil-informative-heating.md @@ -0,0 +1,85 @@ +# Boil heating: information and conditional numerical accuracy + +September 14, 2026. +This diagnostic follows the [completed reduced-target comparison](validation-status-20260914.md#boil-reduced-target-outcome). +It asks whether informative heating observations constrain the thermal parameters under the existing simulator program and declared sensor noise. +It does not change the production agent or establish an adequate joint posterior over the scene and all parameters. + +## Fixed conditional problem + +The physical histories come from the independently verified density trajectories of the two completed reduced fits, numerical seeds 410 and 411. +Selection uses the highest fitting weight with the smallest particle index as a tie-break, independently of future prediction errors. +All nonthermal parameters and each candidate physical history remain fixed. +Only burner radius, heating onset and heating width are inferred. + +The original independent uniform priors remain radius `[0.04, 0.25]`, onset `[5, 80]`, and width `[1, 40]`. +The bubbling sensor standard deviation remains 0.07. +The existing output model has no discrepancy process on bubbling, and none is added here. +The likelihood multiplies the Gaussian bubbling-reading factors through the designated prefix. +The initial reading supplies a parameter-independent constant and is explicitly omitted from the reported conditional normalizer. + +The two prefixes contain 132 and 224 actions. +The first never turns the burner on and retains the exact prior-retention control. +The second includes heating observations and is a separate development case. +Clean evaluator observations and noisy observations after action 224 do not enter this conditional inference. +The source density histories condition on observed robot motion causally; the native audit changes later joint observations and verifies that every prediction through action 224 remains unchanged. +This diagnostic conditions on a fixed physical prefix, and therefore does not integrate uncertainty about that prefix or update the other fitted quantities with the additional observations. + +## Integration and verification + +For a fixed physical prefix, distances to the burner partition the radius prior into intervals with identical heating decisions. +The radius integral is exact on this partition. +Onset and width use product Gauss-Legendre quadrature under their original normalized priors. +The producer groups reading residuals by accumulated integer heat, while the independent reader sums each Gaussian observation residual directly. +Both methods preserve the joint onset-width likelihood; marginal summaries do not imply independent posterior parameters. + +Native job `22755839` completed in 1:20, with 1,320 simulator actions. +It reproduces both complete archived native histories, predictions and carried memory in fresh worlds, and verifies prefix independence from altered later observations. +Reader `22755840` completed in six seconds, checking both prefixes for both selected candidates, the original observations, radius partitions, Gaussian likelihood probes and numerical moments. +The verified result checksum is `43edab5af557698d42eb65ff3a653484ab6b2dc4a543056e38966e4bf38ee810`. + +The initial 128- and 256-node rules disagree by up to 0.174 in parameter means on the informative prefix. +That motivates a separate precision experiment at 512, 1,024 and 2,048 nodes, with absolute tolerances of 0.005 for parameter means, standard deviations and the log normalizer. +It reuses the same verified conditional targets and performs no additional native simulation. +Jobs `22756314` and `22756315` completed in 15 and 12 seconds, respectively. +The independent reader confirms the higher-order integrals through direct residual sums. +Its verified result checksum is `94e57f245aa7d86cb49ee5c071d63b6afbe0a405f833fd256137dd32d2bb79a1`. + +| Comparison of quadrature orders | Maximum mean difference | Maximum standard-deviation difference | Log-normalizer difference | Declared precision screen | +|---|---:|---:|---:|---| +| 512 vs 1,024 | 0.001524 | 0.000992 | 0.008052 | Fails log-normalizer tolerance | +| 1,024 vs 2,048 | 0.000229 | 0.000163 | 0.001142 | Passes | + +This is empirical quadrature stability on this conditional problem, not a certified global integration bound or acceptance of the full-scene sampler. + +## Conditional outcome + +The two selected candidates induce the same heating-count history and scalar conditional likelihood. +They therefore produce identical conditional results; they are not two independent posterior-replication successes. +The higher-precision worker recognizes and reuses that identical calculation explicitly. + +| Parameter | 132-action mean | 132-action standard deviation | 224-action mean | 224-action standard deviation | +|---|---:|---:|---:|---:| +| Burner radius, metres | 0.14500 | 0.06062 | 0.14500 | 0.06062 | +| Heating onset, action steps | 42.50000 | 21.65064 | 30.12237 | 0.29841 | +| Heating ramp width, action steps | 20.50000 | 11.25833 | 5.71475 | 0.51485 | + +The all-off prefix retains all three normalized priors. +The longer prefix strongly constrains onset and width under the unchanged program and sensor likelihood. +In these selected histories, every radius in the original support induces the same heating decisions, so radius remains uniform even after heating. +This conditional invariance does not certify radius independence across all possible scenes. + +The result separates lack of informative observations from failure to compute a low-dimensional conditional distribution. +It supports testing heating-aware fitting without introducing a new bubbling-discrepancy term merely to match historical defaults. +It does not demonstrate held-out forecasting accuracy: no such score was computed here. + +## Next integration requirement + +A complete comparison on the longer prefix must define a new data identity and update the joint scene and parameter inference using all observations through that prefix. +The all-off independence certificate cannot be carried forward for onset and width. +Conditional thermal draws must preserve their dependence on each other and on the candidate scene and nonthermal parameters. +Do not substitute the conditional means in this table into every particle or independently sample these marginal standard deviations. +The original 132-action benchmark remains an extrapolation and prior-retention case. +Future prediction and live-agent acceptance still require their separate comparisons. + +Frozen scripts and reports are in `logs/uncertainty_boil_informative_heating_20260914` and `logs/uncertainty_boil_heating_quadrature_20260914`. diff --git a/docs/uncertainty/bridge-memory-recovery.md b/docs/uncertainty/bridge-memory-recovery.md index e9437d2fbb..8cf632668f 100644 --- a/docs/uncertainty/bridge-memory-recovery.md +++ b/docs/uncertainty/bridge-memory-recovery.md @@ -26,7 +26,7 @@ The failed first resume allocations and their cancelled dependent pipelines rema Array `22714037` resumes these checkpoints with the unchanged frozen fitter, probability model, sampler configuration, 16 ordered workers, and original evaluation budget. Each new allocation requests 128 GiB instead of 64 GiB on the original compute node. The underlying memory-growth cause has not been isolated; the larger allocation provides headroom for the remaining stages. -Readers `22714038` and `22714039` follow the fits and will independently replay the combined pre-interruption and resumed ledgers. +Readers `22714038` and `22714039` completed successfully, independently replaying the combined pre-interruption and resumed ledgers. The replacement forecast pipeline is frozen in `logs/uncertainty_bridge_resumed_forecasts_v2_20260914`. Its forecast and summary calculations are unchanged; source paths point to the resumed fits and new output directory. @@ -37,3 +37,13 @@ The old downstream jobs cannot proceed after their failed dependencies and are t The resumed fit report's elapsed-time field covers only its new allocation. Total inference cost must also include the interrupted allocation, initialization and recovery validation; do not quote the new elapsed-time field as total cost. The numerical and predictive gates remain open. + +## Completed forecasts and comparison + +Both resumed fits, forecasts `22714051` and `22714054`, readers `22714052` and `22714055`, and comparison `22714056` have completed. +All nine directly referenced comparison sources match their recorded hashes. +The two final populations retain one initial lineage each. +Both posterior forecasts and the incumbent selected point predict zero probability for the final clean geometric goal, which the assessment records as true. +The small aggregate goal Brier score of 1/586 reflects that this goal occurs at only one frame. +The maximum between-fit glue-probability gap is 0.1512; the comparison does not establish numerical or predictive acceptance. +See the [consolidated validation status](validation-status-20260914.md#bridge-completed-recovery) for the outcome and cumulative-cost caveat. diff --git a/docs/uncertainty/implementation-progress.md b/docs/uncertainty/implementation-progress.md index 2dfbb0402e..89ac3d102e 100644 --- a/docs/uncertainty/implementation-progress.md +++ b/docs/uncertainty/implementation-progress.md @@ -6,6 +6,13 @@ The incumbent estimator remains the production default. ## Current stage +The [September 14 validation status](validation-status-20260914.md) consolidates the completed Boil and Bridge comparisons, Balloons reader recovery, and the independently verified informative-heating diagnostic. +Full-prefix replay is the required reference; arbitrary mid-trajectory restoration is not a migration gate. +The discrepancy policy now explicitly separates replay bugs, missing program mechanisms, uninformative data and residual model error. +The [informative-heating diagnostic](boil-informative-heating.md) preserves the thermal priors under the 132-action all-off prefix and constrains onset and width after 224 actions in two fixed candidate histories. +Native replay, later-observation isolation, independent direct Gaussian references and a higher-precision integration comparison pass. +This is conditional learnability evidence; a full-scene heating-aware target and held-out forecast remain separate requirements. + Stage 0 interface preservation is complete, with scripted behavior parity checked. Stage A has implemented probability and replay components, but physical support and numerical validation remain incomplete. The active work is Stage B offline comparison; Stage C live posterior use and Stage E retirement have not passed their evidence requirements. @@ -19,14 +26,15 @@ The reduced Boil target removes the three unobserved heating coordinates while p Native preflight `22717213` passes 56 assignments across eight source states, with 792 native actions and four contract guards. Independent reader `22717214` also passes, with 264 fresh native actions and a rejected placeholder coordinate. Both small fitting fixtures and readers now pass, including 4,224 fresh native actions per reader. -Full fits `22717256` and `22717258` are running with final-target verification queued. +Full fits `22717256` and `22717258` and their final-target verification have completed. The [reduced Boil forecast adapter](reduced-boil-forecasts.md) restores independently sampled thermal priors per complete trajectory while preserving retained joint rows and weights. Its twelve source guards, ten malformed-history checks and nonuniform density/variance reference pass in `22717364`. Native forecast fixture `22717373` completed 128 histories and 40,656 native actions. Independent reader `22717375` also passes, checking 304,128 joint factors and using 6,864 additional native actions, including two fresh complete histories. -Both full forecasts, their independent readers and the final comparison are dependency-queued behind successful fit and fixture verification. +Both full forecasts, their independent readers and final comparison `22717441` have completed. Comparison fixture `22717399` preserves all seven earlier controls while explicitly retaining both pending new outcomes. -No full reduced fit or new agent result is available yet. +The reduced fits retain one initial lineage each; bubbling RMSE is 0.30445 and 0.37039 versus the incumbent's 0.01915. +No new agent result or numerical/predictive acceptance follows from this completed comparison. The [Balloons transition diagnostic](balloons-transition-sensitivity.md) now verifies 32 paired continuations and 9,400 native actions. At one selected fitted state, removing future joint and velocity noise eliminates bursts in four continuations, but does not recover the goal; the other selected state does not improve. @@ -39,24 +47,28 @@ Array `22714577` now runs the two original numerical seeds with only the paramet The default-centered seed 621 fit `22714577_1` and its reader `22714629` have completed; all 64 final native targets and the checkpoint reproduce, with one surviving original lineage. This does not establish mixing or prediction quality. The paired seed 620 fit `22714577_0` and reader `22714628` have also completed, with the same final-target reproducibility and one surviving original lineage. -Both default-centered posterior forecasts remain pending or running; their comparison is incomplete. +Both default-centered posterior forecasts and comparison `22715120` have completed. +The changed guide improves box-speed RMSE but worsens box-height RMSE in both numerical fits; both retain one original lineage. The new weighted forecast adapter and independent reader `22715007`/`22715008` have passed, checking both broad and local proposal mappings, complete native prefixes, generation and density histories, and rejection of the old guide when decoding new particles. -Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` are dependency-queued behind the verified completed fits. +Full forecasts `22715115`/`22715117`, readers `22715116`/`22715118`, and comparison `22715120` completed after the verified fits. The comparison retains both earlier fits and the incumbent, including prediction metrics and computation costs; its fixture `22715119` passes while correctly leaving the new outcomes pending. The [parameter-dependence audit](balloons-selective-guidance.md) and independent reader `22715205`/`22715206` have completed all 46 native cases and twenty marginal checks. Changing five parameters separately or together leaves both tested fitting histories and scores exactly unchanged, while the original fits retain very narrow marginals for them. A selective guide with uniform proposals on those coordinates and 20% block refresh moves now passes native validation `22715307`, including 32 default-parity and 32 independent mixture-density checks. -Paired fits `22715405` are running with final-target readers `22715406`/`22715407` dependent on completion. +Paired fits `22715405` and final-target readers `22715406`/`22715407` have completed. The selective proposal's weighted future adapter and independent reader `22715541`/`22715542` now pass, including rejection of a decoder that omits the selective mask. -Forecasts `22715569`/`22715571`, independent readers and the six-population comparison `22715574` are dependency-queued. +Forecasts `22715569`/`22715571` completed, but their reader invocations omitted the required numerical index and failed before verification. +The cancelled comparison `22715574` is replaced by `22755277`, after invocation-recovery reader array `22755276`. The comparison fixture `22715573` passes its marginal and configuration guards, retaining the four pending forecasts explicitly. The comparison preserves the original and default-centered controls and reports parameter spread alongside future predictions. It retains the complete prior, likelihood and Metropolis correction and does not assume global parameter independence. The [Bridge recovery](bridge-memory-recovery.md) validates both stage-26 checkpoints after the original fits exhausted their 64 GiB allocations. Both complete numerical prefixes and four fresh native targets reproduce exactly. -After fixing and checking a JSON-container comparison in the resume launcher, array `22714037` now resumes the same fits with 128 GiB per allocation, with replacement fit readers and the complete forecast/verification pipeline queued. +After fixing and checking a JSON-container comparison in the resume launcher, array `22714037` resumed the same fits with 128 GiB per allocation, followed by replacement fit readers and the complete forecast/verification pipeline. This is recovery from an infrastructure interruption; the incomplete fits are not model or agent outcomes. +The resumed fits and their full verification/forecast/comparison pipeline have now completed. +Both fits retain one initial lineage, and both posterior forecasts and the incumbent miss the final clean geometric goal. The [Fan fixture diagnostic](fan-static-fixtures.md) now isolates a practical forecast sensitivity after the speed-reflection result. Two verified scene-exchange audits use 4,884 native actions and retain three geometry-rejected candidates. diff --git a/docs/uncertainty/reduced-boil-forecasts.md b/docs/uncertainty/reduced-boil-forecasts.md index c4379cc18a..e774add4fe 100644 --- a/docs/uncertainty/reduced-boil-forecasts.md +++ b/docs/uncertainty/reduced-boil-forecasts.md @@ -11,9 +11,9 @@ Independent readers `22717219` and `22717221` replay the complete numerical ledg The verified source checksums are `8490860cad3bb895298296e523cb171ac3f737b56d931679107306bb322fc23f` and `e4b1360238f326cb6c990ceab3f3ac1c79d665a770da0d6739310b0ad6c826b7`. These deliberately small two-temperature fixtures establish implementation consistency, not posterior adequacy. -Full fits `22717256` and `22717258` are running on compute node `node1411` in `mit_preemptable`. +Full fits `22717256` and `22717258` completed on compute node `node1411` in `mit_preemptable`. They retain the original 32 particles, 64 temperatures, eight moves and 20,000-evaluation cap, with the three independent singleton blocks removed. -Their independent readers `22717257` and `22717259` are dependency-queued. +Their independent readers `22717257` and `22717259` completed successfully. The reports retain an unavailable numerical assessment until replication and budget-stability requirements are met. ## Restoring the independent priors in forecasts @@ -53,9 +53,8 @@ It uses 6,864 additional native actions. The verified forecast source checksum is `bbbb8324640aaf3a348a6888e4e21e6de44a539c8698477b2e489c023192bd35`. The reader reconstructs the 81-to-84 coordinate mapping separately, regenerates thermal draws from their recorded seeds, checks every native history and readout, and repeats complete generation and density histories. It independently computes the thermal density average and weighted prediction summaries. -Full forecasts `22717437` and `22717439` are dependency-queued behind the successful fixture reader and the corresponding full-fit reader. -Their independent readers are `22717438` and `22717440`, followed by final comparison `22717441`. -These gates precede full forecast execution. +Full forecasts `22717437` and `22717439`, independent readers `22717438` and `22717440`, and final comparison `22717441` completed successfully. +The [completed comparison](validation-status-20260914.md#boil-reduced-target-outcome) records the results and remaining numerical and predictive failures. ## Comparison and remaining acceptance @@ -68,4 +67,5 @@ The comparison reports fitting and forecast costs, between-fit prediction differ Comparison fixture `22717399` completed its checks and retained all seven original rows while correctly reporting both new full comparisons as pending. Missing results remain pending and cannot support an improvement claim. A completed mechanical comparison would still require numerical and predictive adequacy before live planning integration or retirement of the incumbent. -A separate fitting prefix containing heating remains necessary to test learning from informative data; it will not replace the original all-off extrapolation comparison. +A separate [conditional diagnostic](boil-informative-heating.md) using a 224-action prefix containing heating completed as `22755839`, with independent reader `22755840` and higher-precision follow-up `22756314`/`22756315`. +It holds selected physical histories and nonthermal parameters fixed and does not replace either the full-scene inference comparison or the original all-off extrapolation benchmark. diff --git a/docs/uncertainty/simplification-proposal.md b/docs/uncertainty/simplification-proposal.md index 4ce57d8bfc..269bc51e2e 100644 --- a/docs/uncertainty/simplification-proposal.md +++ b/docs/uncertainty/simplification-proposal.md @@ -2,6 +2,7 @@ September 11, 2026. Revised September 12, 2026 to clarify exact conditioning, inference availability, and the optional execution-filter extension. +Implementation decisions confirmed September 14, 2026: replay from candidate initialization is the required reference, and limited simulator discrepancy is an explicitly evaluated extension after replay and program errors are diagnosed. This is a design proposal, not a description of an implemented or validated replacement. The companion [implementation explanation](explained.md) documents the current behavior and source locations. Implementation of the staged migration is tracked in [implementation progress](implementation-progress.md). @@ -20,7 +21,7 @@ Keeping the existing observation estimator is a valid final choice if those exte This endpoint unifies parameter uncertainty; it does not claim a full joint Bayesian belief during execution. Start by preserving the current agent behind a versioned result interface, without changing its estimates, reports, or decisions. -Verify state restoration and recorded-action replay before implementing parameter inference with uncertain initial states. +Verify candidate initialization and recorded-action replay before implementing parameter inference with uncertain initial states. Evaluate that inference offline using a fixed prior and all available fitting recordings, with intervals and parameter ensembles derived from the same posterior approximation. Introduce it into planning only after it passes prediction checks, while preserving the existing execution estimator and decision rules. Replace execution state estimation last, if it independently demonstrates a benefit. @@ -95,6 +96,10 @@ Only evaluator-only mechanical audits may use evaluator reset state or private d A portable candidate must include full body orientations, original command-weld frames, and commands queued for the next action, even when these quantities are absent from public observations. Their values must come from the candidate prior or simulated history, not privileged recording metadata. Arbitrary mid-trajectory restoration remains an approximation until separately validated against uninterrupted replay. +It is not required for this migration and must not block inference or planning integration that uses full-prefix replay. +Without a validated checkpoint, evaluate each alternative future by reconstructing its candidate and replaying the recorded low-level actions from initialization. +For a stochastic discrepancy model, preserve or explicitly resample the candidate's latent history under the declared conditional law; an unrelated random replay does not reconstruct the same candidate. +Checkpointing and prefix caching are optional optimizations whose identity must include the program, parameters, initial state, actions, and any latent random history. Numerically repeatable candidate replay, faithful evaluator reconstruction, and predictive accuracy of a learned program are three distinct acceptance claims. ### Required initial-state inventory @@ -180,6 +185,30 @@ Evaluate any transition-discrepancy model on held-out development interactions a For a stochastic discrepancy extension, infer the intermediate states under its declared transition distribution; the deterministic initial-state formula above no longer suffices. Persistent mismatch should still inform program revision. +### Discrepancy implementation decision + +Diagnose reconstruction bugs, parameter error, missing program logic, and residual approximation error separately before extending the probability model. +Fix reproducible initialization or replay defects directly. +Expose missing mechanisms and incompatible exact predictions to simulator-program revision; a discrepancy term must not silently excuse them. +Lack of informative observations, such as an all-burners-off fitting prefix, is a separate cause of broad uncertainty and does not by itself justify discrepancy. + +A limited discrepancy extension may proceed when a repeatable residual pattern remains after those checks. +Declare the affected quantities, temporal law, fixed hyperparameters or original hyperprior, and physical or output-level interpretation before running the comparison. +An output discrepancy changes the distribution of observations around a simulated trajectory; a transition discrepancy changes physical histories and therefore can change contacts and events. +These are distinct model changes and must not be substituted for one another. +Keep declared sensor noise unchanged and preserve exact observations through the appropriate conditional construction. + +Use the same discrepancy law in fitting, future generation, and future-density evaluation. +Select a law using fitting or designated development data, freeze it, and evaluate causal predictions on a separate suffix or recording. +If a previously held-out suffix motivates a new law, it becomes development evidence and a new untouched evaluation is required for acceptance. +Compare with the model without that extension where its conditional target is supported, retaining unsupported cases explicitly rather than manufacturing a posterior. +Report numerical repeatability, prediction error, consequential event probabilities, and compute cost; better training likelihood alone is insufficient. +Keep diagnostic interventions that suppress future noise separate from a consistently refitted model. + +The production implementation should not accumulate domain-specific corrections selected to make these recordings pass. +Any domain knowledge needed for a prior or model must have an explicit source available to the agent through the task interface or learned simulator. +Retain the existing execution estimator and decision rules during this evaluation. + ## 3. Standardize the inference result, evaluate the approximation Evaluate a batch sampler over joint dynamics parameters and uncertain episode initial states as the first candidate implementation. @@ -376,7 +405,8 @@ Record later runtime changes separately instead of attributing every difference ### Stage A: define and verify the probability model -Implement and verify the state/restoration contract from section 1 before the new fitter consumes real recordings. +Implement and verify the candidate-initialization and full-prefix replay contract from section 1 before the new fitter consumes real recordings. +Arbitrary mid-trajectory restoration is not an advancement requirement. Complete the five domain inventories and derive the reduced conditional targets before advancing to real-recording posterior comparisons. Implement the observation likelihood, initial-state prior, immutable data identity, and posterior result format beside the existing fitter. Verify the likelihood against the noise injector, including angle handling, missing measurements, and cached observations. diff --git a/docs/uncertainty/validation-status-20260914.md b/docs/uncertainty/validation-status-20260914.md new file mode 100644 index 0000000000..1b9db9091d --- /dev/null +++ b/docs/uncertainty/validation-status-20260914.md @@ -0,0 +1,104 @@ +# Uncertainty replacement: validation status on September 14 + +The replacement remains in offline Stage B validation. +The production MB parameter fitter, execution estimator and decision rules are unchanged. +These experiments use fixed development programs and recorded interactions; numerical fitting seeds are not agent seeds. +No result below establishes a new solve rate or permission to retire the current estimator. + +## Agreed implementation scope + +Replay each candidate from initialization using the recorded low-level actions. +Arbitrary mid-trajectory restoration is an optional optimization and is not a migration requirement. +Initial-state uncertainty and candidate-specific simulator memory remain part of the inference problem. +Fix reproducible replay defects and missing program mechanisms before attributing residual mismatch to an explicitly declared discrepancy law. +Keep sensor noise fixed and test any discrepancy extension consistently in fitting and future generation. +The [proposal](simplification-proposal.md) records the full implementation decisions. + +## Completed comparisons + +| Domain | Latest evidence | Remaining difficulty | +|---|---|---| +| Domino | Both 64- and 128-particle comparisons are complete. | Agreement does not persist across numerical budgets; fixing the initial state is not a validated shortcut. | +| Fan | Fixed-fixture treatment reduces between-fit positional disagreement from 6.69 mm to 0.619 mm, with improved goal Brier scores on this recording. | Maximum future goal-probability disagreement remains 0.367; most sampled histories assign zero density to the recorded future. | +| Boil | Full reduced fits, forecasts, independent readers and comparison are complete. | The thermal prior is now represented correctly, but both fits retain one initial lineage and predictions remain worse than the incumbent. | +| Bridge | Both resumed fits, complete forecasts and independent readers are complete. | Both fits retain one initial lineage; all methods miss the final clean geometric goal, and the two posterior fits disagree on glue events. | +| Balloons | Both default-guided fits, forecasts and comparison are complete; selective fits and forecasts are complete. | Default-guided predictions remain inconsistent; selective forecast readers require invocation recovery before the full comparison can be assessed. | + +The earlier source-program audit found no learnable parameters in the saved Boil and Bridge programs from the original noisy sweep. +Those remain incomplete-model controls. +The positive Boil and Bridge comparisons instead use separately identified historical programs with learnable dynamics, including their learned defaults. +Their selected-point incumbent rows therefore carry historical information that an uninformative fitting prefix cannot recreate from a broad parameter prior. +This is a known comparison difference, not evidence that sensor noise should be enlarged or priors narrowed after seeing future scores. +See the [historical model controls](historical-model-controls.md) and [Boil incumbent control](boil-incumbent-control.md). + +## Boil reduced-target outcome + +The completed source is `logs/uncertainty_boil_reduced_comparison_20260914/full.json`. +All 42 referenced source hashes were checked after completion. +Both full fitting readers and both complete forecast readers passed. + +| Method | Numerical seed | Bubbling RMSE against clean readings | Final goal probability | Surviving initial lineages | +|---|---:|---:|---:|---:| +| Incumbent selected point | N/A | 0.01915 | 1.0000 | N/A | +| Original supported full target | 410 | 0.33856 | 0.5637 | 1 | +| Original supported full target | 411 | 0.31636 | 0.7691 | 1 | +| Reduced target with independent thermal priors | 410 | 0.30445 | 0.6836 | 1 | +| Reduced target with independent thermal priors | 411 | 0.37039 | 0.5156 | 1 | + +The dimension reduction preserves the original probability model but does not solve exploration of the retained scene and parameters. +It improves bubbling error in one fit and worsens it in the other. +The maximum future goal-probability gap between reduced fits is 0.1680. +One initial lineage is a warning about exploration, not by itself proof that every reported posterior moment is wrong. +The independent-replica and budget evidence remains necessary. + +The original 132-action prefix never activates the burner. +The three heating parameters should retain their prior under that prefix, even when a historically learned default predicts the future better. +New study `logs/uncertainty_boil_informative_heating_20260914` compares conditional heating inference at 132 and 224 actions using the same declared sensor variance and original thermal priors. +It selects two candidate scenes using fitting weight and a fixed index tie-break, holds nonthermal quantities fixed, and checks two quadrature budgets. +The longer prefix is a new development case; observations after action 224 and clean evaluator readings are excluded from its inference. +This study can establish conditional learnability, but cannot establish an adequate full-scene posterior or a live-agent improvement. +Native job `22755839` and independent reader `22755840` completed successfully on `mit_preemptable`. +The follow-up precision check `22756314` and independent reader `22756315` also completed; the 1,024/2,048-node comparison passes its declared moment and log-normalizer tolerances. +Conditional onset and width become concentrated at means 30.12237 and 5.71475, while radius remains uniform in these two selected histories. +The two histories give the same conditional heating target, not independent evidence of full-scene posterior agreement. +See the [informative-heating diagnostic](boil-informative-heating.md) for the original failed precision comparison, prior-retention control and remaining integration requirements. + +## Bridge completed recovery + +The completed source is `logs/uncertainty_bridge_resumed_forecasts_v2_20260914/paired-comparison.json`. +All nine directly referenced sources match their recorded hashes. +The original 64 GiB allocations failed with an out-of-memory condition; the checked 128 GiB continuations completed without changing the numerical target or resetting its budget. + +Numerical seeds 810 and 811 used 10,834 and 10,825 fitting evaluations, respectively, and each ends with one initial lineage. +All three methods, including the incumbent selected point, predict final goal probability zero. +The clean assessment contains the goal at the final frame, while evaluating the same geometric predicate on noisy poses misses it. +Their identical goal Brier score of 1/586 therefore conceals a consequential final-frame error rather than demonstrating success. +The maximum between-fit glue-probability gap is 0.1512, and maximum per-coordinate positional RMS disagreement is 8.49 mm. +The future-density calculation has 26/32 zero-density particles in seed 810 and none in seed 811. +These findings keep predictive and numerical acceptance open. + +The final report's fitting time describes the resumed allocation. +Compute accounting must also retain each original 2:51:51 allocation, recovery checks, initialization, forecast generation and independent verification. +Do not interpret resumed elapsed time as total fitting cost. + +## Balloons invocation recovery + +Selective forecast jobs `22715569` and `22715571` completed, but readers `22715570` and `22715572` failed before their Python verifier started. +Their launcher requires both the report path and numerical index; its submission omitted the second argument and shell expansion failed with `$2: unbound variable`. +The dependent comparison `22715574` was cancelled. +This is a verification-launch failure, not a failed fit, prediction, or agent seed. + +The exact failing expansion was reproduced in `logs/uncertainty_balloons_selective_reader_recovery_20260914/reproduction.json`. +Recovery array `22755276` explicitly supplies report and index to the unchanged frozen launcher and verifier. +Comparison `22755277` depends on successful completion of both readers. +No completed fit or forecast is regenerated, and old frozen source files are not edited. +The recovery manifest pins the original reports, scripts and invocation wrapper. + +## Next acceptance work + +Read and verify the recovered Balloons comparison when it completes. +Use the completed informative-heating diagnostic to construct a separately identified heating-aware joint target, retaining correlations and the original all-off control. +Use their results to choose a numerical or model change rather than repeat the same unstable fits at a larger budget without a specific hypothesis. +Existing conditional scalar-discrepancy, joint-variance and physical-transition experiments remain distinct model choices. +Any new law motivated by these already-inspected future recordings needs a new untouched evaluation before acceptance. +Live posterior integration, matched five-domain continual evaluation, and retirement of the current parameter fitter remain required later stages. diff --git a/scripts/plotting/plot_continual_comparisons.py b/scripts/plotting/plot_continual_comparisons.py new file mode 100644 index 0000000000..f3325db3c9 --- /dev/null +++ b/scripts/plotting/plot_continual_comparisons.py @@ -0,0 +1,261 @@ +"""Capture verified continual cohorts and render the paper's Figure 4. + +Capture once with --capture --paper-root PATH; replot the archived +--snapshot without reading live experiment directories. Costs use +successful runs only. +""" +import argparse +import hashlib +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any, Dict, List + +import matplotlib + +matplotlib.use('Agg') +# pylint: disable=wrong-import-position +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.ticker import FuncFormatter, MaxNLocator + +# pylint: enable=wrong-import-position + +ROOT = Path(__file__).resolve().parent.parent.parent +DOMAINS = ['Boil', 'Domino', 'Fan', 'Bridge', 'Balloons'] +ARMS = [ + 'MB', 'MF', 'agent_continual_program_world_model', + 'agent_continual_oracle_dynamics', 'agent_continual_oracle_scene', + 'agent_continual_zero_shot', 'agent_continual_no_fitting', + 'agent_continual_no_uncertainty' +] +LABELS = [ + 'EMPIRIC', 'Direct agent', 'Standalone sim.', 'Oracle dynamics', + 'Oracle scene', 'Zero-shot model', 'No harness fitting', + 'No explicit uncert.' +] +COLORS = [ + '#087f8c', '#bd5929', '#7467a6', '#397957', '#6b9483', '#5588ad', + '#b19658', '#88929d' +] + + +def capture(paper: Path, target: Path) -> None: + """Verify every selected final scorecard before freezing plot inputs.""" + sys.path.insert(0, str(paper / 'scripts')) + spec = importlib.util.spec_from_file_location( + 'paper_artifacts', paper / 'scripts/build_artifacts.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + rows, _ = module.verified_reported_rows() + reports = [ + ROOT / 'docs/comparisons/continual-results.json', + ROOT / 'docs/comparisons/bridge-three-span-results.json' + ] + sources: List[Dict[str, str]] = [] + for index, report in enumerate(reports): + document = json.loads(report.read_text()) + for row in document['rows']: + if (row['domain'] == 'Bridge') != (index == 1): + continue + assert row['finished'] and row['approach'] in ARMS + path = Path(row['scorecard']) + raw = path.read_bytes() + card = json.loads(raw) + totals = card['totals'] + assert card['finished_at'] and card['end_reason'] in { + 'all_levels_won', 'level_lost', 'level_not_won', 'agent_ended', + 'step_cap', 'wall_clock_cap' + } + assert card['seed'] == row['seed'] and card['arm'] == row[ + 'approach'] + assert document['source_commit'].startswith(card['git_sha']) + assert totals['total_steps'] == row['steps'] == sum( + l['steps'] for l in card['levels']) + assert totals['total_resets'] == row['resets'] == sum( + l['resets'] for l in card['levels']) + assert totals['levels_completed'] == row['wins'] == sum( + l['won'] for l in card['levels']) + assert totals['levels_total'] == row['levels'] + rows.append( + dict(domain=row['domain'], + arm=row['approach'], + seed=row['seed'], + won=row['wins'], + levels=row['levels'], + steps=row['steps'], + resets=row['resets'], + source=str(path), + sha256=hashlib.sha256(raw).hexdigest(), + git_sha=card['git_sha'])) + sources.append({ + 'path': + str(report), + 'sha256': + hashlib.sha256(report.read_bytes()).hexdigest() + }) + assert len(rows) == 115 + assert len({(r['domain'], r['arm'], r['seed']) for r in rows}) == 115 + for domain in DOMAINS: + for arm in ARMS: + assert sum(r['domain'] == domain and r['arm'] == arm + for r in rows) == (2 if arm == 'MF' else 3) + payload = { + 'generated_by': + 'scripts/plotting/plot_continual_comparisons.py', + 'policy': + ('Preserve paper MB (3) and historical MF (2); add six comparison ' + 'arms (3 each). Bridge uses three-span integrity-fixed cohort. ' + 'Steps: whole-run successes only; solve: all levels; resets: all ' + 'finished runs.'), + 'sources': + sources, + 'records': + rows, + 'caveats': + [('Cohorts differ in observation handling and agent runtime; not ' + 'a matched causal ablation.'), + ('Only new Bridge standalone permits engine imports; other ' + 'domains retain stricter historical prompt.'), + ('Some historical standalone agents did not use a model; ' + 'uncertainty arms include known custom uncertainty checks.'), + ('Original no-fitting Bridge seed 0 retained; post-completion ' + 'scheduler repeat excluded.'), + 'Balloons is original non-hatch with historical instantaneous goal.'] + } + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, indent=2) + '\n') + + +def render(snapshot: Path, output: Path) -> None: + """Means plus individual seed values, without small-sample CI claims.""" + data = json.loads(snapshot.read_text()) + rows = data['records'] + plt.rcParams.update({ + 'font.family': 'DejaVu Sans', + 'font.size': 9, + 'svg.fonttype': 'none', + 'pdf.fonttype': 42, + 'svg.hashsalt': 'continual-comparisons' + }) + fig, axes = plt.subplots(3, 5, figsize=(12.8, 8.7), sharey=True) + summary: List[Dict[str, Any]] = [] + for col, domain in enumerate(DOMAINS): + for metric, field in enumerate(['solve', 'steps', 'resets']): + ax = axes[metric, col] + for i, arm in enumerate(ARMS): + group = [ + r for r in rows + if r['domain'] == domain and r['arm'] == arm + ] + eligible = [r for r in group if r['won'] == r['levels'] + ] if field == 'steps' else group + vals = [ + 100 * r['won'] / + r['levels'] if field == 'solve' else r[field] + for r in eligible + ] + avg = float(np.mean(vals)) if vals else None + summary.append( + dict(domain=domain, + arm=arm, + metric=field, + mean=avg, + n=len(vals), + values=vals)) + if i % 2 == 0: + ax.axhspan(i - .48, i + .48, color='#f3f6f7', zorder=0) + if vals: + ax.barh(i, avg, height=.56, color=COLORS[i], zorder=2) + jitter = np.linspace(-.15, .15, + len(vals)) if len(vals) > 1 else [0] + ax.scatter(vals, + i + np.asarray(jitter), + s=12, + facecolors='white', + edgecolors='#263c46', + linewidths=.6, + zorder=3, + clip_on=False) + if field == 'steps': + ax.text(.98, + i, + f'n={len(vals)}' if vals else 'no success', + transform=ax.get_yaxis_transform(), + ha='right', + va='center', + fontsize=7, + color='#344a55', + bbox=dict(facecolor='white', + edgecolor='none', + pad=.6)) + ax.set_ylim(7.6, -.7) + ax.set_yticks(range(8), LABELS, fontsize=10) + ax.spines[['top', 'right', 'left']].set_visible(False) + ax.spines['bottom'].set_color('#aebec5') + ax.tick_params(length=0, pad=4, labelsize=10) + ax.set_axisbelow(True) + ax.grid(axis='x', color='#dce4e7', lw=.6) + if field == 'solve': + ax.set_xlim(-3, 108) + ax.set_xticks([0, 50, 100]) + ax.set_title(domain, + fontsize=12, + fontweight='bold', + color='#203744', + pad=12) + ax.set_xlabel('Levels solved (%)', fontsize=11) + elif field == 'steps': + maximum = max( + (r['steps'] for r in rows + if r['domain'] == domain and r['won'] == r['levels']), + default=1) + ax.set_xlim(0, maximum * 1.30) + ax.xaxis.set_major_locator(MaxNLocator(3)) + ax.xaxis.set_major_formatter( + FuncFormatter(lambda x, _: f'{x/1000:g}k' + if x >= 1000 else f'{x:g}')) + ax.set_xlabel('Successful-run steps', fontsize=11) + else: + ax.set_xlim(left=-.05, right=max(1, ax.get_xlim()[1])) + ax.xaxis.set_major_locator(MaxNLocator(3, integer=True)) + ax.set_xlabel('Resets (all runs)', fontsize=11) + fig.subplots_adjust(left=.095, + right=.99, + top=.94, + bottom=.145, + wspace=.23, + hspace=.30) + output.parent.mkdir(parents=True, exist_ok=True) + for extension in ['pdf', 'svg', 'png']: + fig.savefig(output.with_suffix('.' + extension), + dpi=180, + bbox_inches='tight', + pad_inches=.06) + svg = output.with_suffix('.svg') + svg.write_text('\n'.join(line.rstrip() + for line in svg.read_text().splitlines()) + '\n') + plt.close(fig) + output.with_name(output.name + '-summary.json').write_text( + json.dumps(summary, indent=2) + '\n') + + +def main() -> None: + """Optionally capture a snapshot, then render the figure.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--paper-root', + type=Path, + default=ROOT.parent / 'sim-predicator-paper') + parser.add_argument('--snapshot', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--capture', action='store_true') + args = parser.parse_args() + if args.capture: + capture(args.paper_root, args.snapshot) + render(args.snapshot, args.output) + + +if __name__ == '__main__': + main()