From 789e4124fe72bf1ea1d7c9714a0ede1570e2a8c1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 21 Sep 2026 18:04:23 +0200 Subject: [PATCH 1/2] feat(ladder): the model gate's universe is each host's measured Q4_K inventory; no Q4_K rung is optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator 2026-09-21: "you must ensure all models Q4_K CUDA work; the end". The gate measured a hand-picked list with an escape hatch (qwen3-8b-q4km required: false, red on lambda at 0.68.2 and 0.69.0), and a full fleet sweep then found reds that were on no list at all. This row is #3712's done_when 1 + 2 (the gate side). - contract: `ladder.inventory` {dirs, case-insensitive *q4_k*/*q4k* .gguf/.apr patterns, backends [cuda]}; MCL-INV-006/007 and FALSIFY-MCL-011..014. - model_ladder.sh: measures ladder ∪ the host's inventory with ONE measure() function (a rung and an inventory model cannot drift), and writes receipt v2: `inventory` [{file, sha256, bytes}], `file` per row, `apr_sha` (full 40-hex). - check_model_ladder.sh FAILs on: `required` not true on a Q4_K rung, a Q4_K rung that does not claim cuda, a ladder with no inventory, a receipt that is not v2 / has no or an EMPTY inventory, an inventory model MISSING from the run (named), and an inventory-only model not green on cuda (skip, fallback and rc != 0 are RED). - case table 18 -> 26, each red for its own reason; 3 self-mutants, each killed by its case; `--case` naming no case is RED; the root comes from the script path. qwen3-8b-q4km stays `required: false` in the contract by cop ruling: flipping it turns `pv lint contracts/` red on every PR (the armed ladder-green shape reads the committed 0.68.2 lambda receipt, "Empty output"; measured rc 0 -> 1 with only that flip). The gate now refuses the key at T-2, so the release cannot ship on it; the flip folds into 0.69.1 with the qwen3-8b fix and a green lambda receipt. Refs #3712 Co-Authored-By: Claude Opus 5 (1M context) --- contracts/model-capability-ladder-v1.yaml | 58 ++++- docs/roadmaps/entries/PMAT-3712.yaml | 17 ++ docs/roadmaps/roadmap.yaml | 17 ++ scripts/check_model_ladder.sh | 97 +++++++-- .../decline-no-required-host/ladder.yaml | 16 +- .../must_match | 1 - .../green-grown-ladder/ladder.yaml | 17 +- .../green-grown-ladder/ladder_main.yaml | 7 - .../green-grown-ladder/receipts/gx10.json | 33 ++- .../green-grown-ladder/receipts/lambda.json | 33 ++- .../expected_rc | 0 .../green-inventory-beyond-ladder/ladder.yaml | 36 ++++ .../green-inventory-beyond-ladder/must_match | 1 + .../receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 116 ++++++++++ .../version | 0 .../green-rung-listed-one-host/ladder.yaml | 16 +- .../receipts/gx10.json | 33 ++- .../receipts/lambda.json | 21 +- .../lib/model_ladder_cases/green/ladder.yaml | 16 +- .../green/receipts/gx10.json | 21 +- .../green/receipts/lambda.json | 21 +- .../red-absent-required/ladder.yaml | 16 +- .../red-absent-required/receipts/gx10.json | 15 +- .../red-absent-required/receipts/lambda.json | 20 +- .../red-capability/ladder.yaml | 16 +- .../red-capability/receipts/gx10.json | 20 +- .../red-capability/receipts/lambda.json | 20 +- .../ladder.yaml | 16 +- .../receipts/gx10.json | 20 +- .../receipts/lambda.json | 20 +- .../red-dropped-backend/ladder.yaml | 18 +- .../red-dropped-backend/ladder_main.yaml | 7 - .../red-dropped-backend/must_match | 2 +- .../red-dropped-backend/receipts/gx10.json | 20 +- .../red-dropped-backend/receipts/lambda.json | 20 +- .../red-dropped-host-on-rung/ladder.yaml | 16 +- .../red-dropped-host-on-rung/ladder_main.yaml | 7 - .../receipts/gx10.json | 21 +- .../receipts/lambda.json | 21 +- .../red-dropped-rung/ladder.yaml | 16 +- .../red-dropped-rung/ladder_main.yaml | 7 - .../red-dropped-rung/receipts/gx10.json | 12 +- .../red-dropped-rung/receipts/lambda.json | 12 +- .../red-fallback/ladder.yaml | 16 +- .../red-fallback/receipts/gx10.json | 20 +- .../red-fallback/receipts/lambda.json | 20 +- .../model_ladder_cases/red-golden/ladder.yaml | 16 +- .../red-golden/receipts/gx10.json | 20 +- .../red-golden/receipts/lambda.json | 20 +- .../red-inventory-empty/expected_rc | 1 + .../red-inventory-empty/ladder.yaml | 36 ++++ .../red-inventory-empty/must_match | 1 + .../red-inventory-empty/receipts/gx10.json | 73 +++++++ .../red-inventory-empty/receipts/lambda.json | 84 ++++++++ .../red-inventory-empty/version | 1 + .../red-inventory-model-fell-back/expected_rc | 1 + .../red-inventory-model-fell-back/ladder.yaml | 36 ++++ .../red-inventory-model-fell-back/must_match | 1 + .../receipts/gx10.json | 116 ++++++++++ .../receipts/lambda.json | 84 ++++++++ .../red-inventory-model-fell-back/version | 1 + .../expected_rc | 1 + .../ladder.yaml | 36 ++++ .../must_match | 1 + .../receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 116 ++++++++++ .../version | 1 + .../red-inventory-model-missing/expected_rc | 1 + .../red-inventory-model-missing/ladder.yaml | 36 ++++ .../red-inventory-model-missing/must_match | 1 + .../receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 89 ++++++++ .../red-inventory-model-missing/version | 1 + .../red-ladder-without-inventory/expected_rc | 1 + .../red-ladder-without-inventory/ladder.yaml | 27 +++ .../red-ladder-without-inventory/must_match | 1 + .../receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 84 ++++++++ .../red-ladder-without-inventory/version | 1 + .../red-listed-host-absent/ladder.yaml | 16 +- .../red-listed-host-absent/receipts/gx10.json | 21 +- .../receipts/lambda.json | 21 +- .../red-missing-host/ladder.yaml | 16 +- .../red-missing-host/receipts/lambda.json | 20 +- .../red-q4k-required-false/expected_rc | 1 + .../red-q4k-required-false/ladder.yaml | 36 ++++ .../red-q4k-required-false/must_match | 1 + .../red-q4k-required-false/receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 84 ++++++++ .../red-q4k-required-false/version | 1 + .../red-q4k-rung-cpu-only/expected_rc | 1 + .../ladder.yaml | 9 + .../red-q4k-rung-cpu-only/must_match | 1 + .../receipts/gx10.json | 28 ++- .../receipts/lambda.json | 28 ++- .../red-q4k-rung-cpu-only/version | 1 + .../red-receipt-v1-no-inventory/expected_rc | 1 + .../red-receipt-v1-no-inventory/ladder.yaml | 36 ++++ .../red-receipt-v1-no-inventory/must_match | 1 + .../receipts/gx10.json | 84 ++++++++ .../receipts/lambda.json | 72 +++++++ .../red-receipt-v1-no-inventory/version | 1 + .../red-skipped-is-not-passed/ladder.yaml | 16 +- .../receipts/gx10.json | 20 +- .../receipts/lambda.json | 20 +- .../red-stale-version/ladder.yaml | 16 +- .../red-stale-version/receipts/gx10.json | 20 +- .../red-stale-version/receipts/lambda.json | 20 +- .../red-vacuous-empty/ladder.yaml | 16 +- .../red-vacuous-empty/receipts/gx10.json | 5 +- .../red-vacuous-empty/receipts/lambda.json | 5 +- scripts/model_ladder.sh | 203 ++++++++++++------ 113 files changed, 2739 insertions(+), 365 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-3712.yaml delete mode 100644 scripts/lib/model_ladder_cases/green-cpu-only-rung-skips-capability/must_match rename scripts/lib/model_ladder_cases/{green-cpu-only-rung-skips-capability => green-inventory-beyond-ladder}/expected_rc (100%) create mode 100644 scripts/lib/model_ladder_cases/green-inventory-beyond-ladder/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/green-inventory-beyond-ladder/must_match create mode 100644 scripts/lib/model_ladder_cases/green-inventory-beyond-ladder/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/green-inventory-beyond-ladder/receipts/lambda.json rename scripts/lib/model_ladder_cases/{green-cpu-only-rung-skips-capability => green-inventory-beyond-ladder}/version (100%) create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/must_match create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-empty/version create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/must_match create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-fell-back/version create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/must_match create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-golden-skipped/version create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/must_match create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-inventory-model-missing/version create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/must_match create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-ladder-without-inventory/version create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/must_match create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-q4k-required-false/version create mode 100644 scripts/lib/model_ladder_cases/red-q4k-rung-cpu-only/expected_rc rename scripts/lib/model_ladder_cases/{green-cpu-only-rung-skips-capability => red-q4k-rung-cpu-only}/ladder.yaml (83%) create mode 100644 scripts/lib/model_ladder_cases/red-q4k-rung-cpu-only/must_match rename scripts/lib/model_ladder_cases/{green-cpu-only-rung-skips-capability => red-q4k-rung-cpu-only}/receipts/gx10.json (72%) rename scripts/lib/model_ladder_cases/{green-cpu-only-rung-skips-capability => red-q4k-rung-cpu-only}/receipts/lambda.json (72%) create mode 100644 scripts/lib/model_ladder_cases/red-q4k-rung-cpu-only/version create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/expected_rc create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/ladder.yaml create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/must_match create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/receipts/gx10.json create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/receipts/lambda.json create mode 100644 scripts/lib/model_ladder_cases/red-receipt-v1-no-inventory/version diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index c141463952..68c436e3fc 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -44,10 +44,24 @@ entity: # for that arch; a claimed backend that falls back is RED. # sha256 pins the exact file: a rung measured on a different file is a different # measurement (Q5_K taught us three readers can each invent a layout). +# +# #3712 (operator 2026-09-21: "you must ensure all models Q4_K CUDA work; the end"; +# publishing with "most working" is a "p0 tire fire"): +# * NO Q4_K rung is optional. `required: false` on a Q4_K rung is refused by the +# gate, and so is a Q4_K rung whose `backends` does not claim cuda. +# * The universe is what each host HOLDS, not this list. `inventory` names where +# and what to look for; scripts/model_ladder.sh measures every match on the host +# (on `inventory.backends`) and writes the list into the receipt (schema v2). +# An inventory model missing from the run, or not green on CUDA, is a FAIL +# naming it. A rung is still how a model gets a pinned sha256 and a host list. ladder: hosts: - { id: lambda, isa: x86_64, gpu: "RTX 4090", cc: sm_89, required: true } - { id: gx10, isa: aarch64, gpu: "GB10", cc: sm_121, required: true } + inventory: + dirs: ["~/models", "~/.apr/models", "~/.cache/apr/models"] # depth 1, the fleet's model dirs + patterns: ["*q4_k*.gguf", "*q4k*.gguf", "*q4_k*.apr", "*q4k*.apr"] # case-insensitive + backends: [cuda] rungs: - id: qwen2-1.5b-q4km family: qwen2.5 @@ -71,8 +85,8 @@ ladder: gguf: Qwen3-8B-Q4_K_M.gguf sha256: d98cdcbd03e17ce47681435b5150e34c1417f50b5c0019dd560e4882c5745785 backends: [cpu, cuda] - required: false - note: "second dense-Qwen3 size; required once #3413 lands so the fix is proved at two widths" + required: false # REFUSED by check_model_ladder.sh (#3712): T-2 is RED while this key stands + note: "second dense-Qwen3 size. No Q4_K model is optional (#3712), so the release gate refuses this key and the release cannot ship on it. It is not flipped here because the ARMED ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output — the 0.69.0 hold) and would turn pv lint red on every PR; the flip to true lands with the Qwen3-8B CUDA fix and a committed green lambda receipt" - id: qwen35-0.8b-q4km family: qwen3.5 arch: qwen35 @@ -125,20 +139,22 @@ equations: preconditions: - "binary pinned via scripts/apr_bin.sh (built from HEAD) or DOGFOOD_ALLOW_UNPINNED=1 for the published-crate mode" host_receipt: - formula: "receipt(host) = {host, version, sha, gpu, cc, rungs: [{id, present, capability_match, golden_output, backends: {b: {ran, fallback}}}]}" + formula: "receipt(host) = {schema: apr-model-ladder-receipt/v2, host, version, sha, apr_sha, gpu, cc, inventory: [{file, sha256, bytes}], rungs: [{id, file, present, capability_match, golden_output, backends: {b: {ran, fallback, rc}}}]}" domain: "evidence/dogfood/models//.json, written only by scripts/model_ladder.sh" invariants: - "receipt.version equals the cargo root version of the tree it was measured on" - "receipt.sha equals the git HEAD the binary was built from" - "receipt.executed >= 1 — a receipt that measured nothing is a decline (exit 2), never a pass" + - "receipt.inventory is MEASURED on the host (every file under inventory.dirs matching inventory.patterns), never copied from the ladder; an empty inventory is FAIL" preconditions: - "python3 with json on the host (yaml is read by python3 too)" gate_verdict: - formula: "T2_models = ∀ host required: receipt(host) fresh(version) ∧ ∀ rung required: green(rung, host)" + formula: "T2_models = ∀ host required: receipt(host) fresh(version) ∧ ∀ rung required: green(rung, host) ∧ ∀ f ∈ receipt(host).inventory: green(f, host, inventory.backends)" domain: "scripts/check_model_ladder.sh, invoked by scripts/dogfood.sh through [package.metadata.dogfood]" invariants: - "the rung list at origin/main is the floor: a PR may add rungs, hosts or backends and may not remove any (same construction as check_multiplatform_dogfood.sh layer 2)" - "a missing receipt is FAIL, not DEFER: unlike the multiplatform gate this needs no published crate, a dev build measures it" + - "a Q4_K rung (file or id matches q4_?k) is required and claims cuda — the key `required: false` on one is refused, not tolerated (#3712)" preconditions: - "git can read origin/main:contracts/model-capability-ladder-v1.yaml, or the run is the bootstrap" @@ -168,6 +184,16 @@ proof_obligations: property: "The ladder can grow and cannot shrink in the same PR that reads it" formal: "rungs(origin/main) ⊆ rungs(HEAD) ∧ hosts(origin/main) ⊆ hosts(HEAD) ∧ ∀ rung: backends_main(rung) ⊆ backends_head(rung)" applies_to: gate_verdict + - id: MCL-INV-006 + type: invariant + property: "No Q4_K rung is optional, and every one claims CUDA" + formal: "∀ rung ∈ ladder.rungs: q4k(rung) ⟹ rung.required = true ∧ cuda ∈ rung.backends" + applies_to: gate_verdict + - id: MCL-INV-007 + type: invariant + property: "The universe is the host's measured inventory — every Q4_K model it holds is in the run and green on CUDA" + formal: "∀ host required, ∀ f ∈ receipt(host).inventory: ∃ row ∈ receipt(host).rungs: row.file = f ∧ row.present ∧ (f ∈ files(ladder) ∨ green(row, inventory.backends)) ∧ receipt(host).inventory ≠ ∅" + applies_to: host_receipt falsification_tests: - id: FALSIFY-MCL-001 @@ -220,6 +246,26 @@ falsification_tests: prediction: "narrowing a rung from every host at origin/main to hosts: [gx10] makes the gate exit 1 with 'hosts DROPPED'" test: "bash scripts/check_model_ladder.sh --self-test --case red-dropped-host-on-rung" if_fails: "a required (rung, host) pair could be dropped by adding a key instead of removing one" + - id: FALSIFY-MCL-011 + rule: "A Q4_K rung cannot be made optional or CPU-only" + prediction: "`required: false` on a Q4_K rung exits 1 naming the rung (case red-q4k-required-false); a Q4_K rung with backends [cpu] exits 1 (case red-q4k-rung-cpu-only)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-q4k-required-false && bash scripts/check_model_ladder.sh --self-test --case red-q4k-rung-cpu-only" + if_fails: "the 0.69.0 hold: qwen3-8b-q4km was required: false, so an empty-output CUDA model was a note, not a failure" + - id: FALSIFY-MCL-012 + rule: "An inventory model missing from the run is named" + prediction: "a receipt whose inventory lists a file with no present row exits 1 with 'is MISSING from the run' naming the file (case red-inventory-model-missing)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-missing" + if_fails: "the universe is the list again: a model the host holds is never measured" + - id: FALSIFY-MCL-013 + rule: "An inventory model beyond the ladder is judged on CUDA like a rung" + prediction: "an inventory-only row whose golden_output is skipped, or whose cuda fell back, exits 1 naming inv: (cases red-inventory-model-golden-skipped, red-inventory-model-fell-back); green on cuda exits 0 (case green-inventory-beyond-ladder)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-golden-skipped && bash scripts/check_model_ladder.sh --self-test --case red-inventory-model-fell-back && bash scripts/check_model_ladder.sh --self-test --case green-inventory-beyond-ladder" + if_fails: "an unlisted model is recorded but never judged — present is not green" + - id: FALSIFY-MCL-014 + rule: "A receipt without a measured inventory is not evidence" + prediction: "a v1 receipt (no inventory) exits 1; an empty inventory exits 1; a ladder with no inventory block exits 1 (cases red-receipt-v1-no-inventory, red-inventory-empty, red-ladder-without-inventory)" + test: "bash scripts/check_model_ladder.sh --self-test --case red-receipt-v1-no-inventory && bash scripts/check_model_ladder.sh --self-test --case red-inventory-empty && bash scripts/check_model_ladder.sh --self-test --case red-ladder-without-inventory" + if_fails: "absence scored as conformance: a host that listed nothing proved nothing and passed" qa_gate: id: F-MCL-001 @@ -231,5 +277,7 @@ qa_gate: - required_rungs_present_and_green - claimed_backends_ran_without_fallback - ladder_not_shrunk_vs_origin_main - pass_criteria: "check_model_ladder.sh exits 0 with executed >= 1 on every required host" + - q4k_rungs_required_and_claim_cuda + - every_inventory_model_measured_and_green_on_cuda + pass_criteria: "check_model_ladder.sh exits 0 with executed >= 1 and a non-empty measured inventory on every required host" falsification: "plant a receipt with cuda.fallback=true → exit 1" diff --git a/docs/roadmaps/entries/PMAT-3712.yaml b/docs/roadmaps/entries/PMAT-3712.yaml new file mode 100644 index 0000000000..96b0011a17 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3712.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3712 + github_issue: 3712 + item_type: task + title: Model gate universe = each host's measured Q4_K inventory; no Q4_K rung optional; skip/fallback/rc!=0 RED + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 38f3913d9c..738fb0d83a 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -19946,3 +19946,20 @@ roadmap: labels: - kind:code notes: 'ANDON 2, found by aprender-62 as first responder on #3689: workspace-test-shard 1/3 (job 106309701273, runner yoga-build3) failed ont4c3_parity_receipts::the_committed_tree_agrees_with_its_own_denominator with "scripts/parity_receipt_denominator.sh: line 61: python3: command not found" for every receipt. classify() then printed nothing, the loop read that as "other", and the script said "the tree holds 0", exit 1. That is a missing interpreter scored as a count, the class of the git refusal in #3682. The cop ruled (~11:40Z) to fold it into #3689''s push under four conditions, which are done_when 1-5 above. SCOPE: scripts/parity_receipt_denominator.sh, crates/aprender-contracts-cli/tests/ont4c3_parity_receipts.rs, and this fragment. HOW IT IS MET. PY_BIN (seam PARITY_PYTHON, default python3). count_and_check checks `command -v "$PY_BIN"` after listing the tree and before any file: absent -> UNMEASURED line to stderr, return 3, which verify and --print propagate. Each file''s class must now be exactly record|legacy|other; anything else, including nothing, is "ENV the classifier () gave no answer for ", return 2 (the old `*) :` fall-through was the defect). --self-test on a runner with no interpreter prints an UNMEASURED line and exits 0: the classification rows cannot run there. The Rust test accepts exit 3 only when stderr carries both "UNMEASURED runner=" and "reason=no-interpreter interpreter="; any other non-success still fails. MEASURED at this commit: `--self-test` rc 0 with 7 rows, the 5 existing plus "no interpreter is UNMEASURED exit 3 naming it, never a count of 0" and "an interpreter that answers nothing is ENV exit 2, never a count of 0"; a bare run with python3 gives rc 0, "PASS 7 receipt(s) under evidence/parity/**, and evidence/parity/EXPECTED_RECEIPTS says 7." (the T-2 requirement, measured); PARITY_PYTHON=/nonexistent gives rc 3 "UNMEASURED runner=nopy-probe reason=no-interpreter interpreter=/nonexistent/python3 …"; PARITY_PYTHON=/bin/true gives rc 2 "ENV the classifier (/bin/true) gave no answer for evidence/parity/derived_expiries.json". THE YOGA SHAPE, a PATH made of every /usr/bin and /bin executable except python*: the PRE-FIX script (25d2ee264) prints "python3: command not found" 118 times, then "FAIL evidence/parity/EXPECTED_RECEIPTS says 7; the tree holds 0.", rc 1, which is #3689''s failure reproduced; the fix prints "UNMEASURED runner=yoga-shape reason=no-interpreter interpreter=python3 …", rc 3. MUTANT, the silent-classifier arm restored to the old `*) :` fall-through: self-test rc 1, "FAIL a silent interpreter gave exit 1: FAIL evidence/parity/EXPECTED_RECEIPTS says 2; the tree holds 0." THE RUST TEST: `cargo test -p aprender-contracts-cli --test ont4c3_parity_receipts` gives 9 passed with python3; PARITY_PYTHON=/nonexistent/python3 gives 1 passed, printing the UNMEASURED line; PARITY_PYTHON=/bin/true gives FAILED, "exit Some(2)" plus the ENV line, so a crash is still RED and only the named UNMEASURED line buys exit 3. `cargo fmt --all -- --check` rc 0.' +- id: PMAT-3712 + github_issue: 3712 + item_type: task + title: Model gate universe = each host's measured Q4_K inventory; no Q4_K rung optional; skip/fallback/rc!=0 RED + status: in_progress + priority: critical + assigned_to: null + created: '2026-09-21T00:00:00Z' + updated: '2026-09-21T00:00:00Z' + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'ACCEPTANCE (hand-entered from gh#3712; this row = done_when 1 + 2, the GATE side; cop assignment 2026-09-21: worker B). Issue done_when 1, verbatim: "Universe = measured inventory, not a list. The release gate derives the model set from what''s ON each required host (every `*Q4_K*` GGUF/APR under the declared models dir on lambda AND gx10), unioned with the ladder. A model present on a host but missing from the gate''s run is a FAIL naming it. The ladder has no `required: false` for any Q4_K rung; a guard refuses the key on a Q4_K rung (case row + mutant)." Issue done_when 2, verbatim: "Every (model, host) cell is GREEN on CUDA: capability_match passed (not skipped), golden_output passed (not skipped; #3711), `apr run --gpu` rc 0 with no fallback line. One red cell = NO-GO. There is no known, optional or pre-existing exemption, and no threshold like >= N%." WHAT THIS ROW DOES: (a) contracts/model-capability-ladder-v1.yaml gains `ladder.inventory` {dirs, patterns (case-insensitive *q4_k*/*q4k* .gguf/.apr), backends [cuda]}; (b) scripts/model_ladder.sh measures ladder UNION the host''s inventory (every match, depth 1), judges each inventory model on cuda with the SAME measure() as a rung, and writes receipt schema apr-model-ladder-receipt/v2 carrying `inventory` [{file, sha256, bytes}], `file` per row, and `apr_sha` (full 40-hex, for #3715); (c) scripts/check_model_ladder.sh FAILs: `required` not true on a Q4_K rung; a Q4_K rung that does not claim cuda; a ladder with no inventory; a receipt that is not v2 or has no inventory; an EMPTY inventory; an inventory file with no present row (MISSING from the run, named); an inventory-only model not green on cuda (skip, fallback, rc != 0 are RED); (d) case table grows 18 -> 26 (every case red for exactly its own reason, one FAIL line each except the two-host cases), plus 3 self-mutants each killed by its case (q4k-required-false, q4k-without-cuda, inventory-missing); --case with no such case is now RED, and the root is derived from the script path, not git rev-parse (#3581). WHAT IT DOES NOT DO (cop ruling 2026-09-21): qwen3-8b-q4km stays `required: false` in the contract, and the gate refuses that key at T-2, so the release cannot ship on it. Flipping it here would turn pv lint red on EVERY PR: the armed ladder-green shape reads the committed 0.68.2 lambda receipt (golden_output: Empty output), measured rc 0 -> 1 with only that flip. The flip folds into the 0.69.1 batch with the qwen3-8b fix and a committed green lambda receipt. So done_when 1''s `no required: false` clause is NOT met by this row: Refs, not Closes. Done_when 2''s cells turn green only through the model fixes under EPIC #3710; done_when 3 and 4 are aprender-f0''s (#3708). cells[] per verb x context (the widened bar, #3715 proposal) ships with the cop''s widened-bar delta, not here.' diff --git a/scripts/check_model_ladder.sh b/scripts/check_model_ladder.sh index 40be43269b..d22ef8a854 100755 --- a/scripts/check_model_ladder.sh +++ b/scripts/check_model_ladder.sh @@ -3,7 +3,17 @@ # # Reads one receipt per REQUIRED host (written by scripts/model_ladder.sh) for # the version being cut and is green only when every required rung is present -# and green on every one of them. It is declared in Cargo.toml +# and green on every one of them, AND every model the host HOLDS is too. +# +# #3712 (operator 2026-09-21: "you must ensure all models Q4_K CUDA work; the end", +# and publishing with "most working" is a "p0 tire fire"). Three rules, no exemptions +# and no thresholds: +# * No Q4_K rung is optional. `required: false` on a Q4_K rung is REFUSED, and so is +# a Q4_K rung that does not claim cuda. +# * The universe is the host's measured inventory. A receipt must be schema v2 and +# carry a non-empty `inventory`. Every inventory model must appear in the run and be +# green on CUDA, or it is a FAIL naming it. +# * A skipped capability_match or golden_output, a fallback line, or a run rc != 0 is RED. It is declared in Cargo.toml # [package.metadata.dogfood] so scripts/dogfood.sh runs it in every phase; a # missing receipt is FAIL, not DEFER — a dev build can measure this, no # published crate is needed. @@ -31,11 +41,15 @@ while [ $# -gt 0 ]; do --ladder) [ $# -ge 2 ] || { echo "--ladder needs a value" >&2; exit 2; }; LADDER="$2"; shift 2 ;; --ladder-main) [ $# -ge 2 ] || { echo "--ladder-main needs a value" >&2; exit 2; }; LADDER_MAIN_OVERRIDE="$2"; shift 2 ;; --version) [ $# -ge 2 ] || { echo "--version needs a value" >&2; exit 2; }; VERSION_OVERRIDE="$2"; shift 2 ;; - -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + -h|--help) awk 'NR == 1 { next } !/^#/ { exit } { sub(/^# ?/, ""); print }' "$0"; exit 0 ;; *) echo "check_model_ladder: unknown argument '$1'" >&2; exit 2 ;; esac done -cd "$(git rev-parse --show-toplevel 2>/dev/null || pwd)" || exit 2 +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" # before the cd: the mutants copy this file +# The root is derived from this file's path, never from `git rev-parse` (it dies in the CI container, +# aprender#3581) or from the caller's cwd. MODEL_LADDER_ROOT is how a mutant copy, which lives in a +# temp dir, is told the tree it judges. +cd "${MODEL_LADDER_ROOT:-$(dirname "$SELF")/..}" || exit 2 # ---------------------------------------------------------------- the judge # judge → exit 0/1/2 @@ -52,6 +66,32 @@ rungs = L.get("rungs", []) if not hosts: print("decline: ladder names no required host"); sys.exit(2) if not rungs: print("decline: ladder has no rungs"); sys.exit(2) rc = 0 +import re +def is_q4k(r): # a Q4_K model, by its file or its id (#3712) + return bool(re.search(r"q4_?k", f"{r.get('gguf', '')} {r.get('id', '')}", re.I)) +def why_of(x, backends): # every reason a measured row is not green on the claimed backends + why = [] + cm, go = x.get("capability_match") or {}, x.get("golden_output") or {} + claims_gpu = bool({"cuda", "gpu"} & set(backends)) + cap_ok = (cm.get("passed") and not cm.get("skipped")) or (cm.get("skipped") and not claims_gpu) + if not cap_ok: why.append("capability_match " + ("SKIPPED" if cm.get("skipped") else "FAIL") + ": " + str(cm.get("message", ""))[:60]) + if not (go.get("passed") and not go.get("skipped")): why.append("golden_output " + ("SKIPPED" if go.get("skipped") else "FAIL") + ": " + str(go.get("message", ""))[:60]) + be = x.get("backends") or {} + for b in backends: + v = be.get(b) + if v is None: why.append(f"{b}: not measured") + elif v.get("fallback"): why.append(f"{b}: FELL BACK — the claimed backend did not run") + elif not v.get("ran"): why.append(f"{b}: did not run (rc={v.get('rc')})") + return why +# #3712: no Q4_K rung is optional, and every one claims cuda. The key is refused, not tolerated. +for r in rungs: + if is_q4k(r) and r.get("required") is not True: + print(f"FAIL rung {r['id']} is a Q4_K rung with required: {r.get('required')!r} — no Q4_K model is optional; every one must be green on CUDA (#3712)"); rc = 1 + if is_q4k(r) and "cuda" not in (r.get("backends") or []): + print(f"FAIL rung {r['id']} is a Q4_K rung that does not claim cuda — every Q4_K model must be green on CUDA (#3712)"); rc = 1 +inv_backends = list((L.get("inventory") or {}).get("backends") or []) +if not (L.get("inventory") or {}).get("patterns") or "cuda" not in inv_backends: + print("FAIL the ladder declares no inventory (patterns + backends incl. cuda) — the universe cannot be the host's measured Q4_K models (#3712)"); rc = 1 # anti-shrink vs origin/main if main_p and os.path.exists(main_p): try: @@ -89,9 +129,28 @@ for h in hosts: print(f"FAIL {h['id']:7} receipt is for {R.get('version')!r}, this cut is {version!r} — STALE"); rc = 1; continue if int(R.get("executed", 0)) < 1: print(f"FAIL {h['id']:7} receipt executed=0 — a receipt that measured nothing is not evidence"); rc = 1; continue + inv = R.get("inventory") + if R.get("schema") != "apr-model-ladder-receipt/v2" or not isinstance(inv, list): + print(f"FAIL {h['id']:7} receipt carries no measured inventory (schema {R.get('schema')!r}) — the universe is what the host HOLDS, not a list (#3712)"); rc = 1; continue + if not inv: + print(f"FAIL {h['id']:7} measured inventory is EMPTY — a host holding no Q4_K model proved nothing (#3712)"); rc = 1; continue by = {r.get("id"): r for r in R.get("rungs", [])} + by_file = {x.get("file"): x for x in R.get("rungs", []) if x.get("file")} + ladder_files = {r.get("gguf") for r in rungs} + inv_green = 0 + for item in inv: + f = item.get("file") + x = by_file.get(f) + if x is None or not x.get("present"): # held by the host, absent from the run + print(f"FAIL {h['id']:7} inventory model {f} is MISSING from the run — the host holds it, so the release must prove it (#3712)"); rc = 1; continue + if f in ladder_files: + continue # a ladder rung: judged, required, in the rung loop below + why = why_of(x, inv_backends) + if why: print(f"FAIL {h['id']:7} inv:{f:22} " + "; ".join(why)); rc = 1 + else: inv_green += 1; print(f"ok {h['id']:7} inv:{f:22} green on {','.join(inv_backends)}") + print(f"ok {h['id']:7} inventory: {len(inv)} Q4_K model(s) held, every one in the run") for r in rungs: - rid = r["id"]; req = bool(r.get("required")) + rid = r["id"]; req = bool(r.get("required")) or is_q4k(r) x = by.get(rid) tag = "required" if req else "optional" listed = r.get("hosts") @@ -103,18 +162,7 @@ for h in hosts: continue if x.get("sha_ok") is False: print(f"FAIL {h['id']:7} {rid:22} sha256 mismatch — a different file is a different measurement"); rc = 1; continue - why = [] - cm, go = x.get("capability_match") or {}, x.get("golden_output") or {} - claims_gpu = bool({"cuda", "gpu"} & set(r.get("backends", []))) - cap_ok = (cm.get("passed") and not cm.get("skipped")) or (cm.get("skipped") and not claims_gpu) - if not cap_ok: why.append("capability_match " + ("SKIPPED" if cm.get("skipped") else "FAIL") + ": " + str(cm.get("message",""))[:60]) - if not (go.get("passed") and not go.get("skipped")): why.append("golden_output " + ("SKIPPED" if go.get("skipped") else "FAIL") + ": " + str(go.get("message",""))[:60]) - be = x.get("backends") or {} - for b in r.get("backends", []): - v = be.get(b) - if v is None: why.append(f"{b}: not measured") - elif v.get("fallback"): why.append(f"{b}: FELL BACK — the claimed backend did not run") - elif not v.get("ran"): why.append(f"{b}: did not run (rc={v.get('rc')})") + why = why_of(x, r.get("backends", [])) if why: if req: print(f"FAIL {h['id']:7} {rid:22} " + "; ".join(why)); rc = 1 else: print(f"warn {h['id']:7} {rid:22} ({tag}) " + "; ".join(why)) @@ -144,6 +192,23 @@ if [ "$SELF_TEST" = 1 ]; then fi done if [ "$n" -lt 6 ] && [ -z "$ONLY_CASE" ]; then echo "FAIL only $n case(s) ran; the table needs >= 6 to discriminate"; bad=$((bad+1)); fi + if [ -n "$ONLY_CASE" ] && [ "$n" -eq 0 ]; then echo "FAIL no case named $ONLY_CASE under $CASES_DIR -- a case that did not run is not a pass"; bad=$((bad+1)); fi + # Mutants (#3712): each refusal is deleted in a copy of this script, and the case that + # names it must go RED under the copy. A rule no case can tell from its absence is theater. + if [ -z "$ONLY_CASE" ]; then + mdir=$(mktemp -d "${TMPDIR:-/tmp}/ladder-mut.XXXXXX") || exit 2 + mutant() { # mutant