fix(ladder): judge the run verb's output too (#3928) — DRAFT, 0.70.0

docs/roadmaps/entries/PMAT-3928.yaml

@@ -0,0 +1,17 @@
+- id: PMAT-3928
+  github_issue: 3928
+  item_type: task
+  title: 'ladder: judge the run verb''s output too (#3928) — re-review fixes'
+  status: planned
+  priority: high
+  assigned_to: null
+  created: 2026-09-23T19:21:03Z
+  updated: 2026-09-23T19:21:03Z
+  spec: null
+  acceptance_criteria:
+  - 'Cop ruling (B) 2026-09-23 on #3932. (1) why-builder names run; (2) exit codes 3/4 restored on chat/code truncation paths; (3) real gx10 CPU run --verbose capture for layer 3 (both backends, per #3928 acceptance).'
+  phases: []
+  subtasks: []
+  estimated_effort: null
+  labels: []
+  notes: null

docs/roadmaps/roadmap.yaml

@@ -21208,3 +21208,20 @@ roadmap:
   labels:
   - kind:docs
   notes: null
+- id: PMAT-3928
+  github_issue: 3928
+  item_type: task
+  title: 'ladder: judge the run verb''s output too (#3928) — re-review fixes'
+  status: planned
+  priority: high
+  assigned_to: null
+  created: 2026-09-23T19:21:03Z
+  updated: 2026-09-23T19:21:03Z
+  spec: null
+  acceptance_criteria:
+  - 'Cop ruling (B) 2026-09-23 on #3932. (1) why-builder names run; (2) exit codes 3/4 restored on chat/code truncation paths; (3) real gx10 CPU run --verbose capture for layer 3 (both backends, per #3928 acceptance).'
+  phases: []
+  subtasks: []
+  estimated_effort: null
+  labels: []
+  notes: null

scripts/check_ladder_output_judged.sh

@@ -118,6 +118,20 @@ cat <<'JSON'
 JSON
 }
 
+# #3928: the same row with `run` as the offender. Planted separately from the chat
+# fixture because "the detector stopped flagging" and "the verdict stopped reading it"
+# are different failures, and #3901 was the fix that only closed one of them.
+row_run_garbage() {
+cat <<'JSON'
+{"cuda":{"ran":true,"fallback":false,"escaped_special":false,"rc":0,
+ "verbs":{"run":{"ran":true,"rc":0,"output_bad":"gibberish (fragment ' zombie' repeats 3+ times)"},
+          "chat":{"ran":true,"rc":0},
+          "code":{"ran":true,"rc":0},
+          "serve":{"probed":true,"teardown":"clean","routes":{
+            "/v1/completions|stream=false":{"http":200,"output_bad":null}}}}}}
+JSON
+}
+
 QA_OK='{"capability_match":{"passed":true,"skipped":false,"message":"ok"},
         "golden_output":{"passed":true,"skipped":false,"message":"ok"},
         "gates":{},"gates_failed":[],"gates_reported":2}'
@@ -131,14 +145,43 @@ green_of() { # green_of <src> <be-json> -> true|false
 
 check_verdict() { # -> 0 ok
   local src="$1" g
-  g=$(green_of "$src" "$(row_chat_garbage)") || return 2
-  if [ "$g" = "false" ]; then
-    printf '  ok    %-30s green=false\n' "chat-rc0-bad-output"
-    return 0
-  fi
-  printf '  FAIL  %-30s green=%s — a verb that ran and produced garbage is recorded as working\n' \
-    "chat-rc0-bad-output" "$g"
-  return 1
+  local rc=0
+  for case in chat run; do
+    if [ "$case" = chat ]; then g=$(green_of "$src" "$(row_chat_garbage)") || return 2
+    else g=$(green_of "$src" "$(row_run_garbage)") || return 2; fi
+    if [ "$g" = "false" ]; then
+      printf '  ok    %-30s green=false\n' "$case-rc0-bad-output"
+    else
+      printf '  FAIL  %-30s green=%s -- a verb that ran and produced garbage is recorded as working\n' \
+        "$case-rc0-bad-output" "$g"; rc=1
+    fi
+  done
+  return $rc
+}
+
+# A red row must SAY why (#3932 re-review). The verdict reads `run` since #3928; the
+# explanation builder did not, so a row red only on run output printed `unknown` --
+# #3901/#3902's defect, a third time. Same extraction as check_ladder_serve_verdict.sh.
+extract_why() {
+  local src="$1" body
+  body=$(awk '/why=.*python3 -c/{f=1; next} f && /^print\(/{print; exit} f' "$src")
+  grep -q 'w.append' <<< "$body" || { echo "  the extracted why builder does not append reasons" >&2; return 2; }
+  printf '%s' "$body" | sed "s/')\$//"
+}
+
+RED_ONLY_ON_RUN_OUTPUT='{"capability_match":{"passed":true,"skipped":false,"message":"ok"},
+ "golden_output":{"passed":true,"skipped":false,"message":"ok"},
+ "backends":'"$(row_run_garbage)"'}'
+
+check_run_reason() { # -> 0 the run-only red names run, 1 it does not
+  local src="$1" why got
+  why=$(extract_why "$src") || return 2
+  got=$(printf '%s' "$RED_ONLY_ON_RUN_OUTPUT" | python3 -c "$why" 2>/dev/null)
+  case "$got" in
+    *'verb `run` RAN (rc=0) and produced bad output'*)
+      printf '  ok    %-30s %s\n' "reason:run-output" "${got:0:90}" ;;
+    *) printf '  FAIL  %-30s a row red only on run output says: %s\n' "reason:run-output" "${got:-empty}"; return 1 ;;
+  esac
 }
 
 # ── layer 3: WHAT TEXT IS JUDGED (#3925) ─────────────────────────────────────
@@ -223,16 +266,87 @@ Loading model...
 T
 }
 
+# #3928: `apr run --verbose`. VERBATIM from gx10 -- the chatter is the point: kernel
+# counts, VRAM figures and a list of hex pointers, none of which is the model speaking.
+cap_run_verbose() { cat <<'T'
+[GH-480] Patched 2 backward branch(es) for sm_121 JIT workaround
+[trueno#243] Manual graph: 591 kernels. first_args=Some(["0xe326a87db200", "0xe326a87e2e00", "0xe326b87a4000"]), last_args=Some(["0xe326e0320000", "0xe32427c00000"])
+Backend: GPU (NVIDIA GB10, 122502 MB VRAM)
+[DEBUG] generated token ids: [151668, 271, 785, 6722, 315, 9625, 374, 12095, 13]
+
+Output:
+The capital of France is Paris.
+
+Completed in 5.08s (cached)
+T
+}
+# #3928 acceptance asks for BOTH backends. The CPU capture, VERBATIM: gx10, 2026-09-23,
+# apr 0.69.1 (7168e4b68) sha256 01e821432fe8c463…, Qwen3-1.7B-Q4_K_M.gguf sha256 b139949c5bd74937…,
+# the ladder's own invocation (`run --prompt … --max-tokens 16 --verbose --no-gpu`), rc=0. Its chrome
+# differs from the GPU one: `verbose:` preamble, the formatted prompt, the raw decode.
+cap_run_verbose_cpu() { cat <<'T'
+verbose: apr 0.69.1
+verbose: offline = off
+verbose: contract gate = enforced over 1 path(s)
+verbose: model = /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf (1107409472 bytes)
+=== APR Run ===
+
+Source: /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf
+Using mmap for 1056MB model
+[DEBUG] has_chat_template=true, filename_instruct=false
+[DEBUG] formatted_prompt="<|im_start|>user\nWhat is the capital of France? Answer briefly.<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+[DEBUG] add_bos=false, encoded 22 tokens: [151644, 872, 198, 3838, 374, 279, 6722, 315, 9625, 30, 21806, 26753, 13, 151645, 198, 151644, 77091, 198, 151667, 271, 151668, 271]
+Loading model: /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf
+[BOS-FALLBACK] No tokenizer.ggml.bos_token_id in GGUF — using architecture default for 'qwen3'
+Architecture: Transformer [GGUF: qwen3] (28 layers, vocab_size=151936)
+Config: hidden_size=2048, context_length=40960, quant=mixed(Q4_K×168,Q6_K×28) lm_head=Q6_K, threads=20
+Model loaded in 799.4ms
+Backend: CPU (SIMD-accelerated)
+[DEBUG] input_count=22, total_tokens=29, generated_count=7
+[DEBUG] generated token ids: [785, 6722, 315, 9625, 374, 12095, 13]
+[DEBUG] raw decoded: "The capital of France is Paris."
+
+Output:
+The capital of France is Paris.
+
+Completed in 7.04s (cached)
+T
+}
+cap_run_degenerate() { cat <<'T'
+Backend: GPU (NVIDIA GB10, 122502 MB VRAM)
+
+Output:
+zombie zombie zombie zombie
+
+Completed in 5.08s (cached)
+T
+}
+# `Output:` opened and never closed -- the process was cut off mid-reply, so whatever
+# is there is a fragment. A refusal to judge, never a clean reply.
+cap_run_unterminated() { cat <<'T'
+Backend: GPU (NVIDIA GB10, 122502 MB VRAM)
+
+Output:
+The capital of France is Paris.
+T
+}
+
 # name|capture-fn|expect   (clean = no reason · bad = a verdict about the reply ·
-# red = a refusal to judge, which must never be silent)
+# red:<what> = a refusal to judge, which must never be silent, NAMING what is missing:
+# envelope · reply · unclosed. The first cut of #3928 swapped exit codes 3 and 4, so
+# a capture WITH an envelope was reported as missing one; a bare `red` could not see it.)
 extraction_cases() {
 cat <<'CASES'
 chrome-wrapped-correct-answer|cap_correct_answer|clean
 code-json-envelope-not-uuid|cap_code_json|clean
 chrome-does-not-mask-gibberish|cap_chrome_gibberish|bad
 reply-containing-You-judged-whole|cap_reply_with_you|bad
-truncated-capture-no-envelope|cap_no_envelope|red
-envelope-but-no-reply|cap_no_reply|red
+truncated-capture-no-envelope|cap_no_envelope|red:envelope
+envelope-but-no-reply|cap_no_reply|red:reply
+run-verbose-chatter-not-judged|cap_run_verbose|clean
+run-verbose-cpu-chatter-not-judged|cap_run_verbose_cpu|clean
+run-degenerate-reply-in-block|cap_run_degenerate|bad
+run-output-never-closed|cap_run_unterminated|red:unclosed
 CASES
 }
 
@@ -242,8 +356,15 @@ run_extraction_table() { # -> 0 all as expected
     [ -n "$name" ] || continue
     got=$(judge_capture "$src" "$($fn)") || return 2
     if [ -z "$got" ]; then cls=clean
-    elif case "$got" in "could not"*) true ;; *) false ;; esac; then cls=red
-    else cls=bad; fi
+    else
+      case "$got" in
+        "could not find the backend envelope"*) cls=red:envelope ;;
+        "could not locate the"*)                cls=red:reply ;;
+        "could not find the closing"*)          cls=red:unclosed ;;
+        "could not"*)                           cls=red:unnamed ;;
+        *)                                      cls=bad ;;
+      esac
+    fi
     if [ "$cls" = "$want" ]; then
       printf '  ok    %-34s %s\n' "$name" "$cls"
     else
@@ -255,12 +376,12 @@ run_extraction_table() { # -> 0 all as expected
 
 if [ "$SELF_TEST" = 1 ]; then
   [ -f "$SCRIPT" ] || { echo "cannot read $SCRIPT" >&2; exit 2; }
-  m1=$(mktemp); m2=$(mktemp); m3=$(mktemp); m4=$(mktemp)
-  trap 'rm -f "$m1" "$m2" "$m3" "$m4"' EXIT
+  m1=$(mktemp); m2=$(mktemp); m3=$(mktemp); m4=$(mktemp); m5=$(mktemp); m6=$(mktemp); m7=$(mktemp); m8=$(mktemp)
+  trap 'rm -f "$m1" "$m2" "$m3" "$m4" "$m5" "$m6" "$m7" "$m8"' EXIT
 
   echo "self-test: the shipped script"
   run_detector_table "$SCRIPT" > /dev/null && check_verdict "$SCRIPT" > /dev/null \
-    && run_extraction_table "$SCRIPT" > /dev/null \
+    && check_run_reason "$SCRIPT" > /dev/null && run_extraction_table "$SCRIPT" > /dev/null \
     || { echo "SELF-TEST FAILED: the shipped script is already red" >&2; exit 1; }
   echo "  GREEN (expected)"
 
@@ -310,7 +431,59 @@ if [ "$SELF_TEST" = 1 ]; then
   fi
   echo "  RED (expected)"
 
-  echo "self-test: PASS — red when the detector stops flagging, when the verdict stops reading it, when an unlocatable reply passes silently, and when the reply is cut short"
+  # Mutant 5: the extractor is bypassed and the raw capture is judged again -- the
+  # exact state #3925 fixed, and so the one regression this file exists to catch.
+  # aprender-3e ran it by hand while folding #3926 and it was caught; naming it here
+  # means the next person does not have to re-derive that, and a refactor that
+  # reintroduces raw judging fails rather than being noticed in review.
+  sed 's#| assistant_reply); rc=$?#| cat); rc=$?#' "$SCRIPT" > "$m5"
+  cmp -s "$SCRIPT" "$m5" && { echo "SELF-TEST INCONCLUSIVE: mutant 5 changed nothing" >&2; exit 1; }
+  echo "self-test: mutant 5 (extractor bypassed, the transcript judged raw again)"
+  if run_extraction_table "$m5" > /dev/null 2>&1; then
+    echo "SELF-TEST FAILED: mutant 5 passed -- the chrome is being judged and nothing noticed" >&2
+    exit 1
+  fi
+  echo "  RED (expected)"
+
+  # Mutant 6 (#3928): when `Completed in` never arrives, take the rest of the capture
+  # instead of refusing. A cut-off run then reads as a clean reply -- the silent-pass
+  # hazard again, on the path added for `run`.
+  sed -e 's|^elif oi >= 0 and ci >= 0:|elif oi >= 0:|' \
+      -e 's|^    body = lines\[oi + 1:ci\]|    body = lines[oi + 1:]|' "$SCRIPT" > "$m6"
+  cmp -s "$SCRIPT" "$m6" && { echo "SELF-TEST INCONCLUSIVE: mutant 6 changed nothing" >&2; exit 1; }
+  echo "self-test: mutant 6 (an unterminated run reply is taken as whole)"
+  if run_extraction_table "$m6" > /dev/null 2>&1; then
+    echo "SELF-TEST FAILED: mutant 6 passed -- a cut-off run reply reads as clean" >&2
+    exit 1
+  fi
+  echo "  RED (expected)"
+
+  # Mutant 7 (#3928): `run` drops out of the green expression while chat and code stay.
+  # Mutant 2 removes the output_bad check for EVERY verb at once, so it cannot tell
+  # whether `run` was ever wired in -- which is exactly how #3886 folded serve and left
+  # chat and code unread in the same structure. One verb, one mutant.
+  sed 's|and verb_ok(v, "run") and verb_ok(v, "chat")|and verb_ok(v, "chat")|' "$SCRIPT" > "$m7"
+  cmp -s "$SCRIPT" "$m7" && { echo "SELF-TEST INCONCLUSIVE: mutant 7 changed nothing" >&2; exit 1; }
+  echo "self-test: mutant 7 (green stops reading the run verb)"
+  if check_verdict "$m7" > /dev/null 2>&1; then
+    echo "SELF-TEST FAILED: mutant 7 passed -- run garbage no longer reddens the row" >&2
+    exit 1
+  fi
+  echo "  RED (expected)"
+
+  # Mutant 8 (#3932 re-review): the explanation builder stops naming `run`. The row
+  # stays red (mutant 7's check still passes) and says `unknown` -- the verdict moved
+  # and the explanation did not.
+  sed 's|for _vn in ("run", "chat", "code"):|for _vn in ("chat", "code"):|' "$SCRIPT" > "$m8"
+  cmp -s "$SCRIPT" "$m8" && { echo "SELF-TEST INCONCLUSIVE: mutant 8 changed nothing" >&2; exit 1; }
+  echo "self-test: mutant 8 (the red reason stops naming the run verb)"
+  if check_run_reason "$m8" > /dev/null 2>&1; then
+    echo "SELF-TEST FAILED: mutant 8 passed -- a row red only on run output explains nothing" >&2
+    exit 1
+  fi
+  echo "  RED (expected)"
+
+  echo "self-test: PASS — red when the detector stops flagging, when the verdict stops reading it, when an unlocatable reply passes silently, when the reply is cut short, when the extractor is bypassed entirely, when an unterminated run reply is taken as whole, when green stops reading the run verb, and when the red reason stops naming it"
   exit 0
 fi
 
@@ -319,6 +492,7 @@ rc=0
 run_detector_table "$SCRIPT" || rc=$?
 [ "$rc" = 2 ] && exit 2
 check_verdict "$SCRIPT" || rc=1
+check_run_reason "$SCRIPT" || rc=1
 run_extraction_table "$SCRIPT" || rc=$?
 [ "$rc" = 2 ] && exit 2
 if [ "$rc" = 0 ]; then