pgw#999 (P0): the classified adopt refusal reaches the wire, and the rig covers the gate that produced it

changelog.d/pgw999.md

@@ -0,0 +1,34 @@
+### Fixed
+
+- **A refused delegated cell names WHY (pgw#999).** `adopt_delegated_mint`
+  spent a classified `AdoptOutcome` on `bool(...)`, so a mint that sealed and
+  finalized 36/36 entries was refused by three events that all said "a cell
+  this runtime could not adopt". Attempt 26 paid 2 h 45 m and $2.72 of L40S for
+  that sentence. The classification now reaches the wire: `phase` carries the
+  CLASS (`contract_invalid`, `constants_constant_unresolved`,
+  `no_arm_for_mode`, `numerics_refused`, `cell_selection_bug`, …) the way
+  `self_mint_skipped` already does, the detail quotes the gate's own sentence,
+  and `DelegatedResult.reason` carries it up so the executor's decline and the
+  terminal error quote the same token instead of naming their own call sites.
+- The `except Exception` branch that logged an adoption failure to a logger no
+  pod exposes now emits it — `AdoptError.reason` when the exception carries
+  one, the exception TYPE when it does not. Log-only swallowing of an important
+  error is a defect class here, not a style preference.
+
+- **The same discard, one frame deeper.** `provision.arm_aot`'s bucket branch
+  caught a failed lifted-LoRA binding install, PREDICTED its own downstream
+  symptom in a log line ("a lifted artifact will refuse at
+  `assert_lifted_contract`"), and discarded the cause — so the refusal named
+  the gate that noticed instead of the install that failed. The root now rides
+  the refusal detail. This is the bucket-bearing path a `w8a8-lora64` family
+  takes, which is the lane attempt 26 was minting.
+
+### Changed
+
+- The micro-mint rig's adopt leg arms through `provision.arm_aot` — the gate
+  the DELEGATED mint actually uses — instead of `aot_serve.enable`. The two are
+  different: `arm_aot` adds the mode route, the lifted-LoRA install for a
+  bucket-bearing cfg, and the numerics gate. Arming via `enable` left all three
+  uncovered, so `task rig:micro` could be green while the path that refused
+  sdxl's cell had never run locally at all. The cycle stays green (26.0 s,
+  parity 7.15e-07) and now reports the arm's classified reason.

scripts/micro_mint_rig.py

@@ -444,6 +444,8 @@ def run_cycle(
             + (f" parity max|delta|="
                f"{max(parity.values()):.2e} over {len(parity)} arms"
                if parity else "")
+            + (f" arm={adopted['arm_reason']}"
+               if adopted.get("arm_reason") else "")
             if leg.ok else
             (str(adopted.get("error") or adopted.get("miss_log") or "")[-400:]))
     finally:

src/gen_worker/executor.py

@@ -10007,6 +10007,9 @@ async def _delegated_mint_run(
 
         finalized: Dict[int, Any] = {}
         declined: Optional[_MintDeclined] = None
+        # pgw#999: every classified refusal this run saw, so the terminal
+        # RuntimeError names them instead of restating "no advertisable cell".
+        declined_reasons: List[str] = []
         for pids in sharers.values():
             pending = bg.pendings[pids[0]]
             pipe = bg.pipes[pids[0]]
@@ -10042,14 +10045,19 @@ async def _delegated_mint_run(
                 # wire trace whenever a SIBLING pending succeeded (the
                 # `if not finalized: raise` below never fires then).
                 fleet_cells_mod.abandon_self_mint(pending)
+                # pgw#999: `phase` carries the CLASSIFIED reason when the
+                # child's cell was built and then refused arming; it falls
+                # back to the call-site token only when there is genuinely no
+                # classification (no cell was produced at all).
                 activity_mod.emit_event(
                     "self_mint_abort",
                     f"family={pending.family} key={pending.cell_key}: the "
                     f"delegated child produced no adoptable cell "
                     f"({result.detail or result.status}); this object stays "
                     f"eager and nothing is published",
-                    phase="delegated_no_cell",
+                    phase=result.reason or "delegated_no_cell",
                 )
+                declined_reasons.append(result.reason or result.status)
                 continue
             for pid in pids:
                 finalized[pid] = minted
@@ -10060,7 +10068,9 @@ async def _delegated_mint_run(
                 raise declined
             raise RuntimeError(
                 "delegated mint produced no advertisable cell; serving stays "
-                "eager")
+                "eager"
+                + (f" (refused: {', '.join(sorted(set(declined_reasons)))})"
+                   if declined_reasons else ""))
 
         # Publish per shared cell on gw#612's rule: a family cell ships only
         # when EVERY sharer is covered by it — a partial pack bricks every

src/gen_worker/fleet_cells.py

@@ -1557,6 +1557,11 @@ def adopt_delegated_mint(
             os.replace(artifact, pending.target)
         except OSError:
             shutil.copy2(artifact, pending.target)
+    # pgw#999: (reason, detail) of the arm refusal, set by whichever branch
+    # refuses. Initialized to a NAMED unset rather than "" so a branch that
+    # forgets to classify is visible on the wire as a gap in this function
+    # instead of as an empty string that reads like "no reason exists".
+    refusal: Tuple[str, str] = ("unclassified_arm_refusal", "")
     try:
         if pending.recipe == RECIPE_AOT:
             # pgw#805: an exported cell arms through the AOT gates
@@ -1565,12 +1570,29 @@ def adopt_delegated_mint(
             # passes; `provision.enable_compiled` itself is not reusable here
             # because its pgw#709 receipts gate would drop a cell this pod
             # minted seconds ago and the hub has not countersigned yet.
-            armed = bool(provision.arm_aot(
+            # pgw#999: the outcome is KEPT. `arm_aot` returns a classified
+            # `AdoptOutcome` (`contract_invalid`, `constants_unbound`,
+            # `no_arm_for_mode`, `numerics_refused`, …) and this call site
+            # used to spend it on `bool(...)`. That discard cost attempt 26
+            # 2 h 45 m and $2.72: a 36/36 mint sealed, finalized, and was then
+            # refused by three events that all said "could not adopt".
+            outcome = provision.arm_aot(
                 pipe, pending.cfg, pending.cache_dir, pending.target,
-                int(getattr(pending.cfg, "lora_bucket", 0) or 0)))
+                int(getattr(pending.cfg, "lora_bucket", 0) or 0))
+            armed = bool(outcome)
+            if not armed:
+                refusal = (outcome.reason or "unclassified_arm_refusal",
+                           outcome.detail or outcome.identity)
         else:
             armed = bool(cc.enable(pipe, pending.cfg, pending.cache_dir,
                                    artifact=pending.target))
+            if not armed:
+                # `cc.enable` returns a bare bool and RAISES for the refusals
+                # it can name, so a falsy return genuinely carries no reason.
+                # Saying so by name beats inventing one.
+                refusal = ("jit_enable_declined",
+                           "compile_cache.enable declined without raising; it "
+                           "reports no classified reason on this path")
     except cc.CellSelectionBugError as exc:
         # th#883, delegated edition: the child's own cell, whose axes describe
         # exactly this runtime, refused to arm. Loud — it is a bug in the one
@@ -1580,18 +1602,30 @@ def adopt_delegated_mint(
             "mint (family=%s key=%s): %s",
             pending.family, pending.cell_key, exc)
         armed = False
+        refusal = ("cell_selection_bug", str(exc))
     except Exception as exc:  # noqa: BLE001 — adoption failure => eager
         logger.warning(
             "fleet-cells: delegated mint for %s did not adopt (%s)",
             pending.family, exc)
         armed = False
+        # An `AdoptError` already carries the token; anything else is named by
+        # its type rather than flattened into one word nobody can count.
+        refusal = (str(getattr(exc, "reason", "") or "") or type(exc).__name__,
+                   str(exc))
     if not armed:
+        reason, detail = refusal
+        # pgw#999: `phase` is the countable column, so it carries the CLASS —
+        # the same convention `self_mint_skipped` already uses. The old
+        # constant `delegated_adopt_failed` said only which call site fired,
+        # which every reader already knew from the event kind.
+        state["adopt_refusal"] = (reason, detail)
         activity_mod.emit_event(
             "self_mint_abort",
             f"family={pending.family} key={pending.cell_key}: the child "
-            "process produced a cell this runtime could not adopt; serving "
-            "stays eager and nothing is published",
-            phase="delegated_adopt_failed",
+            f"process produced a cell this runtime could not adopt "
+            f"({reason}{': ' + detail if detail else ''}); serving stays "
+            f"eager and nothing is published",
+            phase=reason,
         )
         mark_terminus(pending, TERMINUS_ABORTED)
         state["minted"] = None
@@ -1622,6 +1656,19 @@ def adopt_delegated_mint(
     return minted
 
 
+def adopt_refusal(pending: "PendingSelfMint") -> Tuple[str, str]:
+    """Why :func:`adopt_delegated_mint` refused this pending's cell (pgw#999).
+
+    ``("", "")`` when it did not refuse — the mint adopted, or never got as
+    far as arming. The classification lives on the pending's own state rather
+    than being re-derived by the caller, so the abort event, the delegated
+    result and the executor's decline all quote ONE string that was produced
+    at the one place that knows it.
+    """
+    reason, detail = pending._state.get("adopt_refusal") or ("", "")
+    return str(reason), str(detail)
+
+
 def publish_self_mint(pending: "PendingSelfMint") -> None:
     """Ship a FINALIZED mint to the fleet store, once (gw#612 restructure).
 

src/gen_worker/mint_delegate.py

@@ -120,6 +120,10 @@ class DelegatedResult:
     minted: Optional[Any] = None      # fleet_cells.SelfMint
     attempts: int = 0
     budget: Optional[mint_budget.MintBudget] = None
+    #: pgw#999: the CLASSIFIED reason the child's cell did not adopt, carried
+    #: up so the executor's decline names the same token the abort event did.
+    #: Empty for every outcome that is not an adopt refusal.
+    reason: str = ""
 
     @property
     def declined(self) -> bool:
@@ -394,11 +398,20 @@ async def build_cell(
                     status=ADOPTED, minted=minted, attempts=attempts,
                     budget=budget)
             # The child produced bytes this runtime could not adopt.
-            # adopt_delegated_mint already emitted the typed abort and
-            # cleaned up; retrying cannot change a verify()/drift verdict.
+            # `adopt_delegated_mint` emitted the typed abort and cleaned up;
+            # retrying cannot change a verify()/drift verdict.
+            #
+            # pgw#999: it also RECORDED why, and this is where that used to
+            # die. The sentence below was the whole of what the wire got.
+            reason, why = fleet_cells.adopt_refusal(pending)
             return DelegatedResult(
                 status=FAILED, attempts=attempts, budget=budget,
-                detail="the child's cell did not adopt on this runtime")
+                reason=reason,
+                detail=(
+                    f"the child's cell did not adopt on this runtime "
+                    f"({reason}{': ' + why if why else ''})"
+                    if reason else
+                    "the child's cell did not adopt on this runtime"))
 
         _emit_abort(outcome, family, pending.cell_key, attempts)
         if not (outcome.retryable and attempts < max(1, max_attempts)):

src/gen_worker/models/provision.py

@@ -272,6 +272,9 @@ def arm_aot(
             meta = None
     lifted_target: Any = None
     lifted_installed = False
+    #: pgw#999: why the lifted-binding install failed, if it did. Carried into
+    #: the refusal instead of dying in a logger no pod exposes.
+    lifted_install_error = ""
     mode = str((meta or {}).get("mode") or "")
     if arm_route(mode) is None:
         # A cell whose mode this runtime has no arm for must decline BY NAME
@@ -305,11 +308,27 @@ def arm_aot(
                 lora_lifted.install_lifted_lora_forward(lifted_target, bucket)
                 lifted_installed = True
             except Exception as exc:  # noqa: BLE001 — arm decides
+                # pgw#999: KEPT, not merely logged. This branch predicted its
+                # own downstream symptom ("will refuse at
+                # assert_lifted_contract") and then discarded the cause, so
+                # the refusal that follows names the gate that noticed rather
+                # than the install that failed. Same discard as the one this
+                # issue is closing, one frame deeper, on exactly the
+                # bucket-bearing path a w8a8-lora64 family takes.
+                lifted_install_error = f"{type(exc).__name__}: {exc}"
                 logger.warning(
                     "aot arm: lifted-binding install failed on %r (%s); a "
                     "lifted artifact will refuse at assert_lifted_contract",
                     module_name, exc)
     outcome = aot_serve.enable(pipe, cfg, cache_dir, artifact)
+    if not outcome.armed and lifted_install_error:
+        # The refusal is real; its ROOT is one frame up. Both, in the order a
+        # reader needs them: what refused, and what made it refuse.
+        outcome = AdoptOutcome.miss(
+            outcome.reason or "lifted_install_failed",
+            f"{outcome.detail} [root: lifted-binding install failed on "
+            f"{module_name!r} — {lifted_install_error}]".strip(),
+            outcome.identity)
     if outcome.armed:
         if gate_cell_numerics(pipe, cfg):
             return outcome

tests/harness/rig_vehicles.py

@@ -173,6 +173,7 @@ def _micro_cell() -> Any:
 import torch
 from pathlib import Path
 from gen_worker import aot_cells, aot_serve
+from gen_worker.models import provision
 from gen_worker.registry import CompileCell
 from micro_diffusion.aot_declaration import (
     CFG_ARITY, COND_LEN, PIXEL_ROWS, TOKEN_ROWS)
@@ -233,10 +234,20 @@ def _feed(arity, tokens):
     # PARITY. Adoption that is never CALLED proves the filter, not the cell —
     # and the serve-side call is exactly where pgw#994 lives: a container
     # input expands to N leaves and every contract position after it shifts.
-    outcome = aot_serve.enable(pipe, cfg, Path(%(cache)r), Path(cell.artifact))
+    #
+    # pgw#999: through `provision.arm_aot`, NOT `aot_serve.enable`. They are
+    # different gates and the delegated mint uses this one: it adds the mode
+    # route, the lifted-LoRA install for a bucket-bearing cfg, and the
+    # numerics gate. Arming via `enable` left every one of those uncovered —
+    # which is why a cycle could be green while the path that refused sdxl's
+    # 36/36 cell had never run locally at all.
+    outcome = provision.arm_aot(
+        pipe, cfg, Path(%(cache)r), Path(cell.artifact),
+        int(getattr(cfg, "lora_bucket", 0) or 0))
     out["armed"] = bool(outcome)
+    out["arm_reason"] = str(getattr(outcome, "reason", "") or "")
     out["arm_detail"] = str(getattr(outcome, "detail", "") or
-                            getattr(outcome, "reason", ""))[:400]
+                            getattr(outcome, "identity", ""))[:400]
     if outcome:
         deltas = {}
         with torch.no_grad():