diff --git a/changelog.d/pgw999.md b/changelog.d/pgw999.md new file mode 100644 index 00000000..1aa4418f --- /dev/null +++ b/changelog.d/pgw999.md @@ -0,0 +1,34 @@ +### Fixed + +- **A refused delegated cell names WHY (pgw#999).** `adopt_delegated_mint` + spent a classified `AdoptOutcome` on `bool(...)`, so a mint that sealed and + finalized 36/36 entries was refused by three events that all said "a cell + this runtime could not adopt". Attempt 26 paid 2 h 45 m and $2.72 of L40S for + that sentence. The classification now reaches the wire: `phase` carries the + CLASS (`contract_invalid`, `constants_constant_unresolved`, + `no_arm_for_mode`, `numerics_refused`, `cell_selection_bug`, …) the way + `self_mint_skipped` already does, the detail quotes the gate's own sentence, + and `DelegatedResult.reason` carries it up so the executor's decline and the + terminal error quote the same token instead of naming their own call sites. +- The `except Exception` branch that logged an adoption failure to a logger no + pod exposes now emits it — `AdoptError.reason` when the exception carries + one, the exception TYPE when it does not. Log-only swallowing of an important + error is a defect class here, not a style preference. + +- **The same discard, one frame deeper.** `provision.arm_aot`'s bucket branch + caught a failed lifted-LoRA binding install, PREDICTED its own downstream + symptom in a log line ("a lifted artifact will refuse at + `assert_lifted_contract`"), and discarded the cause — so the refusal named + the gate that noticed instead of the install that failed. The root now rides + the refusal detail. This is the bucket-bearing path a `w8a8-lora64` family + takes, which is the lane attempt 26 was minting. + +### Changed + +- The micro-mint rig's adopt leg arms through `provision.arm_aot` — the gate + the DELEGATED mint actually uses — instead of `aot_serve.enable`. The two are + different: `arm_aot` adds the mode route, the lifted-LoRA install for a + bucket-bearing cfg, and the numerics gate. Arming via `enable` left all three + uncovered, so `task rig:micro` could be green while the path that refused + sdxl's cell had never run locally at all. The cycle stays green (26.0 s, + parity 7.15e-07) and now reports the arm's classified reason. diff --git a/scripts/micro_mint_rig.py b/scripts/micro_mint_rig.py index 594d21fb..3bd0a887 100755 --- a/scripts/micro_mint_rig.py +++ b/scripts/micro_mint_rig.py @@ -444,6 +444,8 @@ def run_cycle( + (f" parity max|delta|=" f"{max(parity.values()):.2e} over {len(parity)} arms" if parity else "") + + (f" arm={adopted['arm_reason']}" + if adopted.get("arm_reason") else "") if leg.ok else (str(adopted.get("error") or adopted.get("miss_log") or "")[-400:])) finally: diff --git a/src/gen_worker/executor.py b/src/gen_worker/executor.py index f71c3e35..a039e3aa 100644 --- a/src/gen_worker/executor.py +++ b/src/gen_worker/executor.py @@ -10007,6 +10007,9 @@ async def _delegated_mint_run( finalized: Dict[int, Any] = {} declined: Optional[_MintDeclined] = None + # pgw#999: every classified refusal this run saw, so the terminal + # RuntimeError names them instead of restating "no advertisable cell". + declined_reasons: List[str] = [] for pids in sharers.values(): pending = bg.pendings[pids[0]] pipe = bg.pipes[pids[0]] @@ -10042,14 +10045,19 @@ async def _delegated_mint_run( # wire trace whenever a SIBLING pending succeeded (the # `if not finalized: raise` below never fires then). fleet_cells_mod.abandon_self_mint(pending) + # pgw#999: `phase` carries the CLASSIFIED reason when the + # child's cell was built and then refused arming; it falls + # back to the call-site token only when there is genuinely no + # classification (no cell was produced at all). activity_mod.emit_event( "self_mint_abort", f"family={pending.family} key={pending.cell_key}: the " f"delegated child produced no adoptable cell " f"({result.detail or result.status}); this object stays " f"eager and nothing is published", - phase="delegated_no_cell", + phase=result.reason or "delegated_no_cell", ) + declined_reasons.append(result.reason or result.status) continue for pid in pids: finalized[pid] = minted @@ -10060,7 +10068,9 @@ async def _delegated_mint_run( raise declined raise RuntimeError( "delegated mint produced no advertisable cell; serving stays " - "eager") + "eager" + + (f" (refused: {', '.join(sorted(set(declined_reasons)))})" + if declined_reasons else "")) # Publish per shared cell on gw#612's rule: a family cell ships only # when EVERY sharer is covered by it — a partial pack bricks every diff --git a/src/gen_worker/fleet_cells.py b/src/gen_worker/fleet_cells.py index 678d2eca..9d89ac50 100644 --- a/src/gen_worker/fleet_cells.py +++ b/src/gen_worker/fleet_cells.py @@ -1557,6 +1557,11 @@ def adopt_delegated_mint( os.replace(artifact, pending.target) except OSError: shutil.copy2(artifact, pending.target) + # pgw#999: (reason, detail) of the arm refusal, set by whichever branch + # refuses. Initialized to a NAMED unset rather than "" so a branch that + # forgets to classify is visible on the wire as a gap in this function + # instead of as an empty string that reads like "no reason exists". + refusal: Tuple[str, str] = ("unclassified_arm_refusal", "") try: if pending.recipe == RECIPE_AOT: # pgw#805: an exported cell arms through the AOT gates @@ -1565,12 +1570,29 @@ def adopt_delegated_mint( # passes; `provision.enable_compiled` itself is not reusable here # because its pgw#709 receipts gate would drop a cell this pod # minted seconds ago and the hub has not countersigned yet. - armed = bool(provision.arm_aot( + # pgw#999: the outcome is KEPT. `arm_aot` returns a classified + # `AdoptOutcome` (`contract_invalid`, `constants_unbound`, + # `no_arm_for_mode`, `numerics_refused`, …) and this call site + # used to spend it on `bool(...)`. That discard cost attempt 26 + # 2 h 45 m and $2.72: a 36/36 mint sealed, finalized, and was then + # refused by three events that all said "could not adopt". + outcome = provision.arm_aot( pipe, pending.cfg, pending.cache_dir, pending.target, - int(getattr(pending.cfg, "lora_bucket", 0) or 0))) + int(getattr(pending.cfg, "lora_bucket", 0) or 0)) + armed = bool(outcome) + if not armed: + refusal = (outcome.reason or "unclassified_arm_refusal", + outcome.detail or outcome.identity) else: armed = bool(cc.enable(pipe, pending.cfg, pending.cache_dir, artifact=pending.target)) + if not armed: + # `cc.enable` returns a bare bool and RAISES for the refusals + # it can name, so a falsy return genuinely carries no reason. + # Saying so by name beats inventing one. + refusal = ("jit_enable_declined", + "compile_cache.enable declined without raising; it " + "reports no classified reason on this path") except cc.CellSelectionBugError as exc: # th#883, delegated edition: the child's own cell, whose axes describe # exactly this runtime, refused to arm. Loud — it is a bug in the one @@ -1580,18 +1602,30 @@ def adopt_delegated_mint( "mint (family=%s key=%s): %s", pending.family, pending.cell_key, exc) armed = False + refusal = ("cell_selection_bug", str(exc)) except Exception as exc: # noqa: BLE001 — adoption failure => eager logger.warning( "fleet-cells: delegated mint for %s did not adopt (%s)", pending.family, exc) armed = False + # An `AdoptError` already carries the token; anything else is named by + # its type rather than flattened into one word nobody can count. + refusal = (str(getattr(exc, "reason", "") or "") or type(exc).__name__, + str(exc)) if not armed: + reason, detail = refusal + # pgw#999: `phase` is the countable column, so it carries the CLASS — + # the same convention `self_mint_skipped` already uses. The old + # constant `delegated_adopt_failed` said only which call site fired, + # which every reader already knew from the event kind. + state["adopt_refusal"] = (reason, detail) activity_mod.emit_event( "self_mint_abort", f"family={pending.family} key={pending.cell_key}: the child " - "process produced a cell this runtime could not adopt; serving " - "stays eager and nothing is published", - phase="delegated_adopt_failed", + f"process produced a cell this runtime could not adopt " + f"({reason}{': ' + detail if detail else ''}); serving stays " + f"eager and nothing is published", + phase=reason, ) mark_terminus(pending, TERMINUS_ABORTED) state["minted"] = None @@ -1622,6 +1656,19 @@ def adopt_delegated_mint( return minted +def adopt_refusal(pending: "PendingSelfMint") -> Tuple[str, str]: + """Why :func:`adopt_delegated_mint` refused this pending's cell (pgw#999). + + ``("", "")`` when it did not refuse — the mint adopted, or never got as + far as arming. The classification lives on the pending's own state rather + than being re-derived by the caller, so the abort event, the delegated + result and the executor's decline all quote ONE string that was produced + at the one place that knows it. + """ + reason, detail = pending._state.get("adopt_refusal") or ("", "") + return str(reason), str(detail) + + def publish_self_mint(pending: "PendingSelfMint") -> None: """Ship a FINALIZED mint to the fleet store, once (gw#612 restructure). diff --git a/src/gen_worker/mint_delegate.py b/src/gen_worker/mint_delegate.py index 5d1656a0..af3a7522 100644 --- a/src/gen_worker/mint_delegate.py +++ b/src/gen_worker/mint_delegate.py @@ -120,6 +120,10 @@ class DelegatedResult: minted: Optional[Any] = None # fleet_cells.SelfMint attempts: int = 0 budget: Optional[mint_budget.MintBudget] = None + #: pgw#999: the CLASSIFIED reason the child's cell did not adopt, carried + #: up so the executor's decline names the same token the abort event did. + #: Empty for every outcome that is not an adopt refusal. + reason: str = "" @property def declined(self) -> bool: @@ -394,11 +398,20 @@ async def build_cell( status=ADOPTED, minted=minted, attempts=attempts, budget=budget) # The child produced bytes this runtime could not adopt. - # adopt_delegated_mint already emitted the typed abort and - # cleaned up; retrying cannot change a verify()/drift verdict. + # `adopt_delegated_mint` emitted the typed abort and cleaned up; + # retrying cannot change a verify()/drift verdict. + # + # pgw#999: it also RECORDED why, and this is where that used to + # die. The sentence below was the whole of what the wire got. + reason, why = fleet_cells.adopt_refusal(pending) return DelegatedResult( status=FAILED, attempts=attempts, budget=budget, - detail="the child's cell did not adopt on this runtime") + reason=reason, + detail=( + f"the child's cell did not adopt on this runtime " + f"({reason}{': ' + why if why else ''})" + if reason else + "the child's cell did not adopt on this runtime")) _emit_abort(outcome, family, pending.cell_key, attempts) if not (outcome.retryable and attempts < max(1, max_attempts)): diff --git a/src/gen_worker/models/provision.py b/src/gen_worker/models/provision.py index bbc4622f..46cf16ba 100644 --- a/src/gen_worker/models/provision.py +++ b/src/gen_worker/models/provision.py @@ -272,6 +272,9 @@ def arm_aot( meta = None lifted_target: Any = None lifted_installed = False + #: pgw#999: why the lifted-binding install failed, if it did. Carried into + #: the refusal instead of dying in a logger no pod exposes. + lifted_install_error = "" mode = str((meta or {}).get("mode") or "") if arm_route(mode) is None: # A cell whose mode this runtime has no arm for must decline BY NAME @@ -305,11 +308,27 @@ def arm_aot( lora_lifted.install_lifted_lora_forward(lifted_target, bucket) lifted_installed = True except Exception as exc: # noqa: BLE001 — arm decides + # pgw#999: KEPT, not merely logged. This branch predicted its + # own downstream symptom ("will refuse at + # assert_lifted_contract") and then discarded the cause, so + # the refusal that follows names the gate that noticed rather + # than the install that failed. Same discard as the one this + # issue is closing, one frame deeper, on exactly the + # bucket-bearing path a w8a8-lora64 family takes. + lifted_install_error = f"{type(exc).__name__}: {exc}" logger.warning( "aot arm: lifted-binding install failed on %r (%s); a " "lifted artifact will refuse at assert_lifted_contract", module_name, exc) outcome = aot_serve.enable(pipe, cfg, cache_dir, artifact) + if not outcome.armed and lifted_install_error: + # The refusal is real; its ROOT is one frame up. Both, in the order a + # reader needs them: what refused, and what made it refuse. + outcome = AdoptOutcome.miss( + outcome.reason or "lifted_install_failed", + f"{outcome.detail} [root: lifted-binding install failed on " + f"{module_name!r} — {lifted_install_error}]".strip(), + outcome.identity) if outcome.armed: if gate_cell_numerics(pipe, cfg): return outcome diff --git a/tests/harness/rig_vehicles.py b/tests/harness/rig_vehicles.py index 034d8db8..36f584ad 100644 --- a/tests/harness/rig_vehicles.py +++ b/tests/harness/rig_vehicles.py @@ -173,6 +173,7 @@ def _micro_cell() -> Any: import torch from pathlib import Path from gen_worker import aot_cells, aot_serve +from gen_worker.models import provision from gen_worker.registry import CompileCell from micro_diffusion.aot_declaration import ( CFG_ARITY, COND_LEN, PIXEL_ROWS, TOKEN_ROWS) @@ -233,10 +234,20 @@ def _feed(arity, tokens): # PARITY. Adoption that is never CALLED proves the filter, not the cell — # and the serve-side call is exactly where pgw#994 lives: a container # input expands to N leaves and every contract position after it shifts. - outcome = aot_serve.enable(pipe, cfg, Path(%(cache)r), Path(cell.artifact)) + # + # pgw#999: through `provision.arm_aot`, NOT `aot_serve.enable`. They are + # different gates and the delegated mint uses this one: it adds the mode + # route, the lifted-LoRA install for a bucket-bearing cfg, and the + # numerics gate. Arming via `enable` left every one of those uncovered — + # which is why a cycle could be green while the path that refused sdxl's + # 36/36 cell had never run locally at all. + outcome = provision.arm_aot( + pipe, cfg, Path(%(cache)r), Path(cell.artifact), + int(getattr(cfg, "lora_bucket", 0) or 0)) out["armed"] = bool(outcome) + out["arm_reason"] = str(getattr(outcome, "reason", "") or "") out["arm_detail"] = str(getattr(outcome, "detail", "") or - getattr(outcome, "reason", ""))[:400] + getattr(outcome, "identity", ""))[:400] if outcome: deltas = {} with torch.no_grad(): diff --git a/tests/test_delegated_adopt_reason_pgw999.py b/tests/test_delegated_adopt_reason_pgw999.py new file mode 100644 index 00000000..8960abcf --- /dev/null +++ b/tests/test_delegated_adopt_reason_pgw999.py @@ -0,0 +1,329 @@ +"""pgw#999 — a refused delegated cell names WHY, on the wire. + +RED AT HEAD, all of it. `adopt_delegated_mint` spent a classified +``AdoptOutcome`` on ``bool(...)``, so three abort events restated one fact +three ways and none carried a cause: + + delegated_adopt_failed : "the child process produced a cell this runtime + could not adopt" + delegated_no_cell : "...produced no adoptable cell (the child's cell + did not adopt on this runtime)" + error : "delegated mint produced no advertisable cell" + +Attempt 26 paid 2 h 45 m and $2.72 of L40S to learn "something". The +classified string existed in-process — ``contract_invalid``, +``constants_unbound``, ``no_arm_for_mode``, ``numerics_refused`` — and was +discarded one frame before the wire. This is the ``worker-errors-to- +orchestrator`` defect class verbatim, so these tests assert the CLASS reaches +the countable field, not merely that some prose got longer. + +Every test states what it would have said at HEAD. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, List, Tuple + +import pytest + +from gen_worker import fleet_cells, mint_delegate +from gen_worker.cell_adopt import AdoptOutcome +from gen_worker.compile_cache import AdoptError, CellSelectionBugError + +FAMILY = "pgw999" + + +@dataclass +class _Cfg: + family: str = FAMILY + lora_bucket: int = 64 + shapes: Tuple[Tuple[int, int], ...] = ((1024, 1024),) + targets: Tuple[str, ...] = ("unet",) + text_lens: Tuple[int, ...] = (77,) + guidance_scales: Tuple[float, ...] = (1.0, 5.0) + regional: bool = False + + +class _Pipe: + pass + + +@pytest.fixture() +def events(monkeypatch: pytest.MonkeyPatch) -> List[Tuple[str, str, str]]: + seen: List[Tuple[str, str, str]] = [] + + def _sink(kind: str, detail: str, phase: str = "", duration_ms: int = 0) -> None: + seen.append((kind, phase, detail)) + + monkeypatch.setattr(fleet_cells.activity_mod, "emit_event", _sink) + monkeypatch.setattr(mint_delegate.activity_mod, "emit_event", _sink) + return seen + + +@pytest.fixture() +def pending(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Any: + """A pending whose cell EXISTS — the pgw#999 shape exactly. + + The mint succeeded (36/36 sealed and finalized on the pod that found + this); the artifact is real bytes on disk and the only open question is + whether the runtime that built it will arm it. + """ + monkeypatch.setattr(fleet_cells, "_unregister", lambda p: None) + monkeypatch.setattr(fleet_cells, "mark_terminus", lambda p, t: None) + artifact = tmp_path / "cell.tar.gz" + artifact.write_bytes(b"a sealed, finalized cell") + return fleet_cells.PendingSelfMint( + family=FAMILY, cell_key="ck1-sealed", ref=f"root/family-{FAMILY}#ck1-sealed", + cfg=_Cfg(), target=artifact, capture_dir=tmp_path / "cap", + mint_root=tmp_path / "root", publisher=None, delegated=True, + recipe=fleet_cells.RECIPE_AOT) + + +def _abort(events: List[Tuple[str, str, str]]) -> Tuple[str, str]: + rows = [(phase, detail) for kind, phase, detail in events + if kind == "self_mint_abort"] + assert len(rows) == 1, f"expected exactly one abort event, got {rows!r}" + return rows[0] + + +def _arm_returns(monkeypatch: pytest.MonkeyPatch, outcome: AdoptOutcome) -> None: + monkeypatch.setattr( + fleet_cells.provision, "arm_aot", lambda *a, **k: outcome) + + +def _arm_raises(monkeypatch: pytest.MonkeyPatch, exc: BaseException) -> None: + def _boom(*a: Any, **k: Any) -> Any: + raise exc + + monkeypatch.setattr(fleet_cells.provision, "arm_aot", _boom) + + +# --------------------------------------------------------------------------- +# 1. The returned classification — the exact attempt-26 path +# --------------------------------------------------------------------------- + + +def test_a_returned_refusal_puts_its_class_in_the_countable_field( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """HEAD said phase='delegated_adopt_failed' — the call site's name, which + every reader already knew from the event KIND. `phase` is the countable + column, so it has to carry the CLASS, the way `self_mint_skipped` already + does.""" + _arm_returns(monkeypatch, AdoptOutcome.miss( + "contract_invalid", + "input_contract records 5 leaves, the traced call takes 3")) + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is None + + phase, detail = _abort(events) + assert phase == "contract_invalid" + assert "input_contract records 5 leaves" in detail + assert "could not adopt" in detail, "the human sentence stays, too" + + +def test_every_classified_reason_survives_verbatim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """The four the issue names, plus the numerics gate. A reason that is + *transformed* on the way to the wire is a reason nobody can group by.""" + monkeypatch.setattr(fleet_cells, "_unregister", lambda p: None) + monkeypatch.setattr(fleet_cells, "mark_terminus", lambda p, t: None) + for i, reason in enumerate(( + "contract_invalid", "constants_unbound", "no_arm_for_mode", + "lane_unavailable", "numerics_refused", "sm_mismatch", + )): + seen: List[Tuple[str, str, str]] = [] + monkeypatch.setattr( + fleet_cells.activity_mod, "emit_event", + lambda kind, detail, phase="", duration_ms=0: seen.append( + (kind, phase, detail))) + _arm_returns(monkeypatch, AdoptOutcome.miss(reason, f"detail for {reason}")) + artifact = tmp_path / f"cell-{i}.tar.gz" + artifact.write_bytes(b"cell") + p = fleet_cells.PendingSelfMint( + family=FAMILY, cell_key=f"ck1-{i}", ref=f"root/family-{FAMILY}#ck1-{i}", + cfg=_Cfg(), target=artifact, capture_dir=tmp_path / f"cap{i}", + mint_root=tmp_path / f"root{i}", publisher=None, delegated=True, + recipe=fleet_cells.RECIPE_AOT) + assert fleet_cells.adopt_delegated_mint(_Pipe(), p, artifact) is None + phase, _detail = _abort(seen) + assert phase == reason + assert fleet_cells.adopt_refusal(p) == (reason, f"detail for {reason}") + + +# --------------------------------------------------------------------------- +# 2. The RAISED classifications — the branch that was a logger.warning +# --------------------------------------------------------------------------- + + +def test_a_raised_AdoptError_is_classified_by_its_own_token( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`AdoptError` has carried `.reason` since it was written. HEAD's + `except Exception` branch logged it to a logger no pod exposes and set + `armed = False` — log-only swallowing, which is a defect class here.""" + _arm_raises(monkeypatch, AdoptError( + "constants_unbound", "7 constants have no resident weight")) + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is None + + phase, detail = _abort(events) + assert phase == "constants_unbound" + assert "7 constants have no resident weight" in detail + + +def test_an_unclassified_exception_is_named_by_its_TYPE_not_flattened( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An exception with no `.reason` still has a name. Collapsing it to one + generic token would rebuild the very hole this issue is closing.""" + _arm_raises(monkeypatch, ValueError("shapes disagree")) + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is None + + phase, detail = _abort(events) + assert phase == "ValueError" + assert "shapes disagree" in detail + + +def test_the_cell_selection_bug_keeps_its_own_loud_class( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """th#883's invariant must not be flattened into the generic refusal set: + a self-requested, identity-verified cell that will not arm is a BUG in + the selection brain, not a compatibility miss.""" + _arm_raises(monkeypatch, CellSelectionBugError("axes describe this runtime")) + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is None + + phase, detail = _abort(events) + assert phase == "cell_selection_bug" + assert "axes describe this runtime" in detail + + +def test_a_silent_falsy_arm_says_SO_rather_than_inventing_a_reason( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`AdoptOutcome.miss("")` is a refusal that classified nothing. The event + must say the classification is MISSING — a blank phase would read as "no + reason exists", which is the lie this whole issue is about.""" + _arm_returns(monkeypatch, AdoptOutcome(armed=False)) + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is None + + phase, _detail = _abort(events) + assert phase == "unclassified_arm_refusal" + + +# --------------------------------------------------------------------------- +# 3. The reason CROSSES the boundaries — one string, three events +# --------------------------------------------------------------------------- + + +def test_the_reason_is_readable_by_the_caller_that_must_requote_it( + pending: Any, events: List[Tuple[str, str, str]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`mint_delegate.build_cell` and the executor each emit their own event + about the same refusal. They read the classification from the one place + that produced it instead of re-deriving three vocabularies.""" + _arm_returns(monkeypatch, AdoptOutcome.miss("no_arm_for_mode", "mode='regional'")) + fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) + + assert fleet_cells.adopt_refusal(pending) == ("no_arm_for_mode", "mode='regional'") + + +def test_a_pending_that_never_refused_reports_no_reason( + pending: Any, monkeypatch: pytest.MonkeyPatch, +) -> None: + """The accessor must not manufacture a refusal for a mint that adopted — + an always-non-empty reason is as useless as an always-empty one.""" + _arm_returns(monkeypatch, AdoptOutcome.hit("family=x key=y")) + monkeypatch.setattr( + fleet_cells, "_packed_metadata", lambda a: {"cell_key": "ck1-sealed"}) + monkeypatch.setattr(fleet_cells, "sha256_file", lambda p: "beef") + + assert fleet_cells.adopt_delegated_mint(_Pipe(), pending, pending.target) is not None + assert fleet_cells.adopt_refusal(pending) == ("", "") + + +def test_the_delegated_result_carries_the_reason_field(monkeypatch: pytest.MonkeyPatch) -> None: + """The transport between the two events. RED at HEAD: `DelegatedResult` + had no `reason` at all, so the executor had nothing to quote and fell back + to naming its own call site.""" + result = mint_delegate.DelegatedResult( + status=mint_delegate.FAILED, reason="contract_invalid", + detail="the child's cell did not adopt on this runtime " + "(contract_invalid: 5 leaves vs 3)") + assert result.reason == "contract_invalid" + assert not result.ok + assert "contract_invalid" in result.detail + + +# --------------------------------------------------------------------------- +# 4. The SAME discard, one frame deeper — `arm_aot`'s lifted-binding install +# --------------------------------------------------------------------------- + + +def test_a_failed_lifted_install_reaches_the_refusal_it_causes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """RED at HEAD. `arm_aot`'s bucket branch caught the install failure, + PREDICTED its own downstream symptom in a log line ("a lifted artifact + will refuse at assert_lifted_contract"), and discarded the cause — so the + refusal named the gate that noticed instead of the install that failed. + + This is the bucket-bearing path a `w8a8-lora64` family takes, which is + exactly the lane attempt 26 was minting. + """ + from gen_worker.models import provision + + class _Target: + pass + + class _PipeWithUnet: + def __init__(self) -> None: + self.unet = _Target() + + # `arm_aot` imports these INSIDE the function body (they drag 39 modules + # onto the `import gen_worker` path), so they are patched on their own + # modules rather than as attributes of `provision`. + from gen_worker import aot_serve, trt_engine + + monkeypatch.setattr( + trt_engine, "unpack_metadata", + lambda p: {"targets": ["unet"], "module": "unet", "mode": ""}) + monkeypatch.setattr(provision, "arm_route", lambda mode: object()) + + from gen_worker.models import lora_lifted + + monkeypatch.setattr(lora_lifted, "lifted_binding", lambda m: None) + + def _boom(target: Any, bucket: int) -> None: + raise RuntimeError("branch containers not allocated for bucket 64") + + monkeypatch.setattr(lora_lifted, "install_lifted_lora_forward", _boom) + monkeypatch.setattr( + aot_serve, "enable", + lambda *a, **k: AdoptOutcome.miss( + "lifted_inputs_unbindable", "module exposes no lifted binding")) + + artifact = tmp_path / "cell.tar.gz" + artifact.write_bytes(b"cell") + outcome = provision.arm_aot( + _PipeWithUnet(), _Cfg(), None, artifact, 64) + + assert not outcome.armed + # The gate that refused is still named... + assert outcome.reason == "lifted_inputs_unbindable" + # ...and so is the ROOT, which is what HEAD threw away. + assert "branch containers not allocated for bucket 64" in outcome.detail + assert "root:" in outcome.detail