From de01e64e16fcc7dfc95df621c62bcc9efecf9874 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 21:48:25 +0900 Subject: [PATCH 1/9] feat(lib): one gate and one grant for an ambiguous resend far the caller observed the exchange and why it failed. For a stage whose commitment is nothing-observed and a cause whose evidence is unknown it answers refused-ambiguous, and it names the only thing that may override that answer: a narrowly scoped recovery a maintainer opted into and bounded. Two separate overrides is one too many. A request that resets before the response head and again after it would buy a replacement send on each side, and the second one is exactly the duplicated inference the refusal exists to prevent. request-resend-gate.ts is the single place the override is applied. It derives stage, cause, permission and send class from request-failure-model.ts and adds nothing of its own except the grant, which it claims at the moment it authorises rather than earlier -- so a caller cannot ask without paying, and a committed or futile failure refuses without draining the replacement a later ambiguous reset would have been entitled to. The cause can be asked in terms of the AttemptRecoveryKind the send will be recorded as, which is what keeps the reason in the log and the reason the gate weighed from being two different values. The grant itself lives on the request's execution budget, beside the physical-send ledger, because it has to be shared in exactly the same places: a combo child derives its own budget from the parent's ledger, and two counters would let one logical request replace an unknown-state send twice. It is not a send budget -- an authorised replacement still has to fit inside remainingBaseSends like everything else. Registers the three test files this branch adds in both the layout map and the independent expectation fixture. --- scripts/test-layout/layout.json | 5 +- src/lib/request-execution-budget.ts | 31 ++++ src/lib/request-resend-gate.ts | 138 ++++++++++++++++ src/server/responses/request-send-budget.ts | 12 ++ structure/overview.md | 7 + tests/fixtures/test-layout-expected.json | 5 +- tests/lib/execution-budget-permits.test.ts | 35 ++++ tests/lib/request-resend-gate.test.ts | 172 ++++++++++++++++++++ 8 files changed, 403 insertions(+), 2 deletions(-) create mode 100644 src/lib/request-resend-gate.ts create mode 100644 tests/lib/request-resend-gate.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index a8d43bc015c..acbb0ee4381 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1610,7 +1610,10 @@ "claude-intercept-settings.test.ts": "claude-integration", "claude-desktop-first-party.test.ts": "claude-integration", "claude-desktop-mode-explanation.test.ts": "claude-integration", - "claude-intercept-integration.test.ts": "server" + "claude-intercept-integration.test.ts": "server", + "request-resend-gate.test.ts": "lib", + "management-provider-reset-replay.test.ts": "server", + "responses-reset-replay.test.ts": "responses" }, "migrated": [ "adapters", diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index f17e44350fe..f6cd5042366 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -165,6 +165,21 @@ export interface RequestExecutionBudget extends TransientSendBudget { readonly alternateTargetSends: number; readonly targetTransitions: number; readonly lastTargetKey: string | undefined; + /** + * Spend one operator-granted replacement for an AMBIGUOUS failure of this logical request, + * up to `limit`. False once the request has none left. + * + * It lives on the budget rather than beside the policy that grants it because it has to be + * shared exactly where the physical-send ledger is shared. A combo child derives its own + * budget from the parent's ledger, and two counters would let a request whose parent leg + * reset before the head and whose child leg reset after it replace an unknown-state send + * twice. It is NOT a send budget: an authorised replacement still has to fit inside + * `remainingBaseSends` like every other send. + * + * Optional so a hand-written stub that satisfies the shape test keeps typechecking; a caller + * that cannot reach it has no operator override, which is the fail-closed answer. + */ + claimAmbiguousResend?(limit: number): boolean; } const RESERVE_FUNDED_CLASSES: ReadonlySet = new Set([ @@ -192,6 +207,12 @@ let logicalRequestSeq = 0; interface SharedSendLedger { spent: number; pendingExternalSends: number; + /** + * Replacements this logical request has already spent on ambiguous failures. Beside `spent` + * for the same reason `pendingExternalSends` is: a derived scope that shared one without the + * other would hand the request a second grant. + */ + ambiguousResendsClaimed: number; readonly observer?: RequestSendObserver; } @@ -239,6 +260,12 @@ function createRequestExecutionBudgetWithLedger( const capped = Number.isFinite(cap) ? Math.trunc(cap) : 0; return Math.max(0, Math.min(capped, policy.baseSendAllowance - counter.spent)); }, + claimAmbiguousResend(limit: number): boolean { + const ceiling = Number.isFinite(limit) ? Math.trunc(limit) : 0; + if (counter.ambiguousResendsClaimed >= ceiling) return false; + counter.ambiguousResendsClaimed += 1; + return true; + }, reserveDispatch(intent: DispatchIntent): DispatchDecision { if (intent.replaySafe === false) return { allowed: false, reason: "not-replay-safe" }; if (counter.spent >= policy.maxTotalModelSends) return { allowed: false, reason: "total-exhausted" }; @@ -336,6 +363,7 @@ export function createRequestExecutionBudget( return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, { spent: 0, pendingExternalSends: 0, + ambiguousResendsClaimed: 0, ...(observer ? { observer } : {}), }); } @@ -370,11 +398,14 @@ function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger { const existing = sharedSendLedgers.get(parent); if (existing) return existing; let pendingExternalSends = 0; + let ambiguousResendsClaimed = 0; return { get spent(): number { return parent.used; }, set spent(next: number) { parent.used = next; }, get pendingExternalSends(): number { return pendingExternalSends; }, set pendingExternalSends(next: number) { pendingExternalSends = next; }, + get ambiguousResendsClaimed(): number { return ambiguousResendsClaimed; }, + set ambiguousResendsClaimed(next: number) { ambiguousResendsClaimed = next; }, }; } diff --git a/src/lib/request-resend-gate.ts b/src/lib/request-resend-gate.ts new file mode 100644 index 00000000000..0107aa648bb --- /dev/null +++ b/src/lib/request-resend-gate.ts @@ -0,0 +1,138 @@ +/** + * Whether one leg of a logical request may send it again, asked in the #5266 vocabulary. + * + * Two pull requests arrived at this question from opposite sides of the response head. #4942 + * asked it for a connection that died before any head; #4989 asked it for an SSE body that + * died after the head while carrying only control events. Both are the same row of the stage + * table: a stage the caller observed nothing at, with a cause that cannot prove the origin did + * not run the turn. `resendPermission` answers `refused-ambiguous` for both, and + * request-failure-model.ts already names the only thing that may override that answer -- a + * narrowly scoped recovery a maintainer opted into and bounded. + * + * One override, not two. The reason this module exists rather than a boolean in each caller is + * that a request which resets before the head and again after it would otherwise buy a + * replacement send on each side, and the second one is exactly the duplicated inference the + * refusal exists to prevent. The allowance is claimed HERE, at the moment of authorisation, so + * a caller cannot ask without paying. + * + * MUST stay a leaf. It imports the vocabulary as values and everything else as types, so it + * reaches no request path that did not already have it. + */ +import { + causeForRecoveryKind, + permitsResend, + resendPermission, + resendSendClass, + type RequestFailureCause, + type RequestFailureStage, + type ResendPermission, +} from "./request-failure-model"; +import type { SendClass } from "./request-execution-budget"; +import type { AttemptRecoveryKind } from "../usage/telemetry-contract"; + +/** + * Why an authorisation was refused. + * + * The three ambiguous members are separate because they need different operator responses: no + * policy is a configuration choice, a request the proxy cannot judge is a property of the turn, + * and a spent allowance means the replacement already went somewhere else in this request. + */ +export const RESEND_REFUSALS = Object.freeze([ + /** The caller already observed output, an effect, or the delivered answer. */ + "committed", + /** Identical bytes would get the identical answer. */ + "futile", + /** The origin's execution state is unknown and no operator policy overrides that. */ + "ambiguous-no-policy", + /** An operator policy exists, but this request's second send could do more than re-infer. */ + "ambiguous-request-not-replayable", + /** The operator policy exists and its replacement was already spent by this request. */ + "ambiguous-allowance-spent", +] as const); + +export type ResendRefusal = typeof RESEND_REFUSALS[number]; + +/** + * The operator-granted replacement for ONE logical request. + * + * `claim` is the single counter both stages draw on. It is a method rather than a number + * because the holder is the request's send ledger, which a combo child shares with its parent; + * a number passed down per leg is what let each leg hold its own. + */ +export interface AmbiguousResendAllowance { + /** True when a second send of this request's body can only repeat the inference. */ + readonly selfContained: boolean; + /** Spend one replacement. False once the request has none left. */ + claim(): boolean; +} + +interface ResendDecisionBase { + readonly stage: RequestFailureStage; + readonly cause: RequestFailureCause; + readonly permission: ResendPermission; + /** + * The recovery this send will be recorded as, when the caller asked in those terms. Carried + * back rather than re-chosen at the call site: the cause was derived from it, so recording a + * different kind would describe the send by a reason the gate never evaluated. + */ + readonly recoveryKind?: AttemptRecoveryKind; +} + +export type ResendDecision = + | ResendDecisionBase & { + readonly allowed: true; + /** Which request-wide send class funds it, or null when the cause funds no resend. */ + readonly sendClass: SendClass | null; + /** True when the table refused and an operator allowance was spent to proceed. */ + readonly spentOperatorAllowance: boolean; + } + | ResendDecisionBase & { readonly allowed: false; readonly refusal: ResendRefusal }; + +/** + * Decide whether this proxy may send the request again after a failure at `stage` caused by + * `cause`, spending `allowance` when the table refuses only because the upstream state is + * unknown. + * + * The allowance is touched on exactly one path: a decision the table would otherwise refuse as + * ambiguous, for a request whose body the caller has judged replayable. A committed or futile + * failure never reaches it, so a turn that already produced output cannot quietly drain the + * replacement a later ambiguous reset would have been entitled to. + */ +export function authorizeResend( + stage: RequestFailureStage, + cause: RequestFailureCause, + allowance?: AmbiguousResendAllowance, + recoveryKind?: AttemptRecoveryKind, +): ResendDecision { + const permission = resendPermission(stage, cause); + const base = { stage, cause, permission, ...(recoveryKind ? { recoveryKind } : {}) }; + if (permitsResend(permission)) { + return { ...base, allowed: true, sendClass: resendSendClass(cause), spentOperatorAllowance: false }; + } + if (permission === "refused-committed") return { ...base, allowed: false, refusal: "committed" }; + if (permission === "refused-futile") return { ...base, allowed: false, refusal: "futile" }; + if (!allowance) return { ...base, allowed: false, refusal: "ambiguous-no-policy" }; + if (!allowance.selfContained) { + return { ...base, allowed: false, refusal: "ambiguous-request-not-replayable" }; + } + // Claimed last, and only here. Asking earlier would spend the request's one replacement on a + // question whose answer was already no. + if (!allowance.claim()) return { ...base, allowed: false, refusal: "ambiguous-allowance-spent" }; + return { ...base, allowed: true, sendClass: resendSendClass(cause), spentOperatorAllowance: true }; +} + +/** + * The same decision, asked in terms of the recovery this proxy will RECORD for the send. + * + * Deriving the cause from the recorded kind is what keeps the log honest: the reason an + * operator reads beside a send count is the reason the gate weighed, because it is the same + * value. A call site that recorded one kind and reasoned about another is how a send count + * stops meaning anything. + */ +export function authorizeResendForRecovery( + stage: RequestFailureStage, + kind: AttemptRecoveryKind, + allowance?: AmbiguousResendAllowance, +): ResendDecision { + return authorizeResend(stage, causeForRecoveryKind(kind), allowance, kind); +} diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 9250162da26..44ca16bc4d1 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -139,6 +139,16 @@ export function createResponsesSendBudget( */ const sendBudgetExhausted = (cap: number = TRANSIENT_RETRY_MAX_ATTEMPTS): boolean => remainingTransientSendBudget(cap) === 0; + /** + * Spend one operator-granted replacement for an ambiguous failure of THIS logical request. + * + * The counter is the execution budget's, so a combo child that derives its own scope draws on + * the same grant. A budget that predates it -- a stub, or a caller that passed the narrow + * holder -- cannot grant anything, and refusing is the fail-closed answer for a send whose + * upstream state is unknown. + */ + const claimAmbiguousResend = (limit: number): boolean => + isRequestExecutionBudget(sendBudget) && sendBudget.claimAmbiguousResend?.(limit) === true; /** * A credential hop reserves the send its own replay will make, and that replay is a recovery * leg. The leg must SPEND the hop's reservation instead of taking a second one: the @@ -263,6 +273,7 @@ export function createResponsesSendBudget( noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, sendBudgetExhausted, + claimAmbiguousResend, get pendingHopPermit(): SingleUseDispatchPermit | undefined { return pendingHopPermit; }, @@ -301,6 +312,7 @@ function adapterDispatchBudgetView( get targetTransitions(): number { return budget.targetTransitions; }, get lastTargetKey(): string | undefined { return budget.lastTargetKey; }, remainingBaseSends: (cap: number): number => budget.remainingBaseSends(cap), + claimAmbiguousResend: (limit: number): boolean => budget.claimAmbiguousResend?.(limit) === true, reserveDispatch(intent: DispatchIntent): DispatchDecision { // A dispatch whose upstream state is unknown is refused on its own merits. A hop that // already paid does not make an unsafe replay safe, so that check stays with the budget. diff --git a/structure/overview.md b/structure/overview.md index db168062a43..ac1fb372792 100644 --- a/structure/overview.md +++ b/structure/overview.md @@ -145,6 +145,13 @@ still cover the rule, which is a judgement only review makes. which of the three refusals it is. The decision is derived from per-stage and per-cause facts rather than written out as a stage-by-cause matrix, so a new member cannot leave a stale cell. Enforced by `tests/lib/failure-stage-model.test.ts`. +- **INV-RESEND-02** — One logical request holds one operator-granted replacement for an ambiguous + failure, however many stages ask for it. `src/lib/request-resend-gate.ts` is the only place + that override is applied, it claims the grant at the moment it authorises rather than earlier, + and a stage the caller observed something at refuses without spending it. The grant never + widens a send budget: an authorised replacement still has to fit the allowance the leg already + had. + Enforced by `tests/lib/request-resend-gate.test.ts`. CI enumerates that domain layout through `scripts/ci/run-bun-test-batches.sh`. Its default general scope and 12-file/120-second process shape leave the dedicated Linux storage-policy and api-usage diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 53fc4cbab3d..bed9ce79b2e 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1442,5 +1442,8 @@ "claude-intercept-settings.test.ts": "claude-integration", "claude-desktop-first-party.test.ts": "claude-integration", "claude-desktop-mode-explanation.test.ts": "claude-integration", - "claude-intercept-integration.test.ts": "server" + "claude-intercept-integration.test.ts": "server", + "request-resend-gate.test.ts": "lib", + "management-provider-reset-replay.test.ts": "server", + "responses-reset-replay.test.ts": "responses" } diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index 3f784e3c145..9f7f96b0ec8 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -482,3 +482,38 @@ describe("derived scopes and the durable spend observer", () => { expect(parent.used).toBe(0); }); }); + +describe("the ambiguous-resend allowance", () => { + test("one logical request holds one grant, and a derived scope shares it", () => { + // The reason the grant lives here rather than beside the policy that issues it: a combo + // child derives its own budget, and two grants would let one turn replace an + // unknown-state send twice -- once on the parent leg, once on the child's. + const parent = createRequestExecutionBudget(CODEX_TEXT_GUARDED_BUDGET_POLICY); + const child = deriveRequestExecutionBudget(parent, CODEX_TEXT_GUARDED_BUDGET_POLICY); + + expect(parent.claimAmbiguousResend?.(1)).toBe(true); + expect(child.claimAmbiguousResend?.(1)).toBe(false); + expect(parent.claimAmbiguousResend?.(1)).toBe(false); + // Raising the ceiling releases exactly the difference, not a fresh grant. + expect(child.claimAmbiguousResend?.(2)).toBe(true); + expect(child.claimAmbiguousResend?.(2)).toBe(false); + }); + + test("a grant is not a send, and a spent send budget is not a spent grant", () => { + const budget = createRequestExecutionBudget(ONE_SEND_LEFT); + expect(budget.reserveDispatch({ sendClass: "initial", targetKey: "t" }).allowed).toBe(true); + expect(budget.remainingBaseSends(5)).toBe(0); + // The grant survives, because it authorises nothing by itself: the send it would fund + // still has to fit in the allowance, which is the caller's check. + expect(budget.claimAmbiguousResend?.(1)).toBe(true); + expect(budget.used).toBe(1); + }); + + test("a ceiling of zero or a nonsense ceiling grants nothing", () => { + const budget = createRequestExecutionBudget(CODEX_TEXT_GUARDED_BUDGET_POLICY); + expect(budget.claimAmbiguousResend?.(0)).toBe(false); + expect(budget.claimAmbiguousResend?.(Number.NaN)).toBe(false); + expect(budget.claimAmbiguousResend?.(Number.POSITIVE_INFINITY)).toBe(false); + expect(budget.claimAmbiguousResend?.(1)).toBe(true); + }); +}); diff --git a/tests/lib/request-resend-gate.test.ts b/tests/lib/request-resend-gate.test.ts new file mode 100644 index 00000000000..8f8bd7dea45 --- /dev/null +++ b/tests/lib/request-resend-gate.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import { + authorizeResend, + authorizeResendForRecovery, + RESEND_REFUSALS, + type AmbiguousResendAllowance, +} from "../../src/lib/request-resend-gate"; +import { + causeDisposition, + causeForRecoveryKind, + permitsResend, + REQUEST_FAILURE_CAUSES, + REQUEST_FAILURE_STAGES, + resendPermission, + resendSendClass, + stageCommitment, +} from "../../src/lib/request-failure-model"; +import { ATTEMPT_RECOVERY_KIND_ROSTER } from "../../src/usage/telemetry-contract"; + +/* + * Holds INV-RESEND-02 from structure/overview.md. + */ + +/** + * A grant with a counted `claim`. `limit` is the number of replacements the whole logical + * request may make, which is the property every test below is really about. + */ +function allowance(options: { selfContained?: boolean; limit?: number } = {}): AmbiguousResendAllowance & { + claims: () => number; +} { + let claimed = 0; + const limit = options.limit ?? 1; + return { + selfContained: options.selfContained ?? true, + claim: () => { + if (claimed >= limit) return false; + claimed += 1; + return true; + }, + claims: () => claimed, + }; +} + +describe("authorizeResend over the whole stage-by-cause cross product", () => { + test("agrees with the table wherever the table already answers", () => { + for (const stage of REQUEST_FAILURE_STAGES) { + for (const cause of REQUEST_FAILURE_CAUSES) { + const permission = resendPermission(stage, cause); + const decision = authorizeResend(stage, cause); + expect(decision.permission).toBe(permission); + expect(decision.stage).toBe(stage); + expect(decision.cause).toBe(cause); + // The gate adds an override for exactly one answer and changes none of the others. + if (permission !== "refused-ambiguous") { + expect(decision.allowed).toBe(permitsResend(permission)); + } + if (decision.allowed) expect(decision.sendClass).toBe(resendSendClass(cause)); + } + } + }); + + test("a stage the caller observed something at refuses whatever the operator granted", () => { + const grant = allowance({ limit: 5 }); + for (const stage of REQUEST_FAILURE_STAGES) { + if (stageCommitment(stage) === "nothing-observed") continue; + for (const cause of REQUEST_FAILURE_CAUSES) { + const decision = authorizeResend(stage, cause, grant); + expect(decision.allowed).toBe(false); + if (!decision.allowed) expect(decision.refusal).toBe("committed"); + } + } + // Nothing committed may drain the grant a later ambiguous failure is entitled to. + expect(grant.claims()).toBe(0); + }); + + test("a futile cause refuses without spending the grant", () => { + const grant = allowance({ limit: 5 }); + for (const cause of REQUEST_FAILURE_CAUSES) { + if (causeDisposition(cause) !== "resend-is-futile") continue; + const decision = authorizeResend("pre-header", cause, grant); + expect(decision.allowed).toBe(false); + if (!decision.allowed) expect(decision.refusal).toBe("futile"); + } + expect(grant.claims()).toBe(0); + }); + + test("every refusal the gate can produce is a declared member", () => { + const produced = new Set(); + const cases: Array = [ + undefined, + allowance({ selfContained: false }), + allowance({ limit: 0 }), + ]; + for (const stage of REQUEST_FAILURE_STAGES) { + for (const cause of REQUEST_FAILURE_CAUSES) { + for (const grant of cases) { + const decision = authorizeResend(stage, cause, grant); + if (!decision.allowed) produced.add(decision.refusal); + } + } + } + for (const refusal of produced) expect(RESEND_REFUSALS).toContain(refusal); + // Every declared member is reachable, so the roster is not carrying a dead name. + expect([...RESEND_REFUSALS].sort()).toEqual([...produced].sort()); + }); +}); + +describe("the ambiguous override", () => { + const ambiguous = REQUEST_FAILURE_STAGES.filter(stage => stageCommitment(stage) === "nothing-observed"); + + test("refuses with no operator policy and never claims", () => { + for (const stage of ambiguous) { + const decision = authorizeResend(stage, "transport-ambiguous"); + expect(decision.allowed).toBe(false); + if (!decision.allowed) expect(decision.refusal).toBe("ambiguous-no-policy"); + } + }); + + test("refuses a request it cannot judge replayable, without spending the grant", () => { + const grant = allowance({ selfContained: false, limit: 3 }); + for (const stage of ambiguous) { + const decision = authorizeResend(stage, "transport-ambiguous", grant); + expect(decision.allowed).toBe(false); + if (!decision.allowed) expect(decision.refusal).toBe("ambiguous-request-not-replayable"); + } + expect(grant.claims()).toBe(0); + }); + + test("one logical request buys one replacement, whichever stage asks first", () => { + // The defect this gate exists to prevent: #4942 asked before the response head and #4989 + // asked after it, and two separate grants would let one turn be sent twice more. + const grant = allowance({ limit: 1 }); + const first = authorizeResend("pre-header", "transport-ambiguous", grant); + expect(first.allowed).toBe(true); + if (first.allowed) expect(first.spentOperatorAllowance).toBe(true); + + for (const stage of ambiguous) { + const later = authorizeResend(stage, "transport-ambiguous", grant); + expect(later.allowed).toBe(false); + if (!later.allowed) expect(later.refusal).toBe("ambiguous-allowance-spent"); + } + expect(grant.claims()).toBe(1); + }); + + test("a grant of two is spent exactly twice", () => { + const grant = allowance({ limit: 2 }); + expect(authorizeResend("pre-header", "transport-ambiguous", grant).allowed).toBe(true); + expect(authorizeResend("protocol-prelude", "transport-ambiguous", grant).allowed).toBe(true); + expect(authorizeResend("headers-only", "transport-ambiguous", grant).allowed).toBe(false); + expect(grant.claims()).toBe(2); + }); +}); + +describe("authorizeResendForRecovery", () => { + test("derives the cause from the kind it will be recorded as, for every kind", () => { + for (const kind of ATTEMPT_RECOVERY_KIND_ROSTER) { + const decision = authorizeResendForRecovery("pre-header", kind, allowance({ limit: 99 })); + expect(decision.cause).toBe(causeForRecoveryKind(kind)); + expect(decision.recoveryKind).toBe(kind); + // The answer is the table's, asked in the other vocabulary. + expect(decision.permission).toBe(resendPermission("pre-header", causeForRecoveryKind(kind))); + } + }); + + test("a connection reset is the ambiguous row at both observable-nothing stages", () => { + for (const stage of ["pre-header", "headers-only", "protocol-prelude"] as const) { + expect(authorizeResendForRecovery(stage, "connection-reset").permission).toBe("refused-ambiguous"); + expect(authorizeResendForRecovery(stage, "connection-reset", allowance()).allowed).toBe(true); + } + expect(authorizeResendForRecovery("semantic-output", "connection-reset", allowance()).allowed).toBe(false); + }); +}); From 5dc887813c8e32c54a89d4e2c59890181c83180e Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 21:48:42 +0900 Subject: [PATCH 2/9] feat(responses): replace an ambiguous native Responses send through the shared gate #4942 and #4989 arrived as two features and are one. Both ask whether a native Responses send that failed with the caller having observed nothing may be sent again; they differ only in where they ask it. #4942 asks before any response head, #4989 after a head whose SSE body carried only control events. Against the landed stage table those are the same row, so this is one rework rather than two merged branches. The provider opts in with providers..retryOnReset, the request has to be one reset-replay.ts can judge self-contained -- store: false, complete input, no server-side continuation state, only client-executed tools -- and the whole logical request holds one replacement grant, whichever stage asks for it. replacements counts duplicate inferences the operator accepts, not retries and not sends, which is why its ceiling is two rather than a send budget. Pre-header: fetchWithResetRetry takes a claim callback rather than a count. A count handed to each leg is a count each leg holds, and the rotation, refresh and same-target 429 legs all carry the same turn. Once a replacement has gone out the leg can only settle as the refusal -- including when a later attempt fails some other way, because throwing there becomes a 502 at the caller and a 502 is what the Codex client re-sends four more times. The 401 replay leg is routed through the same helper for exactly that reason; it used to reject straight into that path. Post-header: the SSE preflight now reports the stage it observed rather than a boolean, and the gate decides. headers-only before any parsed event, protocol-prelude after response.created, semantic-output once anything else arrives -- including a payload the inspector could not parse, because an unreadable frame may be output. #4989 required response.created; the table gives headers-only the same commitment and therefore the same answer, so it is admitted rather than refused. A response.created whose snapshot already carries output items is not a prelude. The replacement send is charged to the same request counter every other send uses and recorded with the kind the gate derived its cause from, so one authorisation is one reason and one send. The deferred preflight only wraps a body when the provider opted in, so a proxy that configures nothing buffers nothing and its first byte is unchanged. Co-authored-by: Fred Amartey <43480311+FredAmartey@users.noreply.github.com> Co-authored-by: lidge-jun --- src/config.ts | 2 +- src/config/load-degrade.ts | 41 +++- src/config/schema/leaf-validators.ts | 15 ++ src/lib/upstream-retry.ts | 133 +++++++++-- src/providers/key-failover.ts | 32 ++- src/server/auth-cors.ts | 10 + src/server/relay.ts | 13 ++ .../responses/combo-stream-preflight.ts | 163 ++++++++++++- src/server/responses/fetch-helpers.ts | 11 +- src/server/responses/passthrough-dispatch.ts | 214 +++++++++++++++--- src/server/responses/reset-replay.ts | 104 +++++++++ src/types.ts | 1 + src/types/provider.ts | 31 +++ tests/lib/upstream-retry.test.ts | 95 ++++++++ .../responses/responses-reset-replay.test.ts | 122 ++++++++++ tests/routing/combo-stream-preflight.test.ts | 52 ++++- .../management-provider-reset-replay.test.ts | 51 +++++ 17 files changed, 1021 insertions(+), 69 deletions(-) create mode 100644 src/server/responses/reset-replay.ts create mode 100644 tests/responses/responses-reset-replay.test.ts create mode 100644 tests/server/management-provider-reset-replay.test.ts diff --git a/src/config.ts b/src/config.ts index 9788f252144..0c63a168bad 100644 --- a/src/config.ts +++ b/src/config.ts @@ -110,7 +110,7 @@ export { sanitizeModelCostsForDisplay, modelPreferHostedToolsConfigError, } from "./config/schema/leaf-validators"; -export { hardenExistingSecret, retryOn429PolicyConfigError } from "./config/load-degrade"; +export { hardenExistingSecret, retryOn429PolicyConfigError, retryOnResetPolicyConfigError } from "./config/load-degrade"; export { backupInvalidConfig } from "./config/salvage"; export type { ConfigDiagnostics, ConfigAdmissionSnapshot } from "./config/diagnostics"; export { diff --git a/src/config/load-degrade.ts b/src/config/load-degrade.ts index 1787ad109a6..daf16aba298 100644 --- a/src/config/load-degrade.ts +++ b/src/config/load-degrade.ts @@ -32,6 +32,7 @@ import { quotaResetNotifySchema, remoteGuiConfigSchema, retryOn429PolicySchema, + retryOnResetPolicySchema, runtimeRoleSchema, spendSchema, } from "./schema/leaf-validators"; @@ -195,18 +196,44 @@ export function sanitizeRetryOn429ForLoad(parsed: unknown): void { * redacted (a malformed write can place a secret in a property name). */ export function retryOn429PolicyConfigError(policy: unknown): string | null { + return strictPolicyConfigError("retryOn429", retryOn429PolicySchema, policy); +} + +/** + * Management write-boundary validation for `retryOnReset`, with the same fail-closed contract + * as `retryOn429PolicyConfigError`: the load-time schema degrades a malformed block to + * "absent", so this is the one place a bad value is refused instead of silently dropped. + */ +export function retryOnResetPolicyConfigError(policy: unknown): string | null { + return strictPolicyConfigError("retryOnReset", retryOnResetPolicySchema, policy); +} + +/** + * The shared body of both. Written once because the two differ only in the field name they + * report, and a second hand-copied formatter is a second place for the redaction to be + * forgotten. + */ +function strictPolicyConfigError( + field: string, + schema: { + safeParse: (value: unknown) => { success: true } | { + success: false; + error: { issues: Array<{ code: string; message: string; path: PropertyKey[]; keys?: string[] }> }; + }; + }, + policy: unknown, +): string | null { if (policy === undefined) return null; - const result = retryOn429PolicySchema.safeParse(policy); + const result = schema.safeParse(policy); if (result.success) return null; const first = result.error.issues[0]; - if (!first) return "retryOn429 is invalid"; - if (first.code === "unrecognized_keys") { + if (!first) return `${field} is invalid`; + if (first.code === "unrecognized_keys" && first.keys) { const names = first.keys.map(key => JSON.stringify(redactSecretString(key))).join(", "); - return `retryOn429 has unrecognized field${first.keys.length > 1 ? "s" : ""}: ${names}`; + return `${field} has unrecognized field${first.keys.length > 1 ? "s" : ""}: ${names}`; } - if (first.path.length === 0) return `retryOn429 is invalid (${first.message})`; - const field = String(first.path[first.path.length - 1]); - return `retryOn429.${field} is invalid (${first.message})`; + if (first.path.length === 0) return `${field} is invalid (${first.message})`; + return `${field}.${String(first.path[first.path.length - 1])} is invalid (${first.message})`; } export function sanitizeCapabilityDeclarationsForLoad(parsed: unknown): void { diff --git a/src/config/schema/leaf-validators.ts b/src/config/schema/leaf-validators.ts index 0906b75671e..0210a9d0c3d 100644 --- a/src/config/schema/leaf-validators.ts +++ b/src/config/schema/leaf-validators.ts @@ -81,6 +81,17 @@ const transientRetryOn5xxPolicySchema = z.object({ attempts: z.number().int().min(1).max(10).optional(), }).strict(); +/** + * `retryOnReset` accepts only these keys. `replacements` counts DUPLICATE inferences the + * operator is willing to risk for one logical request, so the ceiling is two rather than a + * send budget: this is the one send the proxy otherwise refuses outright, and a third of them + * says the connection, not the retry policy, is the problem. + */ +export const retryOnResetPolicySchema = z.object({ + enabled: z.boolean().optional(), + replacements: z.number().int().min(1).max(2).optional(), +}).strict(); + const requestPacingRuleSchema = z.object({ // Keep the RPM-derived timer within the same one-hour bound as minIntervalMs. requestsPerMinute: z.number().min(1 / 60).max(60_000).optional(), @@ -331,6 +342,10 @@ export const providerConfigSchema = z.object({ .optional(), retryOn429: retryOn429PolicySchema.optional(), transientRetryOn5xx: transientRetryOn5xxPolicySchema.optional(), + // Degrades to "absent" like `webSearchBridge`: a malformed hand edit of an opt-in feature + // that is off by default must not send the operator through invalid-config recovery. The + // management write boundary still rejects it loudly (`retryOnResetPolicyConfigError`). + retryOnReset: retryOnResetPolicySchema.optional().catch(undefined), codexAccountMode: z.enum(["pool", "direct"]).optional(), // Validated rather than passed through: this schema ends in `.passthrough()`, so an // undeclared key survives verbatim. A misspelled `codexToolMode` therefore used to be diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 8dd6f1146f1..30f17a7b5d8 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -8,13 +8,15 @@ * becomes a terminal, non-replayable response unless the operation is explicitly safe. * * Deliberately narrow: timeouts, aborts, ECONNREFUSED/DNS/TLS failures, and HTTP error - * statuses (returned as Response, never thrown) are NOT retried. Mid-stream SSE resets are - * out of scope — the response has already resolved by then. + * statuses (returned as Response, never thrown) are NOT retried. A reset after the response + * head is out of scope here, because the response has already resolved by then; the Responses + * transport asks the same question at that stage through the shared resend gate. * * MUST stay a leaf module: imports nothing from server.ts or adapters (kiro-retry imports * the shared abort helpers from here). */ import { clearableDeadline } from "./abort"; +import { redactSecretString } from "./redact"; /** * Responses the origin may already be executing. RFC 9110 §9.2.2 forbids an intermediary @@ -421,6 +423,20 @@ export interface ResetRetryOptions { * uncounted but UNCOUNTABLE: the callback existed on a type those call sites never reach. */ onSendsConsumed?: (sends: number) => void; + /** + * Spend one operator-granted replacement for a pre-header reset this helper would otherwise + * refuse. Absent means no operator policy, which is the fail-closed answer. + * + * A callback rather than a count, and the difference is the whole point. A count handed to + * each leg of a request is a count each leg holds: a rotation leg, a refresh leg and a + * same-target 429 leg carry the same turn, so three numbers is three replacements of one + * possibly-executed inference. The callback draws on ONE allowance held by the logical + * request, which the post-header protocol gate draws on too. + * + * It never widens the send budget. A claimed replacement still has to fit inside + * `attempts`, exactly like every other send this leg makes. + */ + claimAmbiguousResend?: () => boolean; } export interface TransientRetryOptions extends ResetRetryOptions { @@ -494,6 +510,23 @@ export function applyUpstreamRecoveryInit( return { ...init, headers, keepalive: false }; } +/** + * The refusal this proxy returns for an ambiguous reset it will not replace. The WeakSet + * markers protect in-process recovery and the code survives JSON re-wrapping, so a combo or + * account-recovery layer downstream cannot read it as a replayable upstream fault. The raw + * exception is never exposed: it can carry credentials or request data. + */ +export function replayRefusalResponse(): Response { + const response = new Response(JSON.stringify({ error: { + type: "upstream_error", + code: UPSTREAM_RESET_REPLAY_REFUSED_CODE, + message: "The upstream connection closed before a response was received. The request may already have been processed; automatic replay was stopped.", + } }), { status: REPLAY_REFUSED_STATUS, headers: { "content-type": "application/json" } }); + markResponseNonReplayable(response); + markReplayRefusalResponse(response); + return response; +} + /** * Run `doFetch` within one send budget. Connection-reset-shaped rejections are * terminal by default; only an explicitly replay-safe operation receives reset retries @@ -511,6 +544,10 @@ export async function fetchWithResetRetry( if (attempts === 0) throw new SendBudgetExhaustedError(opts.label); let lastError: unknown; let sawReset = false; + // True once this leg has spent the request's operator allowance. From that point the leg can + // only settle as the refusal: a second send of a possibly-executed turn is already out, and + // handing the client anything it would retry compounds it. + let spentOperatorReplacement = false; for (let attempt = 0; attempt < attempts; attempt++) { if (opts.abortSignal?.aborted) throw abortError(opts.abortSignal); // Reported before the await, one physical send at a time: a send that rejects has still @@ -522,31 +559,37 @@ export async function fetchWithResetRetry( } catch (err) { if (opts.abortSignal?.aborted) throw err; if (!isConnectionResetError(err)) { + // Whatever ended the leg, an operator replacement already went out, so the first send + // may have run the turn. Settle it as the refusal instead of throwing into a caller + // whose transport-failure path answers with a client-retryable 502. + if (spentOperatorReplacement) return replayRefusalResponse(); // A reset that already reached the origin is credential-visible // evidence: keep it attached so the terminal rejection cannot be // downgraded to the pre-connection neutral class (#914 review). if (sawReset) throw new UpstreamRetryEvidenceError([], err, true); throw err; } - if (opts.replaySafe !== true) { - // Return evidence instead of throwing a generic transport error: outer catches + if (opts.replaySafe === true) { + // Repeating this operation cannot duplicate anything, so an exhausted budget rethrows + // and the caller's own error path takes over. + if (attempt === attempts - 1) throw err; + } else { + // The stage table refuses an ambiguous pre-header reset. The only thing that overrides + // it is an operator allowance, and claiming it here is what keeps the grant single -- + // the post-header protocol gate spends the same counter for the same logical request. + // Return evidence rather than throwing a generic transport error: outer catches // otherwise turn it into a replayable 502 and a combo/account recovery resends it. - // The WeakSet protects in-process recovery; the code survives JSON re-wrapping. - // Never expose the raw exception, which can contain credentials or request data. - const response = new Response(JSON.stringify({ error: { - type: "upstream_error", - code: UPSTREAM_RESET_REPLAY_REFUSED_CODE, - message: "The upstream connection closed before a response was received. The request may already have been processed; automatic replay was stopped.", - } }), { status: REPLAY_REFUSED_STATUS, headers: { "content-type": "application/json" } }); - markResponseNonReplayable(response); - markReplayRefusalResponse(response); - return response; + if (attempt + 1 >= attempts || opts.claimAmbiguousResend?.() !== true) { + return replayRefusalResponse(); + } + spentOperatorReplacement = true; } - if (attempt === attempts - 1) throw err; sawReset = true; lastError = err; console.warn( - `[upstream-retry] connection reset${opts.label ? ` (${opts.label})` : ""} — retrying (${attempt + 2}/${attempts})`, + `[upstream-retry] connection reset${opts.label ? ` (${opts.label})` : ""} — ${ + spentOperatorReplacement ? "replacing" : "retrying" + } (${attempt + 2}/${attempts})`, ); await sleepWithAbort(retryBackoffDelayMs(attempt, { baseDelayMs: RESET_RETRY_BASE_DELAY_MS, @@ -661,3 +704,61 @@ export async function fetchWithTransientRetry( opts.onSendsConsumed?.(sent); } } + +export type ProtocolSafeRefetch = (signal?: AbortSignal) => Promise; + +export interface ProtocolSafeRefetchOptions extends ResetRetryOptions { + /** The replacement must match the response contract already selected for the client. */ + acceptResponse?: (response: Response) => boolean; + /** + * Spend the logical request's allowance, immediately before the replacement send. + * + * Asked here and not earlier so a failure this helper would refuse on its own terms -- a + * non-reset error, a cancelled caller, a spent send budget -- cannot drain the one + * replacement a later ambiguous reset was entitled to. False refuses the replacement. + */ + authorize?: () => boolean; +} + +/** + * Attempt ONE caller-authorized replacement of a stream that died after the response head. + * + * The caller owns the proof that nothing was observed -- it comes from protocol inspection, + * not from this module -- and owns the physical-send budget. What lives here is the part that + * is easy to get wrong: a replacement is only usable if it is a fresh, unlocked, unread body + * that matches the contract already promised to the client, and anything else has to be + * cancelled and the original failure preserved. + */ +export async function refetchAfterProtocolSafeReset( + doFetch: ProtocolSafeRefetch, + err: unknown, + opts: ProtocolSafeRefetchOptions = {}, +): Promise { + if (!isConnectionResetError(err) || opts.abortSignal?.aborted || opts.attempts === 0) return null; + const label = opts.label + ? " (" + redactSecretString(opts.label).replace(/[\r\n\u0000-\u001f\u007f]/g, "").slice(0, 128) + ")" + : ""; + if (opts.authorize && !opts.authorize()) { + console.warn("[upstream-retry] post-header reset replacement refused" + label + "; preserving original stream error"); + return null; + } + let replacement: Response; + try { + replacement = await doFetch(opts.abortSignal); + } catch { + console.warn("[upstream-retry] protocol-safe refetch failed" + label + "; preserving original stream error"); + return null; + } + const body = replacement.body; + let accepted = !opts.abortSignal?.aborted && replacement.ok && body !== null + && !replacement.bodyUsed && !body.locked && !isNonReplayableResponse(replacement); + try { if (accepted && opts.acceptResponse) accepted = opts.acceptResponse(replacement); } + catch { accepted = false; } + if (!accepted || opts.abortSignal?.aborted || body?.locked) { + try { void body?.cancel().catch(() => {}); } catch { /* already locked or closed */ } + console.warn("[upstream-retry] protocol-safe refetch rejected" + label + "; preserving original stream error"); + return null; + } + console.warn("[upstream-retry] pre-output Responses reset" + label + "; using one replacement stream"); + return replacement; +} diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index e4552df141a..c96b555029b 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -13,7 +13,7 @@ import type { ProviderApiKeySelection } from "../types/provider"; import { routedProviderConfig } from "../router"; import { getProviderRegistryEntry } from "./registry"; import { normalizedBaseUrl } from "./quota/vendor-probes-key"; -import type { OcxConfig, OcxProviderConfig, RateLimitRetryPolicy, TransientRetryPolicy } from "../types"; +import type { OcxConfig, OcxProviderConfig, RateLimitRetryPolicy, ResetReplayPolicy, TransientRetryPolicy } from "../types"; import { OPENCODE_GO_SESSION_HEADER } from "./opencode-go-transport"; import { resolveProviderTransport, type OcxProviderTransport } from "./xai-transport"; import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; @@ -326,6 +326,16 @@ const DEFAULT_TRANSIENT_RETRY = { attempts: 3, } as const satisfies Required; +/** + * Default used when a provider opts in with a bare `retryOnReset: {}`: one replacement for + * the whole logical request. `replacements` counts duplicate inferences the operator accepts, + * not retries and not sends. + */ +const DEFAULT_RESET_REPLAY = { + enabled: true, + replacements: 1, +} as const satisfies Required; + /** Map<`${providerName}\0${keyId}`, KeyCooldown> */ const keyCooldowns = new Map(); @@ -616,6 +626,26 @@ export function transientRetryPolicyFor( }; } +/** + * Normalize a provider's `retryOnReset` policy, or return null when it is absent or explicitly + * disabled. + * + * No auth-mode gate: the canonical ChatGPT backend is `forward` auth and is the send this + * policy exists for. Whether a given REQUEST may be replaced is a per-body decision made in + * src/server/responses/reset-replay.ts, and how many replacements the request gets is the + * shared allowance on its execution budget. This function only reads the operator's intent. + */ +export function resetReplayPolicyFor( + provider: Pick, +): Required | null { + const policy = provider.retryOnReset; + if (!policy || policy.enabled === false) return null; + return { + enabled: policy.enabled ?? DEFAULT_RESET_REPLAY.enabled, + replacements: policy.replacements ?? DEFAULT_RESET_REPLAY.replacements, + }; +} + /** * Wait before the next same-target replay: upstream Retry-After (seconds or HTTP-date) when * `respectRetryAfter` is on and the header parses, capped at `maxIntervalMs`; otherwise the diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index ad987bf4e5a..a0ef077dd5d 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -11,6 +11,7 @@ import { providerWebSearchBridgeConfigError, requestPacingConfigError, retryOn429PolicyConfigError, + retryOnResetPolicyConfigError, sanitizeModelCostsForDisplay, } from "../config"; import { @@ -723,6 +724,10 @@ export function providerManagementConfigError( delete canonicalCandidate.modelCosts; // requestPacing is a user-owned transport overlay, not part of the canonical seed. delete canonicalCandidate.requestPacing; + // retryOnReset is the same kind of overlay: it tunes how this provider's own Responses + // sends recover, not what the canonical forward seed is. Validated below + // (retryOnResetPolicyConfigError). + delete canonicalCandidate.retryOnReset; // Context windows are the same kind of user-owned overlay as requestPacing: the operator // narrowing what their own native rows advertise. They can only ever LOWER the measured // window (see nativeOpenAiContextWindow), so admitting them cannot widen what the proxy @@ -769,6 +774,10 @@ export function providerManagementConfigError( // it before it reaches the management API response. return `provider ${JSON.stringify(redactSecretString(name))} ${retryOn429Error}`; } + const retryOnResetError = retryOnResetPolicyConfigError(raw.retryOnReset); + if (retryOnResetError) { + return `provider ${JSON.stringify(redactSecretString(name))} ${retryOnResetError}`; + } const requestPacingError = requestPacingConfigError(raw.requestPacing); if (requestPacingError) { return `provider ${JSON.stringify(redactSecretString(name))} ${requestPacingError}`; @@ -1049,6 +1058,7 @@ const PROVIDER_CONFIG_FIELD_POLICY = { showThinkingSummary: "editor", retryOn429: "editor", transientRetryOn5xx: "editor", + retryOnReset: "editor", reasoningSplitModels: "editor", reasoningDetailsModels: "editor", thinkingToggleModels: "editor", diff --git a/src/server/relay.ts b/src/server/relay.ts index 7e84092fb62..4e2f8bbd67a 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -989,6 +989,14 @@ export type SseInspectorHandlers = { * with an empty `output`. */ onParsedPayload?: (payload: unknown) => void; + /** + * A complete data payload that did not parse as a JSON event, `[DONE]` included. + * + * An inspector that only hears about parsed events cannot tell "nothing has been emitted" + * from "something was emitted that I could not read", and a replay decision needs that + * difference: an unreadable payload is a payload the caller may already have seen. + */ + onOpaquePayload?: () => void; onFirstOutput?: () => void; /** * Provider-scoped compatibility: persist the completed snapshot under the @@ -1177,6 +1185,11 @@ export function createSseInspector(handlers: SseInspectorHandlers): SseInspector if (handlers.onParsedPayload && parsed !== undefined) { try { handlers.onParsedPayload(parsed); } catch { /* inspection must never throw into the pump */ } } + // The other half of the same observation. A payload that did not parse still reached the + // caller, so a consumer deciding whether anything has been emitted has to hear about it. + if (handlers.onOpaquePayload && parsed === undefined) { + try { handlers.onOpaquePayload(); } catch { /* inspection must never throw into the pump */ } + } reportFirstOutput.parsed(parsed); const status = terminalStatusFromParsed(parsed); const policyTerminal = status === "failed" diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index 22e1ae12e45..032d936e329 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -1,6 +1,7 @@ import type { ResponsesTerminalStatus } from "../../bridge"; import { comboFailureDecision } from "../../combos"; import { httpStatusFromTerminalError } from "../../lib/errors"; +import type { RequestFailureStage } from "../../lib/request-failure-model"; import type { RequestLogContext } from "../request-log"; import { createSseInspector } from "../relay"; import { MAX_CLIENT_SSE_FRAME_BYTES } from "../sse-frame-buffer"; @@ -108,9 +109,42 @@ export function comboStreamPayloadCommitsOutput(payload: unknown): boolean { if (!payload || typeof payload !== "object" || Array.isArray(payload)) return true; const type = (payload as { type?: unknown }).type; if (typeof type !== "string") return true; + if (type === "response.created") { + // A created event is a control frame only while its snapshot is empty. An origin that + // resumes a turn can put completed items in it, and treating that as a prelude would let + // a replacement re-emit output the caller already received. + const response = (payload as { response?: unknown }).response; + if (response && typeof response === "object" && !Array.isArray(response)) { + const output = (response as { output?: unknown }).output; + if (Array.isArray(output) && output.length > 0) return true; + } + } return !PRE_OUTPUT_CONTROL_EVENTS.has(type) && !TERMINAL_EVENTS.has(type); } +/** + * How far this SSE body got, in the vocabulary of src/lib/request-failure-model.ts. + * + * The preflight cannot separate `semantic-output` from `side-effect`: it classifies any event + * that is not a lifecycle control frame as committing, without reading item types. Both stages + * refuse a resend, so the distinction would change no decision -- it is named here so a later + * reader does not mistake the collapse for an omission. + * + * A terminal that settled carrying no output is `protocol-prelude`, not `terminal`. That is + * the failure model's own rule: `terminal` means the answer was delivered, and an empty + * completion delivered none. + */ +function observedResponsesStage(state: { + readonly outputCommitted: boolean; + readonly terminalStatus: ResponsesTerminalStatus | undefined; + readonly responseCreated: boolean; +}): RequestFailureStage { + if (state.outputCommitted) return "semantic-output"; + if (state.terminalStatus === "completed") return "terminal"; + if (state.responseCreated || state.terminalStatus !== undefined) return "protocol-prelude"; + return "headers-only"; +} + function replayBufferedResponse( response: Response, reader: ReadableStreamDefaultReader, @@ -186,13 +220,22 @@ function failedTerminalResponse( export type ComboStreamPreflightResult = | { kind: "accepted"; response: Response } - | { kind: "failed"; response: Response }; + | { kind: "failed"; response: Response } + /** + * The body errored mid-stream and `replayReadErrors` asked for the prefix back rather than + * a rethrow. `stage` is how far the inspection actually got; whether that permits a + * replacement is the resend gate's decision, not this function's. Callers that only act on + * a projected terminal can treat this exactly as `accepted`, which is what it was before + * the stage became observable. + */ + | { kind: "read-error"; response: Response; error: unknown; stage: RequestFailureStage }; /** - * Buffer a combo child's downstream SSE only until the request becomes unsafe to - * replay or reaches a terminal. This owns exactly one body reader. The aggregate - * buffer is capped by bytes and retained chunks; hitting either cap commits the - * current target instead of growing memory or guessing that replay is safe. + * Buffer a Responses SSE only until the request becomes unsafe to replay or reaches a + * terminal. Combo failover and native post-header reset recovery share this protocol + * boundary, because they are asking the same question about the same bytes. This owns exactly + * one body reader. The aggregate buffer is capped by bytes and retained chunks; hitting either + * cap commits the current target instead of growing memory or guessing that replay is safe. */ export async function preflightComboStreamResponse( response: Response, @@ -211,12 +254,18 @@ export async function preflightComboStreamResponse( const buffered: Uint8Array[] = []; let bufferedBytes = 0; let outputCommitted = false; + let responseCreated = false; let terminalStatus: ResponsesTerminalStatus | undefined; let retryableTerminalPayload: Record | undefined; const inspector = createSseInspector({ logCtx, + // A payload the inspector could not parse still reached this proxy, and it may be output. + // Committing on it is what keeps an unreadable frame from reading as an empty prelude. + onOpaquePayload: () => { outputCommitted = true; }, onParsedPayload: payload => { if (terminalStatus !== undefined || outputCommitted || retryableTerminalPayload) return; + if (payload !== null && typeof payload === "object" && !Array.isArray(payload) + && (payload as { type?: unknown }).type === "response.created") responseCreated = true; const retryable = retryableTerminal(payload); const matchedBareError = retryable && payload !== null && typeof payload === "object" && !Array.isArray(payload) && (payload as { type?: unknown }).type === "error"; @@ -239,7 +288,9 @@ export async function preflightComboStreamResponse( // The native relay still owns post-header transport failures. Preserve // the bounded prefix and the errored reader; cancelling it here would // erase the failure before either client relay or inspection sees it. - return { kind: "accepted", response: replayBufferedResponse(response, reader, buffered) }; + const replay = replayBufferedResponse(response, reader, buffered); + const stage = observedResponsesStage({ outputCommitted, terminalStatus, responseCreated }); + return { kind: "read-error", response: replay, error, stage }; } if (next.done) { inspector.finish(); @@ -277,3 +328,103 @@ export async function preflightComboStreamResponse( inspector.dispose(); } } + +/** Produce a replacement body for a mid-stream failure at `stage`, or null to keep the error. */ +export type ProtocolSafeResetRecovery = ( + error: unknown, + stage: RequestFailureStage, +) => Promise; + +/** + * Defer protocol inspection until the downstream actually pulls the body. + * + * Direct passthrough must return response headers before the first SSE event arrives, so the + * inspection cannot be awaited at the dispatch site the way combo routing awaits it. Wrapping + * the body moves it to the first pull, which is the earliest moment the client is willing to + * wait anyway. + */ +export function deferProtocolSafeResetRecovery( + response: Response, + logCtx: RequestLogContext, + recover: ProtocolSafeResetRecovery, + options?: { allowMissingContentType?: boolean }, +): Response { + if (!response.body) return response; + + let reader: ReadableStreamDefaultReader | undefined; + let initialization: Promise | undefined; + let closed = false; + + const cancelBody = (body: ReadableStream | null, reason?: unknown): void => { + try { void body?.cancel(reason).catch(() => {}); } catch { /* already locked or closed */ } + }; + const initialize = async (): Promise => { + const preflight = await preflightComboStreamResponse( + response, + logCtx, + () => false, + { allowMissingContentType: options?.allowMissingContentType === true, replayReadErrors: true }, + ); + let selected = preflight.response; + if (preflight.kind === "read-error") { + const replacement = await recover(preflight.error, preflight.stage); + if (replacement) { + cancelBody(selected.body, "using protocol-safe replacement stream"); + selected = replacement; + } + } + if (closed) { + cancelBody(selected.body, "downstream cancelled before protocol preflight completed"); + return; + } + reader = selected.body?.getReader(); + }; + + const body = new ReadableStream({ + async pull(controller) { + try { + initialization ??= initialize(); + await initialization; + if (closed) return; + if (!reader) { + closed = true; + controller.close(); + return; + } + const next = await reader.read(); + if (closed) return; + if (next.done) { + closed = true; + try { reader.releaseLock(); } catch { /* already released */ } + reader = undefined; + controller.close(); + return; + } + controller.enqueue(next.value); + } catch (error) { + if (closed) return; + closed = true; + try { reader?.releaseLock(); } catch { /* errored reader */ } + reader = undefined; + controller.error(error); + } + }, + cancel(reason) { + if (closed) return; + closed = true; + if (reader) { + try { void reader.cancel(reason).catch(() => {}); } catch { /* already closed */ } + try { reader.releaseLock(); } catch { /* already released */ } + reader = undefined; + } else { + cancelBody(response.body, reason); + } + }, + }, { highWaterMark: 0 }); + + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} diff --git a/src/server/responses/fetch-helpers.ts b/src/server/responses/fetch-helpers.ts index 917d74f8ad0..61981aac401 100644 --- a/src/server/responses/fetch-helpers.ts +++ b/src/server/responses/fetch-helpers.ts @@ -174,6 +174,14 @@ export function sendWithConnectionPolicy( } export interface ProviderFetchOptions { + /** + * Keep this send on HTTP even where the WebSocket upstream would normally be selected. + * + * Set by a caller replacing an HTTP stream that already failed: a WS create frame is a + * different send on a different transport, and the replacement has to be the same kind of + * exchange the client is already reading. + */ + httpOnly?: boolean; nativeControl?: NativeResponseControl; providerName?: string; modelId?: string; @@ -261,7 +269,8 @@ export function providerFetch( // else keeps the provider's HTTP fetch. See ws-upstream.ts for the details. const unpaced = async (input: Parameters[0], init?: RequestInit) => { const upstreamWebsocket = provider.upstreamWebsocket === true; - if (typeof input === "string" && init && shouldUseCodexWsUpstream(input, init, runtime, upstreamWebsocket)) { + if (!options.httpOnly && typeof input === "string" && init + && shouldUseCodexWsUpstream(input, init, runtime, upstreamWebsocket)) { const egress = egressFor(input); if (providerEgressIsExplicit(egress)) { warnEgressWebsocketDowngradeOnce(providerName, describeProviderEgressForLog(egress)); diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 6676bbd1d40..7bd9b64253c 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -102,7 +102,7 @@ import { clearCodexModelDenialEvidence, recordCodexModelDenialEvidence, } from "../../codex/model-entitlements"; -import { readCodexWsStage } from "./codex-ws-wire"; +import { isCodexWsUpstreamResponse, readCodexWsStage } from "./codex-ws-wire"; import { linkAbortSignal } from "./core-lifetime"; import type { CodexAuthContext } from "../../codex/auth-context"; import { checkOutboundBodySize, describeOutboundBodyRefusal } from "./outbound-body-guard"; @@ -112,7 +112,8 @@ import { fetchWithTransientRetry, applyUpstreamRecoveryInit, isNonReplayableResponse, - prepareSameTarget429Wait, + refetchAfterProtocolSafeReset, + prepareSameTarget429Wait, sleepWithAbort, } from "../../lib/upstream-retry"; import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; @@ -151,7 +152,9 @@ import { reasoningEffortRejectionText, } from "./core-opaque-recovery"; import type { RequestLogContext } from "../request-log"; -import { preflightComboStreamResponse } from "./combo-stream-preflight"; +import { deferProtocolSafeResetRecovery, preflightComboStreamResponse } from "./combo-stream-preflight"; +import { authorizeResendForRecovery } from "../../lib/request-resend-gate"; +import { ambiguousResendAllowanceFor, selfContainedResponsesBody } from "./reset-replay"; import { upstreamErrorMessageFromPayload, ENCRYPTED_FUNCTION_OUTPUT_REJECTION } from "../../lib/errors"; import { isTransientConsoleGoUploadRejection } from "../../providers/opencode-zen-rate-limit"; import { planReasoningEffortDowngrade } from "../../providers/reasoning-metadata"; @@ -189,6 +192,8 @@ export async function preparePassthroughExchange( | "genericFailovers" | "applyFailoverSnapshot" | "noteRoutedAttemptSend" + | "selectionIsCurrent" + | "requestBindings" >, responseEffects: Pick< ResponsesEffects, @@ -206,6 +211,7 @@ export async function preparePassthroughExchange( | "recoverySendAllowance" | "recoveryClassFor" | "sendBudgetExhausted" + | "claimAmbiguousResend" | "reserveCredentialHop" | "pendingHopPermit" | "workflowRootId" @@ -239,6 +245,7 @@ export async function preparePassthroughExchange( recoverySendAllowance, recoveryClassFor, sendBudgetExhausted, + claimAmbiguousResend, reserveCredentialHop, workflowRootId, } = sendBudgetState; @@ -704,6 +711,35 @@ export async function preparePassthroughExchange( ); const configuredTransientSendBudgetExhausted = (): boolean => transientSendPolicy() !== null && transientSendAttempts() === 0; + /** + * Judged once. The inbound body does not change between legs, and every rebuild this lane + * performs only ever REMOVES a hazard -- `previous_response_id` is expanded, hosted tools + * are lowered into client execution -- so a body that was replaceable stays replaceable. + * Memoized rather than recomputed because it walks the input array, and a provider that + * never opted in must not pay for it at all. + */ + let selfContainedJudgment: boolean | undefined; + const requestIsSelfContained = (): boolean => + selfContainedJudgment ??= selfContainedResponsesBody(parsed._rawBody); + /** + * The operator's replacement grant for THIS logical request. + * + * Read per leg because `route.provider` is reassigned by credential rotation and transport + * resolution inside the recovery loop, exactly like `transientSendPolicy`. The counter it + * claims from is not per leg: it lives on the request's execution budget, which a combo + * child shares, so every ambiguous stage of this request draws on the same grant. + */ + const ambiguousResend = () => + ambiguousResendAllowanceFor(route.provider, requestIsSelfContained, claimAmbiguousResend); + /** + * The pre-header row of the stage table, asked through the one gate. + * + * `fetchWithResetRetry` takes a plain callback because it is a leaf that must not import + * the server tree; routing the answer through `authorizeResendForRecovery` here is what + * keeps the decision derived from the table rather than restated as a boolean. + */ + const claimPreHeaderResend = (): boolean => + authorizeResendForRecovery("pre-header", "connection-reset", ambiguousResend()).allowed; /** * Refuse a built body that exceeds the operator's configured ceiling, before it is sent. * @@ -845,6 +881,7 @@ export async function preparePassthroughExchange( }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: noteTransientSends, + claimAmbiguousResend: claimPreHeaderResend, // The OpenCode Go destination stalls-then-drops inference sends (ambiguous // pre-header resets surfacing as refused 429s); its subscription traffic is // inference-only, so a bounded reset replay here absorbs the blip instead of @@ -949,7 +986,8 @@ export async function preparePassthroughExchange( route.provider.authMode === "forward") .then(adoptObservedResponse); }, - { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts, onSendsConsumed: noteTransientSends }, + { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts, + onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return { failed: transportFailureResponse(err) }; @@ -1030,35 +1068,54 @@ export async function preparePassthroughExchange( // every other build site; a replay is exactly when a grown payload reappears. const replayBodyRefusal = refuseOversizedOutboundBody(request); if (replayBodyRefusal) return replayBodyRefusal; - transportState.noteRoutedAttemptSend(passthroughEstimate, "oauth-401"); - upstreamResponse = await fetchWithHeaderTimeout( - request.url, - { method: request.method, headers: request.headers, body: request.body }, - upstream.signal, - connectMs, - parsed.stream, - // The replay-dispatched signal is what bounds the rest of this logical request, so it - // has to describe a send that actually happened. fetchWithHeaderTimeout awaits pacing - // admission BEFORE calling the executor, so signalling at the call site would spend the - // budget even when a rejected pacing wait means nothing reaches the network. Wrapping - // the executor moves the signal to the last moment before the send, where a throw from - // here on is a genuine transport attempt. - storedPoolReplayDispatchNotifier( - providerFetch(route.provider, options.codexWsRuntimeIdentity, { - nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt - && responseEffects.plaintextV2AgentMessageToolNames.size === 0 - ? options.nativeControl : undefined, - dispatchOverride: oauthDispatch(request), - providerName: route.providerName, - modelId: route.modelId, - onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), - beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider) - ? createCodexReserveDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined, - }), - codex401ReplayKind === "stored" ? options.onStoredPool401ReplayDispatched : undefined, - ), - route.provider.authMode === "forward", - ).then(adoptObservedResponse); + // The replay-dispatched signal is what bounds the rest of this logical request, so it + // has to describe a send that actually happened. fetchWithHeaderTimeout awaits pacing + // admission BEFORE calling the executor, so signalling at the call site would spend the + // budget even when a rejected pacing wait means nothing reaches the network. Wrapping + // the executor moves the signal to the last moment before the send, where a throw from + // here on is a genuine transport attempt. The notifier is built once for the whole leg: + // it fires on the first dispatch, and a replacement is another send of the same replay + // rather than a second one to announce. + const oauthReplayExecutor = storedPoolReplayDispatchNotifier( + providerFetch(route.provider, options.codexWsRuntimeIdentity, { + nativeControl: nativeResponseControlEligible(route.provider, options.nativeControl) && options.inboundTransport === "websocket" && !options.comboAttempt + && responseEffects.plaintextV2AgentMessageToolNames.size === 0 + ? options.nativeControl : undefined, + dispatchOverride: oauthDispatch(request), + providerName: route.providerName, + modelId: route.modelId, + onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), + beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider) + ? createCodexReserveDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined, + }), + codex401ReplayKind === "stored" ? options.onStoredPool401ReplayDispatched : undefined, + ); + // Routed through the shared helper so an ambiguous reset on THIS leg answers with the + // same refusal every other leg gives. A bare fetch here rejected instead, and the + // caller's transport-failure path turns a rejection into a client-retryable 502 -- + // which invites the whole turn to be sent again, on a leg whose first send may already + // have run it. + upstreamResponse = await fetchWithTransientRetry( + recovery => { + transportState.noteRoutedAttemptSend(passthroughEstimate, recovery ?? "oauth-401"); + return fetchWithHeaderTimeout( + request.url, + applyUpstreamRecoveryInit({ + method: request.method, + headers: request.headers, + body: request.body, + }, recovery), + upstream.signal, + connectMs, + parsed.stream, + oauthReplayExecutor, + route.provider.authMode === "forward", + ).then(adoptObservedResponse); + }, + { abortSignal: upstream.signal, label: safeHostLabel(request.url), + attempts: remainingTransientSendBudget(transientSendAttempts()), + onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + ); } catch (err) { return transportFailureResponse(err); } finally { @@ -1182,7 +1239,8 @@ export async function preparePassthroughExchange( route.provider.authMode === "forward") .then(adoptObservedResponse); }, - { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: noteTransientSends }, + { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), + onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1314,7 +1372,8 @@ export async function preparePassthroughExchange( route.provider.authMode === "forward") .then(adoptObservedResponse); }, - { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: noteTransientSends }, + { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), + onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1571,6 +1630,91 @@ export async function preparePassthroughExchange( continue passthroughRecovery; } } + + // The post-header row of the same table. A native SSE body can die after the head with the + // caller having observed nothing, which is the identical question the pre-header helper + // answers -- and the identical grant, because both claim from this request's one allowance. + const streamRecoveryContentType = upstreamResponse.headers.get("content-type")?.toLowerCase() ?? ""; + const protocolRecoveryCandidate = upstreamResponse.ok + && !!upstreamResponse.body + && !isNonReplayableResponse(upstreamResponse) + // The WebSocket transport settles its own ambiguous failures and marks them + // non-replayable; re-reading its body here would be a second owner of one exchange. + && !isCodexWsUpstreamResponse(upstreamResponse) + // A downstream WebSocket turn that fell back to HTTP must relay response.created + // immediately so the client can address the turn and receive explicit control refusal. + // This preflight retains that event until output commits, so the two contracts cannot + // share one body owner. + && !(options.nativeControl && options.inboundTransport === "websocket") + && ambiguousResend() !== undefined + && remainingTransientSendBudget(transientSendAttempts()) > 0 + && (streamRecoveryContentType.includes("text/event-stream") || (!streamRecoveryContentType && parsed.stream)); + if (protocolRecoveryCandidate) { + upstreamResponse = deferProtocolSafeResetRecovery( + upstreamResponse, + { model: logCtx.model, provider: logCtx.provider }, + (error, stage) => refetchAfterProtocolSafeReset( + (signal = upstream.signal) => fetchWithHeaderTimeout( + request.url, + applyUpstreamRecoveryInit({ + method: request.method, + headers: request.headers, + body: request.body, + }, "connection-reset"), + signal, + connectMs, + true, + providerFetch(route.provider, options.codexWsRuntimeIdentity, { + // A replacement HTTP body must not open a fresh WebSocket exchange: the turn it + // replaces was an HTTP stream, and a WS create frame is a different send. + httpOnly: true, + providerName: route.providerName, + modelId: route.modelId, + dispatchOverride: oauthDispatch(request), + beforeDispatch: headers => { + if (signal.aborted) throw signal.reason; + if (!transportState.selectionIsCurrent(transportState.requestBindings.get(request))) { + throw new Error("Credential selection changed before pre-output stream recovery"); + } + if (isCanonicalOpenAiForwardProvider(route.provider)) { + createCodexReserveDispatchGuard( + admissionState.authCtx, + options.codexAuthPolicy ?? config, + route.modelId, + options.admission, + options.visionDescribeTerminal === true, + )?.(headers); + } + // Recorded with the kind the gate derived its cause from, at the moment the + // send actually leaves. One authorisation, one recorded reason, one send. + transportState.noteRoutedAttemptSend(passthroughEstimate, "connection-reset"); + // Charged to the SAME request counter every other send goes through. The + // replacement is bought here rather than by a nested retry helper, so there is + // one charge for one send and no per-layer counter to reconcile. + noteTransientSends(1); + }, + }), + route.provider.authMode === "forward", + ).then(adoptObservedResponse), + error, + { + abortSignal: upstream.signal, + label: safeHostLabel(request.url), + attempts: remainingTransientSendBudget(transientSendAttempts()), + // The whole decision, including the stage the preflight observed and the grant the + // pre-header helper shares. A committed stage refuses here without touching the + // allowance, which is what keeps a turn that already emitted output from draining + // the replacement a later ambiguous reset would have been entitled to. + authorize: () => authorizeResendForRecovery(stage, "connection-reset", ambiguousResend()).allowed, + acceptResponse: candidate => { + const type = candidate.headers.get("content-type")?.toLowerCase() ?? ""; + return type.includes("text/event-stream") || (!type && parsed.stream); + }, + }, + ), + { allowMissingContentType: !streamRecoveryContentType && parsed.stream }, + ); + } break; } diff --git a/src/server/responses/reset-replay.ts b/src/server/responses/reset-replay.ts new file mode 100644 index 00000000000..4ee7e607ae8 --- /dev/null +++ b/src/server/responses/reset-replay.ts @@ -0,0 +1,104 @@ +/** + * The operator opt-in that overrides the stage table's refusal, and the one request-wide + * allowance both ambiguous stages claim from. + * + * `resendPermission` answers `refused-ambiguous` for a native Responses send that died with + * the caller having observed nothing -- before the response head, or after it while the SSE + * body carried only control events. This module is the one place that answer is overridden, + * and it is narrow on three axes at once: the provider has to opt in, the request has to be + * one whose second send cannot do more than run the same inference again, and the whole + * logical request gets a fixed number of replacements no matter how many legs ask. + * + * The judgment is made on the inbound body the client sent, which is already parsed. It is + * conservative for the outbound request: the proxy expands `previous_response_id` and lowers + * hosted tools into client execution, so every hazard that reaches the wire was visible here, + * and a hazard visible here may already have been removed. A cheap fail-closed answer beats + * re-parsing a multi-megabyte outbound body on every send. + */ +import type { OcxProviderConfig } from "../../types"; +import { resetReplayPolicyFor } from "../../providers/key-failover"; +import type { AmbiguousResendAllowance } from "../../lib/request-resend-gate"; + +/** Input items a client owns end to end: replaying them re-runs nothing but the model. */ +const CLIENT_INPUT_ITEM_TYPES: ReadonlySet = new Set([ + "message", "reasoning", "compaction", + "function_call", "function_call_output", + "custom_tool_call", "custom_tool_call_output", + "tool_search_call", +]); +const MESSAGE_ROLES: ReadonlySet = new Set(["user", "assistant", "system", "developer"]); +/** Bounded traversal: a catalog is operator data, not a reason to walk forever. */ +const MAX_TOOL_ENTRIES = 4096; +const MAX_TOOL_DEPTH = 4; + +function record(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * True when every tool in the catalog is executed by the client. Hosted tools (`web_search`, + * `mcp`, `code_interpreter`, ...) run on the origin during the turn, so an unknown or hosted + * type fails the whole catalog rather than being skipped: a tool this proxy does not + * recognise is a tool it cannot vouch for. + */ +function clientExecutedTools(tools: unknown, budget: { remaining: number }, depth = 0): boolean { + if (!Array.isArray(tools) || depth > MAX_TOOL_DEPTH) return false; + return tools.every(tool => { + budget.remaining -= 1; + if (budget.remaining < 0 || !record(tool)) return false; + if (tool.type === "function" || tool.type === "custom") return true; + if (tool.type === "tool_search") return tool.execution === "client"; + return tool.type === "namespace" && typeof tool.name === "string" + && clientExecutedTools(tool.tools, budget, depth + 1); + }); +} + +/** + * A Responses body whose second send can only repeat the inference: nothing stored, no + * server-side continuation state, complete input, and only client-executed tools. Deferred + * tool declarations inside `input` are checked by the same rule as the root catalog, so a + * hosted tool cannot ride in through `additional_tools` or a `tool_search_output`. + */ +export function selfContainedResponsesBody(body: unknown): boolean { + if (!record(body)) return false; + if (body.store !== false || body.background === true) return false; + if (body.previous_response_id != null || body.conversation != null || Object.hasOwn(body, "stream_id")) return false; + const input = body.input; + if (typeof input !== "string" && !Array.isArray(input)) return false; + const budget = { remaining: MAX_TOOL_ENTRIES }; + if (body.tools !== undefined && !clientExecutedTools(body.tools, budget)) return false; + if (typeof input === "string") return true; + return input.every(item => { + if (!record(item)) return false; + if (item.type === "additional_tools" || item.type === "tool_search_output") { + return clientExecutedTools(item.tools, budget); + } + if (item.type === undefined) return typeof item.role === "string" && MESSAGE_ROLES.has(item.role); + return typeof item.type === "string" && CLIENT_INPUT_ITEM_TYPES.has(item.type); + }); +} + +/** + * The allowance for ONE logical request, or nothing when the provider did not opt in. + * + * Built once per request and handed to every leg. `claim` spends the request's counter, which + * lives on the execution budget and is therefore shared with a combo child's derived scope -- + * that sharing is the reason the pre-header helper takes a callback instead of a number. + * + * The body judgment is carried rather than applied here, because the gate has to be able to + * say WHY it refused: "the operator granted nothing" and "this request cannot be replayed" are + * different operator problems, and folding them together is what made the old refusal a single + * undifferentiated no. + */ +export function ambiguousResendAllowanceFor( + provider: Pick, + inboundBody: unknown, + claim: (limit: number) => boolean, +): AmbiguousResendAllowance | undefined { + const policy = resetReplayPolicyFor(provider); + if (policy === null) return undefined; + return { + selfContained: selfContainedResponsesBody(inboundBody), + claim: () => claim(policy.replacements), + }; +} diff --git a/src/types.ts b/src/types.ts index 747fc17c756..a779713dcc9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -106,6 +106,7 @@ export type { ResponsesItemIdRepairConfig, RateLimitRetryPolicy, TransientRetryPolicy, + ResetReplayPolicy, ProviderWebSearchBridgeBackend, ProviderWebSearchBridgeConfig, ProviderCostOverlay, diff --git a/src/types/provider.ts b/src/types/provider.ts index 5e0bd79b583..3dbf793d9fd 100644 --- a/src/types/provider.ts +++ b/src/types/provider.ts @@ -67,6 +67,31 @@ export interface TransientRetryPolicy { attempts?: number; } +/** + * Opt-in replacement of a native Responses send whose upstream connection closed while the + * caller had observed nothing (`providers..retryOnReset`). + * + * Covers both ambiguous stages the proxy can be in: no response head at all, and a head whose + * SSE body carried only control events. Disabled unless the object is present; a bare `{}` + * opts in with defaults. Only a request the proxy can judge self-contained is ever replaced; + * see `src/server/responses/reset-replay.ts`. The replacement inference may still be billed if + * the origin had already started the first one, which is what makes this opt-in rather than + * default. + */ +export interface ResetReplayPolicy { + /** Master switch. Presence of the object also enables the policy (default true). */ + enabled?: boolean; + /** + * Replacement sends one LOGICAL request may make, across every leg and every combo child + * (1..2, default 1). + * + * Not a per-leg retry count and not a send budget. A request that resets before the head and + * again after it draws on this one number, and each replacement still has to fit inside the + * send allowance the leg already had. + */ + replacements?: number; +} + /** * Same-target 429 wait-and-retry policy (`providers..retryOn429`). When present and not * explicitly disabled, the proxy waits and replays the identical request on the same key before @@ -914,6 +939,12 @@ export interface OcxProviderConfig { * with defaults. Key-auth `openai-chat` only. */ transientRetryOn5xx?: TransientRetryPolicy; + /** + * Opt-in replacement of a native Responses send that died while the caller had observed + * nothing (`providers..retryOnReset`). Disabled unless present; a bare `{}` opts in + * with defaults. Native Responses sends only, and only for self-contained requests. + */ + retryOnReset?: ResetReplayPolicy; /** * Model ids whose OpenAI-compatible chat endpoint accepts `reasoning_split: true` and returns * thinking separately in `reasoning_content` / `reasoning_details` instead of visible content. diff --git a/tests/lib/upstream-retry.test.ts b/tests/lib/upstream-retry.test.ts index e8d09b309e3..f55d8ecf300 100644 --- a/tests/lib/upstream-retry.test.ts +++ b/tests/lib/upstream-retry.test.ts @@ -509,6 +509,101 @@ describe("ambiguous reset safety", () => { }); }); +describe("operator-granted replacement of an ambiguous reset", () => { + test("no claim callback keeps the refusal and never sends again", async () => { + const mock = mockDoFetch([bunResetError(), new Response("duplicate")]); + const response = await fetchWithResetRetry(mock.doFetch, { attempts: 3 }); + expect(response.status).toBe(429); + expect(mock.calls).toHaveLength(1); + }); + + test("a granted claim buys exactly one more send and is asked exactly once", async () => { + silenceWarn(); + const reports: number[] = []; + let asked = 0; + const mock = mockDoFetch([bunResetError(), new Response("ok")]); + const response = await fetchWithResetRetry(mock.doFetch, { + attempts: 3, + onSendsConsumed: count => reports.push(count), + claimAmbiguousResend: () => { asked += 1; return asked === 1; }, + }); + expect(await response.text()).toBe("ok"); + expect(mock.calls).toHaveLength(2); + expect(asked).toBe(1); + expect(reports).toEqual([1, 1]); + }); + + test("a spent grant settles as the refusal rather than sending again", async () => { + silenceWarn(); + const mock = mockDoFetch([bunResetError(), bunResetError(), new Response("duplicate")]); + const response = await fetchWithResetRetry(mock.doFetch, { + attempts: 3, + // The shape a request-wide allowance of one produces on its second question. + claimAmbiguousResend: (() => { let left = 1; return () => left-- > 0; })(), + }); + expect(response.status).toBe(429); + expect(isNonReplayableResponse(response)).toBe(true); + expect((await response.json()).error.code).toBe(UPSTREAM_RESET_REPLAY_REFUSED_CODE); + expect(mock.calls).toHaveLength(2); + }); + + test("the grant never widens the send budget it was given", async () => { + const mock = mockDoFetch([bunResetError(), new Response("duplicate")]); + let asked = 0; + const response = await fetchWithResetRetry(mock.doFetch, { + attempts: 1, + claimAmbiguousResend: () => { asked += 1; return true; }, + }); + expect(response.status).toBe(429); + expect(mock.calls).toHaveLength(1); + // Asking would have spent the request's one replacement on a send there was no room for. + expect(asked).toBe(0); + }); + + test("a replay-safe operation never consults the grant", async () => { + silenceWarn(); + let asked = 0; + const mock = mockDoFetch([bunResetError(), new Response("ok")]); + const response = await fetchWithResetRetry(mock.doFetch, { + attempts: 3, replaySafe: true, claimAmbiguousResend: () => { asked += 1; return true; }, + }); + expect(await response.text()).toBe("ok"); + expect(asked).toBe(0); + }); + + test("a non-reset failure after a replacement settles as the refusal, not a rejection", async () => { + silenceWarn(); + // The hazard the refusal exists for: a thrown transport error here becomes a 502 at the + // caller, and a 502 is what the Codex client retries -- so the turn whose first send may + // already have run would be sent again, four more times. + const mock = mockDoFetch([bunResetError(), new Error("upstream fetch failed")]); + const response = await fetchWithResetRetry(mock.doFetch, { + attempts: 3, claimAmbiguousResend: () => true, + }); + expect(response.status).toBe(429); + expect(isNonReplayableResponse(response)).toBe(true); + expect((await response.json()).error.code).toBe(UPSTREAM_RESET_REPLAY_REFUSED_CODE); + expect(mock.calls).toHaveLength(2); + }); + + test("the transient layer carries the grant into its inner reset layer", async () => { + silenceWarn(); + const reports: number[] = []; + const mock = mockDoFetch([ + new Response("busy", { status: 503 }), bunResetError(), new Response("ok"), + ]); + const response = await fetchWithTransientRetry(mock.doFetch, { + attempts: 3, + onSendsConsumed: count => reports.push(count), + claimAmbiguousResend: () => true, + }); + expect(await response.text()).toBe("ok"); + expect(mock.calls).toHaveLength(3); + // One report, from the one layer that owns the budget: three sends, counted once each. + expect(reports).toEqual([3]); + }); +}); + describe("ambiguous reset safety through error formatting", () => { test("every terminal code survives formatting without advertising Retry-After", async () => { for (const code of ["upstream_no_response", "upstream_closed_before_response", "upstream_reset_replay_refused"]) { diff --git a/tests/responses/responses-reset-replay.test.ts b/tests/responses/responses-reset-replay.test.ts new file mode 100644 index 00000000000..81b252738d6 --- /dev/null +++ b/tests/responses/responses-reset-replay.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test } from "bun:test"; +import { + ambiguousResendAllowanceFor, + selfContainedResponsesBody, +} from "../../src/server/responses/reset-replay"; +import { authorizeResendForRecovery } from "../../src/lib/request-resend-gate"; + +const clientTurn = { + store: false, + input: [ + { type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }, + { type: "function_call", name: "read", call_id: "c1", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: "ok" }, + ], + tools: [{ type: "function", name: "read" }], +}; + +describe("selfContainedResponsesBody", () => { + test("accepts a turn whose second send can only repeat the inference", () => { + expect(selfContainedResponsesBody(clientTurn)).toBe(true); + expect(selfContainedResponsesBody({ store: false, input: "plain prompt" })).toBe(true); + expect(selfContainedResponsesBody({ + store: false, + input: [{ role: "user", content: "hi" }], + })).toBe(true); + }); + + test("refuses anything that leaves state behind or continues someone else's turn", () => { + for (const override of [ + { store: true }, + { store: undefined }, + { background: true }, + { previous_response_id: "resp_1" }, + { conversation: "conv_1" }, + { stream_id: undefined }, + { input: undefined }, + { input: { not: "a list" } }, + ]) { + expect(selfContainedResponsesBody({ ...clientTurn, ...override })).toBe(false); + } + expect(selfContainedResponsesBody(null)).toBe(false); + expect(selfContainedResponsesBody([clientTurn])).toBe(false); + }); + + test("refuses a catalog carrying anything the origin would execute", () => { + for (const tools of [ + [{ type: "web_search" }], + [{ type: "function", name: "read" }, { type: "mcp", server_label: "s" }], + [{ type: "tool_search", execution: "server" }], + [{ type: "namespace", name: "ns", tools: [{ type: "code_interpreter" }] }], + [{ type: "namespace" }], + "not a list", + ]) { + expect(selfContainedResponsesBody({ ...clientTurn, tools })).toBe(false); + } + expect(selfContainedResponsesBody({ + ...clientTurn, + tools: [{ type: "namespace", name: "ns", tools: [{ type: "function", name: "read" }] }], + })).toBe(true); + }); + + test("a hosted tool cannot ride in through a deferred declaration", () => { + expect(selfContainedResponsesBody({ + ...clientTurn, + input: [...clientTurn.input, { type: "additional_tools", tools: [{ type: "web_search" }] }], + })).toBe(false); + expect(selfContainedResponsesBody({ + ...clientTurn, + input: [...clientTurn.input, { type: "tool_search_output", tools: [{ type: "function", name: "read" }] }], + })).toBe(true); + }); + + test("an input item type this proxy does not recognise is refused", () => { + expect(selfContainedResponsesBody({ + ...clientTurn, + input: [{ type: "image_generation_call", id: "ig1" }], + })).toBe(false); + }); +}); + +describe("ambiguousResendAllowanceFor", () => { + test("absent or disabled policy grants nothing", () => { + const claim = () => true; + expect(ambiguousResendAllowanceFor({}, () => true, claim)).toBeUndefined(); + expect(ambiguousResendAllowanceFor({ retryOnReset: { enabled: false } }, () => true, claim)).toBeUndefined(); + }); + + test("a bare opt-in spends one replacement for the whole request", () => { + const limits: number[] = []; + let left = 1; + const grant = ambiguousResendAllowanceFor({ retryOnReset: {} }, () => true, limit => { + limits.push(limit); + return left-- > 0; + })!; + expect(grant.selfContained).toBe(true); + expect(grant.claim()).toBe(true); + expect(grant.claim()).toBe(false); + // Both questions were asked at the same ceiling, against the one request-wide counter. + expect(limits).toEqual([1, 1]); + }); + + test("the body judgment is lazy and reaches the gate as a refusal reason", () => { + let judged = 0; + const grant = ambiguousResendAllowanceFor({ retryOnReset: {} }, () => { judged += 1; return false; }, () => true)!; + expect(judged).toBe(0); + const decision = authorizeResendForRecovery("pre-header", "connection-reset", grant); + expect(decision.allowed).toBe(false); + if (!decision.allowed) expect(decision.refusal).toBe("ambiguous-request-not-replayable"); + expect(judged).toBe(1); + }); + + test("a provider that opted in still funds only what the operator asked for", () => { + const seen: number[] = []; + const grant = ambiguousResendAllowanceFor( + { retryOnReset: { replacements: 2 } }, + () => true, + limit => { seen.push(limit); return true; }, + )!; + expect(grant.claim()).toBe(true); + expect(seen).toEqual([2]); + }); +}); diff --git a/tests/routing/combo-stream-preflight.test.ts b/tests/routing/combo-stream-preflight.test.ts index 798cce0ef22..36aa183538a 100644 --- a/tests/routing/combo-stream-preflight.test.ts +++ b/tests/routing/combo-stream-preflight.test.ts @@ -3,6 +3,7 @@ import { comboStreamPayloadCommitsOutput, preflightComboStreamResponse, } from "../../src/server/responses/combo-stream-preflight"; +import { stageCommitment } from "../../src/lib/request-failure-model"; import type { RequestLogContext } from "../../src/server/request-log"; import { MAX_CLIENT_SSE_FRAME_BYTES } from "../../src/server/sse-frame-buffer"; @@ -538,7 +539,7 @@ describe("combo stream preflight", () => { expect(source.cancelSpy()!.mock.calls).toHaveLength(0); }); - test("replayReadErrors accepts a reconstructed prefix and the same reader.read error", async () => { + test("replayReadErrors returns a reconstructed prefix, the same read error, and the observed stage", async () => { const readError = new Error("preflight-read-reset"); const source = prefixThenReadError(createdPrefix, readError); const result = await preflightComboStreamResponse( @@ -547,7 +548,14 @@ describe("combo stream preflight", () => { undefined, { replayReadErrors: true }, ); - expect(result.kind).toBe("accepted"); + expect(result.kind).toBe("read-error"); + if (result.kind === "read-error") { + expect(result.error).toBe(readError); + // response.created and nothing else: the failure model puts that in the prelude, and a + // prelude is a stage at which the caller has observed nothing. + expect(result.stage).toBe("protocol-prelude"); + expect(stageCommitment(result.stage)).toBe("nothing-observed"); + } expect(source.cancelSpy()).toBeDefined(); expect(source.cancelSpy()!.mock.calls).toHaveLength(0); const reader = result.response.body!.getReader(); @@ -559,4 +567,44 @@ describe("combo stream preflight", () => { expect(source.cancelSpy()!.mock.calls).toHaveLength(0); }); + test("a read error before any event is headers-only, and after output is committed", async () => { + const readError = new Error("preflight-read-reset"); + const bare = await preflightComboStreamResponse( + prefixThenReadError(new TextEncoder().encode(""), readError).response, + { model: "m1", provider: "a" }, + undefined, + { replayReadErrors: true }, + ); + expect(bare.kind).toBe("read-error"); + if (bare.kind === "read-error") { + expect(bare.stage).toBe("headers-only"); + expect(stageCommitment(bare.stage)).toBe("nothing-observed"); + } + + const outputPrefix = new TextEncoder().encode(`data: ${JSON.stringify({ + type: "response.output_text.delta", delta: "hi", + })}\n\n`); + const committed = await preflightComboStreamResponse( + prefixThenReadError(outputPrefix, readError).response, + { model: "m1", provider: "a" }, + undefined, + { replayReadErrors: true }, + ); + expect(committed.kind).toBe("read-error"); + if (committed.kind === "read-error") { + expect(committed.stage).toBe("semantic-output"); + expect(stageCommitment(committed.stage)).not.toBe("nothing-observed"); + } + }); + + test("a response.created carrying output is not a prelude", () => { + expect(comboStreamPayloadCommitsOutput({ + type: "response.created", response: { id: "r1", output: [] }, + })).toBe(false); + expect(comboStreamPayloadCommitsOutput({ + type: "response.created", + response: { id: "r1", output: [{ type: "message", role: "assistant" }] }, + })).toBe(true); + }); + }); diff --git a/tests/server/management-provider-reset-replay.test.ts b/tests/server/management-provider-reset-replay.test.ts new file mode 100644 index 00000000000..27f7f2c2df5 --- /dev/null +++ b/tests/server/management-provider-reset-replay.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { providerManagementConfigError } from "../../src/server/auth-cors"; + +// Lives apart from management-provider-validation.test.ts because that file sits at its +// file-size ratchet cap; the helpers it needs are small enough to repeat here. +const canonicalDirect = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", +} as const; + +test("provider management validates retryOnReset bounds and unknown keys", () => { + const base = { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1" }; + expect(providerManagementConfigError("custom", { ...base, retryOnReset: {} })).toBeNull(); + expect(providerManagementConfigError("custom", { ...base, retryOnReset: { enabled: true, replacements: 2 } })).toBeNull(); + expect(providerManagementConfigError("custom", { ...base, retryOnReset: { replacements: 0 } })) + .toContain("retryOnReset.replacements is invalid"); + expect(providerManagementConfigError("custom", { ...base, retryOnReset: { replacements: 3 } })) + .toContain("retryOnReset.replacements is invalid"); + // The old field name from the pre-rework branch is rejected rather than silently ignored: + // `attempts` was a per-leg send count and this is a per-request replacement count. + expect(providerManagementConfigError("custom", { ...base, retryOnReset: { attempts: 2 } })) + .toContain("retryOnReset has unrecognized field"); + expect(providerManagementConfigError("custom", { ...base, retryOnReset: true })) + .toContain("retryOnReset is invalid"); + // The canonical openai row is the main target of this policy, and a full-object write + // compares it against the seed with an exact key match: the field must be admitted + // there like requestPacing is, while its value is still validated. + expect(providerManagementConfigError("openai", { ...canonicalDirect, retryOnReset: { replacements: 2 } })).toBeNull(); + expect(providerManagementConfigError("openai", { ...canonicalDirect, retryOnReset: { replacements: 3 } })) + .toContain("retryOnReset.replacements is invalid"); + // A secret-shaped unknown field name and a secret-shaped provider name are both redacted. + const secretError = providerManagementConfigError("custom", { ...base, retryOnReset: { "sk-super-secret-9876": true } })!; + expect(secretError).toContain("retryOnReset has unrecognized field"); + expect(secretError).not.toContain("sk-super-secret-9876"); + const secretNameError = providerManagementConfigError("sk-super-secret-9876", { ...base, retryOnReset: { replacements: 0 } })!; + expect(secretNameError).toContain("retryOnReset.replacements is invalid"); + expect(secretNameError).not.toContain("sk-super-secret-9876"); + expect(secretNameError).toContain("[REDACTED]"); +}); + +test("retryOn429 keeps its own field name after the formatter was shared", () => { + // The two validators now run through one body. A shared formatter that reported the wrong + // field name would send an operator to the wrong key in their config. + const base = { adapter: "openai-responses", baseUrl: "https://api.openai.com/v1" }; + expect(providerManagementConfigError("custom", { ...base, retryOn429: { attempts: 0 } })) + .toContain("retryOn429.attempts is invalid"); + expect(providerManagementConfigError("custom", { ...base, retryOn429: { nope: 1 } })) + .toContain("retryOn429 has unrecognized field"); +}); From 099a7c99daf265ef96282dc1cb31769a702e0513 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 21:48:52 +0900 Subject: [PATCH 3/9] docs: record the ambiguous-resend gate and what the R4 remainders reach structure/transports/responses.md said a pre-header reset is always the terminal refusal and that only a replay-safe operation opts into reset retries. Both sentences are now wrong in the same place, so the reset-retry section gains the gate beside it, the combo streaming boundary says that native post-header recovery shares the same reader and reports a stage, and the core-module table gains reset-replay.ts and the grant it hands to request-send-budget.ts. docs-site documents retryOnReset as a provider field in the English source and in all seven locale tables, and the server reference paragraph that explains the 429 refusal now says how an operator opts out of it and what the grant covers. The devlog records the two remainders honestly. Neither #4191 nor #5180 is reached by this gate, and both investigations found something worth not losing: the WebSocket stage projection reports semantic-output for a failure carrying only response.created, and the #5180 symptom is a missing policy default plus a cooldown a single-key provider cannot currently write. --- .../040_r4_retry_rework.md | 53 +++++++++++++++++ .../050_r4_remainders.md | 59 +++++++++++++++++++ .../fr/reference/configuration/providers.md | 1 + .../ja/reference/configuration/providers.md | 1 + .../ko/reference/configuration/providers.md | 1 + .../docs/reference/configuration/providers.md | 1 + .../docs/reference/configuration/server.md | 11 ++++ .../ru/reference/configuration/providers.md | 1 + .../tr/reference/configuration/providers.md | 1 + .../reference/configuration/providers.md | 1 + .../reference/configuration/providers.md | 1 + structure/transports/responses.md | 39 +++++++++++- 12 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 devlog/_plan/260920_round2_followups/040_r4_retry_rework.md create mode 100644 devlog/_plan/260920_round2_followups/050_r4_remainders.md diff --git a/devlog/_plan/260920_round2_followups/040_r4_retry_rework.md b/devlog/_plan/260920_round2_followups/040_r4_retry_rework.md new file mode 100644 index 00000000000..6022b10626a --- /dev/null +++ b/devlog/_plan/260920_round2_followups/040_r4_retry_rework.md @@ -0,0 +1,53 @@ +# R4 — #4942 and #4989 as one ambiguous-resend gate + +Status: OPEN. Branch `codex/260920-r4-retry-rework`, cut from `origin/dev` at `d6d87440b7`. + +## Why the two pull requests are one change + +#4942 (FredAmartey) replays a native Responses send whose connection died before any response +head, behind a per-provider opt-in. #4989 (lidge-jun) replaces a native Responses SSE stream that +died after the head while the body had carried only control events. Written apart they read as two +features. Against the stage table #5266 landed in `src/lib/request-failure-model.ts` they are one +row: a stage whose `stageCommitment` is `nothing-observed`, with a cause whose +`causeEvidence` is `unknown`. `resendPermission` answers `refused-ambiguous` for both, and +the module already names the only thing that may override it — "a narrowly scoped, explicitly +opted-in recovery that a maintainer reasoned about and bounded". + +Two overrides is one too many. #4942 spreads `replayResets: 2` into every dispatch leg of the +request and #4989 takes `Math.min(1, remaining)` of the transient budget at the stream boundary, +so one logical request that reset before the head and again after it would buy a replacement on +each. The rework gives the override a single per-request allowance and makes both stages claim +from it. + +## Shape + +- `src/lib/request-resend-gate.ts` — the one gate. Pure table lookup for the stages the caller + already observed something at, plus the operator override for the ambiguous row. It never + restates the table: stage, cause, permission and send class all come from + `request-failure-model.ts`, and the cause comes from the `AttemptRecoveryKind` that will be + recorded, so the reason in the log and the send it authorised cannot disagree. +- `src/lib/request-execution-budget.ts` — the allowance lives on the shared send ledger, which is + what a combo child inherits through `deriveRequestExecutionBudget`. Parent and child therefore + cannot each hold one. +- `src/server/responses/reset-replay.ts` — the provider opt-in and the body judgment from #4942, + plus the per-request authority both call sites use. +- `src/lib/upstream-retry.ts` — the pre-header claim, as a callback rather than a number. +- `src/server/responses/combo-stream-preflight.ts` — the preflight reports the stage it observed + instead of a boolean, so the gate rather than the preflight decides. + +## Stage classification at the stream boundary + +#4989 gated on `responseCreated && !outputCommitted && !terminal`. That is `protocol-prelude`. +A read error before any parsed event is `headers-only`, which the table gives the same +commitment and therefore the same answer; the rework admits it rather than refusing a row the +table permits. Everything else the preflight can see is `semantic-output` or `terminal`, and +those refuse regardless of cause. + +## In scope from the remainders + +#4191 and #5180 only to the extent the resend gate reaches them. Recorded in 050. + +## Verification + +Static review plus exact-head hosted CI. Local suites, individual tests, typecheck, build, +install and live `ocx` execution are NOT RUN by lane policy. diff --git a/devlog/_plan/260920_round2_followups/050_r4_remainders.md b/devlog/_plan/260920_round2_followups/050_r4_remainders.md new file mode 100644 index 00000000000..4c826a9d172 --- /dev/null +++ b/devlog/_plan/260920_round2_followups/050_r4_remainders.md @@ -0,0 +1,59 @@ +# R4 — what the remainders reach, and what they do not + +The lane brief put #4191 and #5180 in R4 "as far as the rework reaches". This records where that +line actually fell, with the evidence, so the next lane starts from a finding rather than a +re-investigation. + +## #4191 — reached: nothing. Found: a wrong stage in the shared vocabulary + +The resend gate does not consult the WebSocket projection. The post-header path excludes a +`isCodexWsUpstreamResponse` body on purpose: the WS transport settles its own ambiguous +failures and marks them non-replayable, and a second reader of one exchange is a defect, not a +recovery. So the durable threading the round-2 plan names is untouched here. + +The investigation did surface a real defect in the projection itself. +`classifyCodexWsFailure` in `src/server/responses/codex-ws-wire.ts` returns +`after-response-started` — which `CODEX_WS_FAILURE_PROJECTION` maps to `semantic-output` — +as soon as `relayedEvents > 0`. But `src/server/responses/codex-ws-exchange.ts` increments +`relayedEvents` for every non-metadata Responses event, and `response.created` is one: +`controlFrame` is set only when the metadata channel consumes the frame, not for lifecycle +events. `src/lib/request-failure-model.ts` puts `response.created` in `protocol-prelude` +and requires an output-bearing event for `semantic-output`. A WS failure carrying only a +created event therefore projects as committed output today. + +It is left here rather than fixed because the fix needs a counter the classifier does not have, +and `CodexWsStageRecord` is derived from `CodexWsFailureStage` by `Omit`, so adding one +lands in a persisted record whose read-back whitelist in `src/usage/log.ts` would reject every +row written before it. Adding the counter and `Omit`-ing it from the durable twin avoids that, +but the output-bearing predicate lives in `combo-stream-preflight.ts` and restating it in the +exchange is the class of duplication this round already paid for three times. It belongs with +the lane that threads `failureStage` / `failureCause` into the record, where both halves can +be written once. + +The SSE fallback the issue asks for stays out regardless. After `ws.send()` returns, a +fallback is a second physical send on another transport, which is a transport decision with its +own duplicate-inference policy — not a retry-gate change. + +## #5180 — reached: nothing. The symptom is upstream of this gate + +The reported failure is a key-auth `openai-chat` provider answering a bare 429. Traced on +current `dev`: `rateLimitRetryPolicyFor` returns null for every provider except the +OpenCode Go destination, so the same-target wait never runs; key rotation needs a pool of at +least two; and `fetchWithResetRetry` returns the first received HTTP response without +consulting its status. One send, 429 returned, which is exactly what the reporter saw. +`Retry-After` is forwarded to the client — synthesized as `2` for a bare retryable 429 by +`src/lib/retry-after.ts` — but the proxy never waits on it itself. + +None of that is an ambiguous-resend question: a received 429 is `headers-only` with cause +`rate-limit`, which the stage table already answers `permitted` and funds from the +`transient` class. It needs no grant and no override. What it needs is a policy default and a +process-wide cooldown that a single-key provider can write, and `keyCooldowns` cannot be +reused unchanged because both its identity and its write path require a multi-key pool. + +One adjacent accounting gap is worth recording for whoever takes it. On the generic adapter +path, `prepareAdapterExchange` passes `attempts` and `onSendsConsumed` to its retry helper +only when `transientRetryOn5xx` is configured. An unconfigured provider's initial send is +therefore recorded in the attempt log but never charged to the request-wide send counter. It is +bounded today — without `replaySafe` the reset helper makes exactly one send — so it is an +under-count rather than an amplification, and widening it without a suite to run is not a change +worth making blind. diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index e96e79f5f20..45ab2192c50 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -139,6 +139,7 @@ sauvegarde dont le contenu diffère, puis réécrit en identifiants sans préfix | `responsesSnapshotRepair?` | `boolean` | Réparation côté client désactivée par défaut pour les instantanés du cycle de vie des réponses clairsemés dans SSE et JSON. Remplit les métadonnées d'état canonique, de sortie et d'outil manquantes tandis que l'inspection brute et la persistance restent inchangées. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Fournisseurs à clé API uniquement (`authMode: "key"`). Nouvelle tentative facultative sur la même cible après un 429 : lorsque `retryOn429` est absent, la fonctionnalité est désactivée ; la présence d'un objet l'active, sauf avec `enabled: false`. Après un 429, le proxy attend selon `Retry-After` reçu en amont ou selon l'intervalle fixe, puis relit la requête à l'identique avec la même clé avant tout basculement de clé. Ce comportement couvre la boucle principale de récupération d'un tour textuel, le protocole de transfert Responses, le pont d'images et de vidéos, le service auxiliaire de recherche Web et les continuations du terminal. Seules les réponses HTTP 429 reçues avant le début de la diffusion peuvent être relues ; les transports `runTurn` personnalisés ne font pas partie de la boucle de nouvelle tentative HTTP. `attempts` compte les relectures avec la même clé après le premier 429, soit `attempts` + 1 envois au total, et constitue un budget commun à toute la requête, partagé entre la boucle principale de récupération, la continuation de la garde du terminal et les nouvelles tentatives du pont. L'épuisement de `attempts` arrête uniquement les relectures supplémentaires avec la même clé : le basculement normal de clé ou la gestion de l'erreur finale s'applique ensuite selon les cibles disponibles. Sur le protocole de transfert authentifié par clé, aucun basculement n'est possible ; le 429 final est donc renvoyé sans modification. Codex ne retente jamais lui-même une requête après un 429 : cette option constitue ainsi la seule protection pour les fournisseurs à clé unique. Valeurs par défaut : `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (chaque attente est plafonnée à `maxIntervalMs`, lui-même plafonné à 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Fournisseurs `openai-chat` et `openai-responses` authentifiés par clé uniquement. Les fournisseurs `authMode: "forward"` (le pool de comptes ChatGPT) ne lisent jamais cette option et conservent l'échelle par défaut. Nouvelle tentative facultative pour les états transitoires reçus en amont avant le début de la diffusion (500, 502, 503, 504, 520, 521, 522) : l'absence de l'option la désactive ; la présence d'un objet l'active, sauf avec `enabled: false`. Ce comportement couvre la requête Responses initiale, la continuation de la garde du terminal, le point de terminaison natif `/v1/chat/completions` et les réémissions liées à la récupération après un 429 ou à la récupération de compte. `attempts` représente le nombre TOTAL d'envois en amont autorisés pour une requête, premier envoi compris (de 1 à 10, valeur par défaut : 3). Il constitue un budget commun à la requête, partagé avec la récupération après une réinitialisation de connexion ; ainsi, `3` signifie qu'au plus trois requêtes réelles atteignent le fournisseur. Les attentes utilisent une temporisation exponentielle à base fixe de 400 ms, plafonnée à 5 s, et respectent `Retry-After`. Cette option est distincte de `retryOn429`, qui traite la limitation de débit ; les échecs en cours de diffusion ne sont jamais relus. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Fournisseurs `openai-responses` natifs uniquement, `authMode: "forward"` compris. Remplacement facultatif d'un envoi qui a échoué alors que l'appelant n'avait rien observé : l'absence de l'option la désactive ; la présence d'un objet l'active, sauf avec `enabled: false`. Couvre les deux étapes ambiguës — une connexion rompue avant tout en-tête de réponse, et un corps SSE rompu après l'en-tête alors qu'il ne portait que des événements de contrôle. Seule une requête autonome est remplacée : `store: false`, `input` complet, ni `previous_response_id`, ni `conversation`, ni `stream_id`, et uniquement des outils exécutés par le client. `replacements` est le nombre d'envois de remplacement qu'UNE requête logique peut effectuer, toutes étapes et tous enfants de combo confondus (de 1 à 2, valeur par défaut : 1). Ce n'est ni un nombre de tentatives par étape ni un budget d'envoi : un remplacement doit toujours tenir dans l'allocation d'envois dont l'étape disposait déjà. Une requête qui a déjà émis une sortie ou un appel d'outil n'est jamais remplacée, quelle que soit cette valeur. L'inférence de remplacement peut tout de même être facturée si l'origine avait déjà démarré la première, d'où la désactivation par défaut. | | `autoToolChoiceOnlyModels?` | `string[]` | Modèles dont `tool_choice` accepte uniquement `auto` ou `none` ; les choix forcés sont dévalorisés. | | `preserveReasoningContentModels?` | `string[]` | Modèles nécessitant un assistant préalable `reasoning_content` dans l'historique des discussions. | | `reasoningDetailsModels?` | `string[]` | Modèles dont le point de terminaison renvoie la réflexion sous forme de tableau structuré `reasoning_details` (MiniMax série M avec `reasoning_split`) ; les deltas de flux sont des instantanés cumulatifs comparés par préfixe, et la réflexion conservée est rejouée sous forme de tableau `reasoning_details` plutôt que de chaîne `reasoning_content`. | diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 6a2364e23c0..bac2502e0ec 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -131,6 +131,7 @@ account を削除しても mapping は保持され、同じ id を再追加す | `responsesSnapshotRepair?` | `boolean` | デフォルトで無効のクライアント向け修復です。SSE と JSON の Responses ライフサイクルで欠落した status、output、ツールメタデータを補完し、raw 検査と永続化は変更しません。 | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。`attempts` を使い切っても同一キーでの再送が止まるだけで、通常のキー フェイルオーバーまたは最終エラー処理が利用可能なターゲットに応じて続きます — キー認証の passthrough ワイヤにはフェイルオーバーがないため、使い切った 429 はそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | キー認証の `openai-chat` および `openai-responses` プロバイダーのみ。`authMode: "forward"` のプロバイダー（ChatGPT アカウントプール）はこのオプションを読まず、既定の再試行段数を維持します。ストリーム開始前に上流から返される一時的なステータス（500、502、503、504、520、521、522）に対するオプトインの再試行です。設定がなければ無効で、オブジェクトを指定すると `enabled: false` でない限り有効になります。最初の Responses リクエスト、ターミナルガード継続、ネイティブの `/v1/chat/completions`、および 429／アカウント回復時の再取得が対象です。`attempts` は最初の送信を含め、1 回のリクエストで許可される上流への送信総数です（1～10、デフォルトは 3）。接続リセット回復と共有するリクエスト単位の単一予算であるため、`3` を指定した場合、プロバイダーに到達する実リクエストは最大 3 回です。待機には 400 ms を基準とする固定式の指数バックオフを使用し、上限は 5 秒で、`Retry-After` に従います。レート制限を扱う `retryOn429` とは別の機能であり、ストリーム開始後の失敗は再送されません。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | ネイティブ `openai-responses` プロバイダー専用で、`authMode: "forward"` も含みます。呼び出し側が何も観測しないまま失敗した送信を、オプトインで置き換えます。設定がなければ無効で、オブジェクトを指定すると `enabled: false` でない限り有効になります。レスポンスヘッダーが届く前に接続が切れた場合と、ヘッダー後に SSE 本文が制御イベントだけを運んだまま切れた場合の両方が対象です。置き換えるのは自己完結したリクエストだけで、`store: false`、完全な `input`、`previous_response_id` / `conversation` / `stream_id` がないこと、クライアントが実行するツールのみ、が条件です。`replacements` は、すべてのレッグとすべてのコンボ子リクエストを合わせて 1 つの論理リクエストが行える置き換え送信の回数です（1..2、デフォルトは 1）。レッグ単位の再試行回数でも送信予算でもないため、置き換え送信もそのレッグがすでに持つ送信許容量に収まる必要があります。すでに出力やツール呼び出しを送ったリクエストは、この値に関わらず置き換えません。元の送信がすでに開始されていた場合は置き換えた推論も課金される可能性があるため、既定では無効です。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` が `auto` または `none` のみを受け入れるモデル。強制的な選択は格下げされます。 | | `preserveReasoningContentModels?` | `string[]` |チャット履歴に以前のアシスタント `reasoning_content` が必要なモデル。 | | `reasoningDetailsModels?` | `string[]` | thinking を構造化された `reasoning_details` 配列で返すモデル（`reasoning_split` 使用の MiniMax M シリーズ）。ストリーム差分は累積スナップショットとして prefix-diff され、保持された reasoning は `reasoning_content` 文字列ではなく `reasoning_details` 配列としてリプレイされます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 824a9826940..8fe624c51c9 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -131,6 +131,7 @@ managed map을 활성화하면 privacy-safe selector를 만들고, 이후 계정 | `responsesSnapshotRepair?` | `boolean` | 기본값이 꺼진 클라이언트용 복구입니다. SSE와 JSON의 Responses 수명 주기에서 누락된 status, output, 도구 메타데이터를 채우며 raw 검사와 영속화는 변경하지 않습니다. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. `attempts`를 모두 소진해도 동일 키 재전송만 중단되며, 이후에는 일반 키 장애 조치 또는 최종 오류 처리가 사용 가능한 대상에 따라 진행됩니다 — 키 인증 passthrough 와이어에는 장애 조치가 없으므로 소진된 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 키 인증 `openai-chat` 및 `openai-responses` 프로바이더 전용입니다. `authMode: "forward"` 프로바이더(ChatGPT 계정 풀)는 이 옵션을 읽지 않고 기본 재시도 단계를 유지합니다. 스트림 시작 전의 일시적인 업스트림 상태(500, 502, 503, 504, 520, 521, 522)를 선택적으로 재시도합니다. 이 옵션이 없으면 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 최초 Responses 요청, 터미널 가드 연속 요청, 네이티브 `/v1/chat/completions`, 429/계정 복구 재조회를 포함합니다. `attempts`는 최초 전송을 포함하여 요청 하나에 허용되는 업스트림 전송의 총횟수(1..10, 기본값 3)입니다. 연결 재설정 복구와 요청 단위 예산 하나를 공유하므로 `3`이면 실제로 프로바이더에 도달하는 요청은 최대 세 번입니다. 대기에는 400ms로 고정된 지수 백오프를 사용하고 상한은 5초이며 `Retry-After`를 따릅니다. 속도 제한을 처리하는 `retryOn429`와는 별개이며, 스트림 도중의 실패는 절대 재전송하지 않습니다. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 네이티브 `openai-responses` 프로바이더 전용이며 `authMode: "forward"`도 포함합니다. 호출자가 아무것도 관측하지 못한 채 실패한 전송을 선택적으로 대체합니다. 이 옵션이 없으면 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 응답 헤더가 오기 전에 연결이 끊어진 경우와, 헤더 이후 SSE 본문이 제어 이벤트만 실은 채 끊어진 경우를 모두 다룹니다. 자체 완결된 요청만 대체합니다. `store: false`, 완전한 `input`, `previous_response_id`·`conversation`·`stream_id` 없음, 클라이언트가 실행하는 도구만 해당합니다. `replacements`는 모든 구간과 모든 콤보 자식을 합쳐 논리 요청 하나가 만들 수 있는 대체 전송 횟수입니다(1..2, 기본값 1). 구간별 재시도 횟수도 전송 예산도 아니므로, 대체 전송도 해당 구간이 이미 가진 전송 허용량 안에 들어가야 합니다. 이미 출력이나 도구 호출을 내보낸 요청은 이 값과 무관하게 대체하지 않습니다. 원본 전송이 이미 시작됐다면 대체한 추론도 과금될 수 있어서 기본값은 꺼짐입니다. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. | | `reasoningDetailsModels?` | `string[]` | thinking을 구조화된 `reasoning_details` 배열로 반환하는 모델(`reasoning_split` 사용 MiniMax M 시리즈). 스트림 델타는 누적 스냅샷이라 prefix-diff로 처리하고, 보존된 reasoning은 `reasoning_content` 문자열 대신 `reasoning_details` 배열로 리플레이합니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 6f780ed4ed6..3cae671a466 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -212,6 +212,7 @@ Providers can expose a built-in shorthand, such as `agy` for `google-antigravity | `webSearchBridge?` | `{ enabled?: boolean; backend?: "ollama" \| "openai" \| "anthropic" \| "xai" \| "gemini" \| "exa"; maxSearches?: number; timeoutMs?: number; endpoint?: string }` | Key-auth `openai-responses` passthrough providers only. Off by default. Codex always declares the hosted `web_search` tool, and the passthrough relays it on the assumption the destination executes it. A gateway that does not run hosted search answers with a `function_call` named `web_search` that nothing runs, and the undeclared-tool guard ends the turn. With `enabled: true` and an explicit `backend` OpenCodex intercepts that call, runs the search itself, feeds the result back to the same upstream, and shows Codex a hosted `web_search_call` cell. Never armed for `authMode: "forward"` (ChatGPT already searches) or for a provider that executes hosted search upstream. `backend` is required; there is no implicit default and a missing credential for the named backend leaves the bridge disarmed rather than falling through to another paid search. `ollama` reuses this provider's own API key on `POST /api/web_search`, so the origin must be `https://ollama.com` unless the operator names `endpoint` explicitly. `openai` / `anthropic` / `xai` / `gemini` / `exa` reuse the matching sidecar executor and that executor's own credential (`webSearchSidecar.exaApiKey` for Exa). The search model comes from `webSearchSidecar.model` only when `webSearchSidecar.backend` resolves to the same backend this bridge names; otherwise the bridge runs that backend's own default, because a model chosen for one vendor is rejected by another. An unset `webSearchSidecar.backend` resolves to `openai`, so an unset-backend model reaches an `openai` bridge and no other. There is no per-provider bridge model override. Streaming turns only. A turn that mixes `web_search` with another client tool call still fails closed rather than dropping the client's call. Assistant text such as XML-like `` prose is not executed. Defaults: `maxSearches: 3` (1..10), `timeoutMs: 60000` (1000..600000). | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries. Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Key-auth `openai-chat` and `openai-responses` providers only. `authMode: "forward"` providers (the ChatGPT account pool) never read this option and keep the default ladder. Opt-in retry for pre-stream transient upstream statuses (500, 502, 503, 504, 520, 521, 522): absent means off, object presence enables it unless `enabled: false`. Covers the initial Responses request, the Responses passthrough lane and each of its recovery legs (OAuth-401 replay, same-target 429 replay, validated rebuild), the terminal-guard continuation, and native `/v1/chat/completions`. `attempts` is the TOTAL number of upstream sends allowed for one request including the first (1..10, default 3) — it is one budget shared with connection-reset recovery, so `3` means at most three real requests reach the provider. On the Responses passthrough lane the configured value is additionally intersected with the request-wide send allowance, so a value below that allowance narrows the ladder exactly while a value above it does not raise the bound. Waits use a fixed 400 ms exponential backoff capped at 5 s and honor `Retry-After`. Separate from `retryOn429`, which handles rate limiting; mid-stream failures are never replayed. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Native `openai-responses` providers, including `authMode: "forward"`. Opt-in replacement of a send that failed while the caller had observed nothing: absent means off, object presence enables it unless `enabled: false`. Covers both ambiguous stages — a connection that died before any response header, and an SSE body that died after the header while carrying only control events. Only a self-contained request is ever replaced: `store: false`, complete `input`, no `previous_response_id`, `conversation` or `stream_id`, and only client-executed tools. `replacements` is the number of replacement sends ONE logical request may make across every leg and every combo child (1..2, default 1) — not a per-leg retry count and not a send budget, so a replacement still has to fit inside the send allowance the leg already had. A request that already emitted output or a tool call is never replaced, whatever this is set to. The replacement inference may still be billed if the origin had already started the first one, which is why this is off by default. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | | `reasoningDetailsModels?` | `string[]` | Models whose endpoint returns thinking as a structured `reasoning_details` array (MiniMax M-series with `reasoning_split`); stream deltas are cumulative snapshots that are prefix-diffed, and preserved reasoning replays as a `reasoning_details` array instead of a `reasoning_content` string. | diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 6deae1998c5..19b3bb2f002 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -68,6 +68,17 @@ it, nor does it record the refusal as rate-limit or quota evidence against the c was holding. Tool-call side requests such as vision and web search are replayed normally, because repeating them cannot duplicate a turn. +A native Responses provider can opt into replacing that send with +[`retryOnReset`](providers.md#provider-entries-ocxproviderconfig). The same grant covers the +case where the connection survives the header and the SSE body then dies carrying only control +events, because the caller has observed nothing in either one. A replacement happens only when +the request is self-contained (`store: false`, complete input, client-executed tools only, no +server-side continuation state), and one logical request gets the configured number of +replacements in total — across every recovery leg and every combo child, not one each. The +refusal returns as soon as that grant is spent, the leg has no send left, or a replacement fails +for any other reason. A request that already emitted output or a tool call keeps the refusal +regardless. A caller that cancels mid-replacement gets the cancellation, not the refusal. + `noProxy` accepts either a comma-separated string or an array. Both forms add entries without replacing an inherited `NO_PROXY`: diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 7107a9cb120..6c683c8b51f 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -144,6 +144,7 @@ cross-route credential fallback не существует. Строки API GPT- | `responsesSnapshotRepair?` | `boolean` | По умолчанию выключенная клиентская repair для неполных lifecycle snapshot'ов Responses в SSE и JSON. Добавляет отсутствующие status, output и tool metadata, не меняя raw inspection и persistence. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста. Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Только для провайдеров `openai-chat` и `openai-responses` с аутентификацией по ключу. Провайдеры с `authMode: "forward"` (пул аккаунтов ChatGPT) никогда не читают эту настройку и сохраняют число повторов по умолчанию. Опциональный повтор при временных статусах апстрима до начала потока (500, 502, 503, 504, 520, 521, 522): если параметр отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. Покрывает исходный запрос `Responses`, продолжение терминального предохранителя, нативный `/v1/chat/completions`, а также повторные запросы при восстановлении после 429 или ошибки учётной записи. `attempts` — ОБЩЕЕ число разрешённых отправок в апстрим для одного запроса, включая первую (1..10, по умолчанию 3). Это единый бюджет на запрос, общий с восстановлением после сброса соединения, поэтому `3` означает, что до провайдера дойдут не более трёх реальных запросов. Ожидание использует экспоненциальную задержку с фиксированной начальной величиной 400 мс, ограниченную 5 с, и учитывает `Retry-After`. Параметр не связан с `retryOn429`, который обрабатывает ограничение частоты запросов; сбои после начала потока никогда не воспроизводятся. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Только для нативных провайдеров `openai-responses`, включая `authMode: "forward"`. Необязательная замена отправки, которая завершилась неудачей, когда вызывающая сторона ещё ничего не наблюдала: если параметр отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. Покрывает обе неоднозначные стадии — обрыв соединения до любого заголовка ответа и обрыв тела SSE после заголовка, когда оно несло только управляющие события. Заменяется только самодостаточный запрос: `store: false`, полный `input`, отсутствие `previous_response_id`, `conversation` и `stream_id`, и только инструменты, исполняемые клиентом. `replacements` — число замещающих отправок, которые ОДИН логический запрос может сделать по всем участкам и всем дочерним запросам комбо (1..2, по умолчанию 1). Это не число повторов на участок и не бюджет отправок, поэтому замена всё равно должна поместиться в уже имеющийся у участка лимит отправок. Запрос, который уже выдал вывод или вызов инструмента, не заменяется никогда, каким бы ни было это значение. Замещающий вывод модели всё равно может быть оплачен, если источник уже начал первый, поэтому параметр выключен по умолчанию. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. | | `reasoningDetailsModels?` | `string[]` | Модели, чей endpoint возвращает thinking как структурированный массив `reasoning_details` (MiniMax M-series с `reasoning_split`); потоковые дельты — кумулятивные снимки, сравниваемые по префиксу, а сохранённый reasoning воспроизводится массивом `reasoning_details` вместо строки `reasoning_content`. | diff --git a/docs-site/src/content/docs/tr/reference/configuration/providers.md b/docs-site/src/content/docs/tr/reference/configuration/providers.md index 5a0c198d754..1028d5ee95a 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/tr/reference/configuration/providers.md @@ -145,6 +145,7 @@ alanlı seçilmiş kimlikleri yalın kimliklere yeniden yazar. | `responsesSnapshotRepair?` | `boolean` | SSE ve JSON'daki seyrek Responses yaşam döngüsü anlık görüntüleri için varsayılan olarak devre dışı bırakılmış istemciye yönelik onarım. Ham inceleme ve kalıcılık değişmeden kalırken eksik kurallı durumu, çıktıyı ve araç meta verilerini doldurur. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Yalnızca API anahtarı sağlayıcıları (`authMode: "key"`). İsteğe bağlı aynı hedef 429 yeniden denemesi: `retryOn429` olmadığında özellik kapalıdır; nesnenin varlığı `enabled: false` olmadığı sürece özelliği etkinleştirir. 429'da proxy bekler (yukarı akış `Retry-After` veya sabit aralık) ve herhangi bir anahtar yük devretmesinden önce aynı istek üzerinde aynı anahtarla aynı isteği yeniden oynatır — ana metin turu kurtarma döngüsü, Responses doğrudan geçiş hattı, görsel/video köprüsü, web araması sidecar'ı ve terminal devamları genelinde. Yalnızca akış öncesi HTTP 429 yanıtları yeniden oynatma için uygundur; özel `runTurn` aktarımları HTTP yeniden deneme döngüsünün dışındadır. `attempts`, ilk 429'dan sonraki aynı anahtar yeniden oynatmalarını sayar (toplam gönderim = `attempts` + 1) ve ana kurtarma döngüsü, terminal koruma devamı ve köprü yeniden denemeleri tarafından paylaşılan tek bir istek genelinde bütçedir. `attempts`'ı tüketmek yalnızca daha fazla aynı anahtar yeniden oynatmasını durdurur: normal anahtar yük devretmesi veya nihai hata işleme daha sonra kullanılabilir hedeflere göre geçerli olur — anahtar kimlik doğrulamalı doğrudan geçiş hattında yük devretme yoktur, bu nedenle tükenen 429 olduğu gibi görünür. Codex'in kendisi 429'u asla yeniden denemez, bu nedenle tek anahtarlı sağlayıcılar için tek savunma budur. Varsayılanlar: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (tek bir bekleme `maxIntervalMs` ile sınırlandırılır, kendisi de 600000 ile sınırlandırılır), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Yalnızca anahtarla kimlik doğrulanan `openai-chat` ve `openai-responses` sağlayıcıları. `authMode: "forward"` sağlayıcıları (ChatGPT hesap havuzu) bu seçeneği hiç okumaz ve varsayılan merdiveni korur. Akış öncesi geçici yukarı akış durumları (500, 502, 503, 504, 520, 521, 522) için isteğe bağlı yeniden deneme: seçenek belirtilmezse kapalıdır; nesnenin varlığı, `enabled: false` olmadığı sürece özelliği etkinleştirir. İlk Responses isteğini, terminal koruma devamını, yerel `/v1/chat/completions` isteklerini ve 429/hesap kurtarma yeniden getirmelerini kapsar. `attempts`, bir istek için ilk gönderim dahil izin verilen yukarı akış gönderimlerinin TOPLAM sayısıdır (1..10, varsayılan 3) — bağlantı sıfırlama kurtarmasıyla paylaşılan, istek kapsamlı tek bütçedir; dolayısıyla `3`, sağlayıcıya en fazla üç gerçek isteğin ulaşması anlamına gelir. Beklemelerde 400 ms'lik sabit üstel geri çekilme uygulanır, süre 5 sn ile sınırlandırılır ve `Retry-After` dikkate alınır. Hız sınırlamasını işleyen `retryOn429` seçeneğinden ayrıdır; akış ortası hataları hiçbir zaman yeniden oynatılmaz. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Yalnızca yerel `openai-responses` sağlayıcıları, `authMode: "forward"` dahil. Çağıranın hiçbir şey gözlemlemediği bir anda başarısız olan gönderimin isteğe bağlı olarak değiştirilmesi: seçenek belirtilmezse kapalıdır; nesnenin varlığı, `enabled: false` olmadığı sürece özelliği etkinleştirir. İki belirsiz aşamayı da kapsar: yanıt başlığı gelmeden kopan bağlantı ve başlıktan sonra yalnızca denetim olayları taşırken kopan SSE gövdesi. Yalnızca kendi kendine yeten bir istek değiştirilir: `store: false`, eksiksiz `input`, `previous_response_id`, `conversation` veya `stream_id` bulunmaması ve yalnızca istemcinin yürüttüğü araçlar. `replacements`, BİR mantıksal isteğin tüm bacaklar ve tüm combo alt istekleri boyunca yapabileceği değiştirme gönderimi sayısıdır (1..2, varsayılan 1). Bacak başına yeniden deneme sayısı da gönderim bütçesi de değildir; bu yüzden bir değiştirme gönderimi, ilgili bacağın hâlihazırda sahip olduğu gönderim payına sığmak zorundadır. Halihazırda çıktı veya araç çağrısı üretmiş bir istek, bu değer ne olursa olsun asla değiştirilmez. Kaynak ilk çıkarımı zaten başlatmışsa değiştirilen çıkarım yine ücretlendirilebilir; bu nedenle varsayılan olarak kapalıdır. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`'u yalnızca `auto` veya `none` kabul eden modeller; zorunlu seçimlerin derecesi düşürülür. | | `preserveReasoningContentModels?` | `string[]` | Sohbet geçmişinde önceki asistan `reasoning_content`'ini gerektiren modeller. | | `reasoningDetailsModels?` | `string[]` | Thinking'i yapılandırılmış bir `reasoning_details` dizisi olarak döndüren modeller (`reasoning_split` ile MiniMax M-serisi); akış deltaları önek farkıyla işlenen kümülatif anlık görüntülerdir ve korunan reasoning, `reasoning_content` dizesi yerine `reasoning_details` dizisi olarak yeniden oynatılır. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 90ab7966d17..5e748183d92 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -131,6 +131,7 @@ selector，而不是分配一个新名称。 | `responsesSnapshotRepair?` | `boolean` | 默认关闭的客户端修复，用于补全 SSE 与 JSON 中稀疏 Responses 生命周期快照缺失的 status、output 和工具元数据；原始检查与持久化保持不变。 | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商（`authMode: "key"`）。可选的同目标 429 重试：未配置 `retryOn429` 时功能关闭；对象存在即启用，除非 `enabled: false`。收到 429 时等待（上游 `Retry-After` 或固定间隔）后在相同 key 上重放完全相同请求，再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应；自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数（总发送次数 = `attempts` + 1），是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算；`attempts` 耗尽只会停止进一步的同 key 重放：随后按可用目标进行正常的 key 故障转移或最终错误处理——key 认证的 passthrough 线路上没有故障转移，因此耗尽的 429 会原样透出。Codex 自身从不重试 429，因此这是单 key 提供商唯一的防线。默认值：`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`（单次等待以 `maxIntervalMs` 为上限，其本身上限 600000）、`respectRetryAfter: true`。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 仅限使用 key 认证的 `openai-chat` 与 `openai-responses` 提供商。`authMode: "forward"` 的提供商（ChatGPT 账号池）从不读取此选项，保持默认重试次数。可选的流开始前上游瞬态状态码（500、502、503、504、520、521、522）重试：未配置时关闭；对象存在即启用，除非 `enabled: false`。覆盖初始 Responses 请求、终结守卫续接、原生 `/v1/chat/completions`，以及 429/账户恢复重新获取。`attempts` 是单个请求允许向上游发送的总次数，包含首次发送（1..10，默认 3）；它是与连接重置恢复共享的按请求预算，因此 `3` 表示最多只有三个实际请求到达提供商。等待采用固定 400 毫秒的指数退避，上限为 5 秒，并遵循 `Retry-After`。此选项独立于处理速率限制的 `retryOn429`；流开始后的故障绝不会重放。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 仅限原生 `openai-responses` 提供商，包含 `authMode: "forward"`。可选地替换一次在调用方尚未观察到任何内容时就失败的发送：未配置时关闭；对象存在即启用，除非 `enabled: false`。涵盖两个不确定阶段——响应头到达前连接断开，以及响应头之后 SSE 正文只承载控制事件时断开。只有自包含的请求才会被替换：`store: false`、完整的 `input`、没有 `previous_response_id`／`conversation`／`stream_id`，且只使用由客户端执行的工具。`replacements` 是单个逻辑请求在所有环节和所有组合子请求中可以进行的替换发送次数（1..2，默认 1）；它既不是按环节的重试次数，也不是发送预算，因此替换发送仍必须落在该环节已有的发送额度之内。已经产生输出或工具调用的请求，无论此值为何都不会被替换。如果上游已经开始了第一次推理，被替换的推理仍可能计费，因此该选项默认关闭。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型；强制选择会被降级。 | | `preserveReasoningContentModels?` | `string[]` | 需要在聊天历史中保留先前 assistant `reasoning_content` 的模型。 | | `reasoningDetailsModels?` | `string[]` | 以结构化 `reasoning_details` 数组返回思考内容的模型（启用 `reasoning_split` 的 MiniMax M 系列）；流式增量为累积快照，按前缀差分处理，保留的推理以 `reasoning_details` 数组而非 `reasoning_content` 字符串回放。 | diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md index d90eea6e7c6..c10abd0d869 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md @@ -103,6 +103,7 @@ ocx models provider openrouter on | `parallelToolCalls?` | `boolean` | 切換平行工具呼叫。OpenAI Chat 預設開啟；非 chat adapter 僅在明確 `true` 時廣告。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 預設停用的下游 SSE 修復，用於精確佔位 id 與缺失的終端 id。Function-call id 永不被重寫。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 僅限使用金鑰認證的 `openai-chat` 與 `openai-responses` 供應商。`authMode: "forward"` 的供應商（ChatGPT 帳號池）從不讀取此選項，維持預設重試次數。選擇性重試串流開始前的暫時性上游狀態（500、502、503、504、520、521、522）：未設定時停用；只要有此物件即啟用，除非 `enabled: false`。涵蓋初始 `Responses` 請求、終止防護續接、原生 `/v1/chat/completions`，以及 429／帳號復原的重新擷取。`attempts` 是單一請求允許傳送至上游的總次數，包含第一次（1..10，預設 3）；這是與連線重設復原共用的單一請求範圍預算，因此 `3` 表示最多只有三個實際請求會送達供應商。等待採固定 400 毫秒、上限 5 秒的指數退避，並遵循 `Retry-After`。此機制獨立於處理速率限制的 `retryOn429`；串流中的失敗絕不重播。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 僅限原生 `openai-responses` 供應商，包含 `authMode: "forward"`。可選擇性地替換一次在呼叫端尚未觀察到任何內容時就失敗的傳送：未設定時停用；只要有此物件即啟用，除非 `enabled: false`。涵蓋兩個不確定階段——回應標頭抵達前連線中斷，以及標頭之後 SSE 內文只載有控制事件時中斷。只有自我完備的請求才會被替換：`store: false`、完整的 `input`、沒有 `previous_response_id`／`conversation`／`stream_id`，且僅使用由用戶端執行的工具。`replacements` 是單一邏輯請求在所有環節與所有組合子請求中可進行的替換傳送次數（1..2，預設 1）；它既不是各環節的重試次數，也不是傳送預算，因此替換傳送仍必須落在該環節既有的傳送額度之內。已經產生輸出或工具呼叫的請求，無論此值為何都不會被替換。若上游已經開始第一次推論，被替換的推論仍可能計費，因此此選項預設停用。 | | `autoToolChoiceOnlyModels?` | `string[]` | 其 `tool_choice` 僅接受 `auto` 或 `none` 的模型；強制選擇被降級。 | | `preserveReasoningContentModels?` | `string[]` | 需要在 chat 歷史中保留先前 assistant `reasoning_content` 的模型。 | | `reasoningDetailsModels?` | `string[]` | 以結構化 `reasoning_details` 陣列回傳思考內容的模型（啟用 `reasoning_split` 的 MiniMax M 系列）；串流增量為累積快照，以前綴差分處理，保留的推理以 `reasoning_details` 陣列而非 `reasoning_content` 字串重播。 | diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 09bf0d1ccb7..3b40dd43eda 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -849,6 +849,29 @@ compact, and native Chat — are deliberately not opted in. Adapters with their `fetchResponse` (kiro, cursor, google) keep their own retry policies; kiro imports the shared abort/sleep helpers from this module. +## Ambiguous-resend gate + +A model POST that fails with the caller having observed nothing is one question asked at two +points: before any response head, and after a head whose SSE body carried only control events. +`src/lib/request-resend-gate.ts` is the single answer. It derives stage, cause, permission and +send class from `src/lib/request-failure-model.ts` and adds exactly one thing the table names +but does not implement — the narrowly scoped operator override for `refused-ambiguous`. + +The override is bounded on three axes at once. The provider opts in with +`providers..retryOnReset`; the request must be one +`src/server/responses/reset-replay.ts` can judge self-contained, meaning nothing stored, no +server-side continuation state, complete input and only client-executed tools; and the whole +logical request holds one replacement grant, whichever stage asks for it. The grant lives on the +request's execution budget, so a combo child that derives its own scope draws on the same +counter rather than holding a second. A replacement never widens a send budget: it still has to +fit inside the allowance the leg already had, and it is charged to the same counter every other +send goes through. + +A committed or futile failure refuses without touching the grant, so a turn that already emitted +output cannot drain the replacement a later ambiguous reset would have been entitled to. The +cause is derived from the `AttemptRecoveryKind` the send will be recorded as, which is what +keeps the reason in the log and the reason the gate weighed from being two different values. + ## Console upload rejection recovery `src/providers/opencode-zen-rate-limit.ts` recognizes the complete Console upload-rejection envelope only at the effective HTTPS opencode.ai Zen/Go generation endpoint. A provider row name cannot authorize another destination. The two recovery loops in `src/server/responses/core.ts` wait 800 ms and replay the captured serialized request once; cancellation, nonreplayable responses, other errors and a second upload rejection keep their failure semantics. The recovery kind is persisted as `console-go-upload-retry` and has a localized Logs label. @@ -899,6 +922,13 @@ An HTTP 200 does not by itself commit a streaming combo child. The combo parent downstream Responses SSE through `src/server/responses/combo-stream-preflight.ts`, which owns one reader and buffers only until one of these boundaries: +Native post-header reset recovery shares that boundary, because it is asking the same question +about the same bytes. With `replayReadErrors`, the preflight reports the stage it observed — +`headers-only` before any parsed event, `protocol-prelude` after `response.created`, +`semantic-output` once anything else arrives, including a payload it could not parse — and the +resend gate decides. A `response.created` whose snapshot already carries output items is not a +prelude. + - a non-control Responses event begins client-visible output or a tool/action item, after which the target is committed and cross-target replay is forbidden; - a `response.failed` terminal arrives first, in which case the terminal is converted back through @@ -1012,7 +1042,8 @@ is composed from the following owners in `src/server/responses/`; none is a gene | `request-transport.ts` | Live credential selection, dispatch bindings, adapter replacement and same-target request identity. | | `request-sidecar-auth.ts` | Sidecar credential resolution and vision preprocessing. | | `response-effects.ts` | Completion notification, replay publication and live request-tool aliases. | -| `request-send-budget.ts` | Request-wide send accounting, remaining allowance and the pending recovery permit. | +| `request-send-budget.ts` | Request-wide send accounting, remaining allowance, the pending recovery permit and the shared ambiguous-resend grant. | +| `reset-replay.ts` | The operator opt-in for replacing an ambiguous native Responses send, and the per-request grant both stages claim from. | | `request-spend.ts` | This request's entries in the durable spend ledger: one per physical send, settled from the terminal usage. | | `passthrough-execution.ts` | Native host-lease transfer and the enclosing dispatch/delivery `finally`. | | `passthrough-dispatch.ts` | Native request preparation, upstream sends and pre-commit recovery. | @@ -1250,7 +1281,11 @@ turn up to four more times, and a 429 is where the client stops. `upstream_reset_replay_refused`. No response headers is not evidence that the model POST was never processed, so the decision not to replay is ours, made before any response existed — the same shape as `request_send_budget_exhausted`, and it takes the same status -for the same reason. Only an explicitly replay-safe operation opts into reset retries. +for the same reason. An explicitly replay-safe operation retries instead, and a provider that +opted into `retryOnReset` may spend the request's single replacement grant; once that grant is +gone, or the leg has no send left, or a later attempt fails any other way, the leg settles as +this same refusal. Nothing on that path hands the client a status that invites the whole turn +to be sent again. See [ambiguous-resend gate](#ambiguous-resend-gate). **An upstream reset observed mid-stream or after a terminal keeps its existing behaviour.** The passthrough read path still settles a genuine upstream reset as a synthetic 502, and the From d23bfe6f84fce10730fd2d80d03cbdd2640d4790 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 22:06:46 +0900 Subject: [PATCH 4/9] test(responses): count the post-header replacement as a transient-retry send site The source oracle balanced `fetchWithTransientRetry(` occurrences against the call sites that take `attempts` from the provider resolver. The post-header replacement reaches upstream through `refetchAfterProtocolSafeReset` instead, so it drew on the resolver without being counted as a site and the equality broke at 6 against 5. Counting both helpers keeps the equality exact and widens what it protects: a second send helper added on the fixed constant now fails here rather than balancing silently. --- .../responses-passthrough-transient-policy.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/responses/responses-passthrough-transient-policy.test.ts b/tests/responses/responses-passthrough-transient-policy.test.ts index 77bc00487fe..211d30d1ed7 100644 --- a/tests/responses/responses-passthrough-transient-policy.test.ts +++ b/tests/responses/responses-passthrough-transient-policy.test.ts @@ -43,7 +43,12 @@ describe("the Responses passthrough lane reads the provider transient policy", ( }); test("every transient-retry send takes its attempts from that resolver", () => { - const sites = occurrences(packed, "fetchWithTransientRetry("); + // Every helper this lane sends through, not just the one it started with. The post-header + // replacement reaches upstream exactly like the legs above it and has to draw on the same + // resolver; counting only `fetchWithTransientRetry` would let a second send helper be added + // on the constant while this file still reported balance. + const sites = occurrences(packed, "fetchWithTransientRetry(") + + occurrences(packed, "refetchAfterProtocolSafeReset("); // The lane's initial send plus its recovery legs. A site that stops being counted here is a // site that stopped being governed by the policy. expect(sites).toBeGreaterThanOrEqual(4); From 71241593ee2470b1325477e258fe75bd3951f861 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 22:12:22 +0900 Subject: [PATCH 5/9] fix(responses): make the self-contained judgment reach the allowance `ambiguousResendAllowanceFor` declared its second parameter as `unknown` and handed it straight to `selfContainedResponsesBody`, while the dispatch site passed the memoized predicate. A function is not a record, so the judgment was always false and every opted-in reset refused as `ambiguous-request-not-replayable`. The feature was inert and nothing in the transport tests could see it, because they never reach the body judgment. The parameter is now `() => boolean` and the property is a getter, so the laziness the call site wanted is real and passing a body instead of a predicate is a typecheck failure rather than a silent false. Also stop cancelling the original body from the deferred wrapper once the preflight owns its reader: that body is locked, so the cancellation rejected and was swallowed. `initialize` already releases whichever body it selected when it observes a cancelled downstream, and that is the one that has to be let go. --- src/server/responses/combo-stream-preflight.ts | 7 ++++++- src/server/responses/reset-replay.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index 032d936e329..7e76fc43ffb 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -416,9 +416,14 @@ export function deferProtocolSafeResetRecovery( try { void reader.cancel(reason).catch(() => {}); } catch { /* already closed */ } try { reader.releaseLock(); } catch { /* already released */ } reader = undefined; - } else { + } else if (initialization === undefined) { + // Nothing has read the upstream yet, so this body is still ours to cancel. cancelBody(response.body, reason); } + // A cancel while the preflight is mid-flight falls through deliberately. That body is + // locked by the preflight's own reader, so cancelling it here would reject and be + // swallowed; `initialize` sees `closed` when it settles and releases whichever body it + // ended up selecting, which is the one that actually has to be let go. }, }, { highWaterMark: 0 }); diff --git a/src/server/responses/reset-replay.ts b/src/server/responses/reset-replay.ts index 4ee7e607ae8..b8e47d44b93 100644 --- a/src/server/responses/reset-replay.ts +++ b/src/server/responses/reset-replay.ts @@ -89,16 +89,20 @@ export function selfContainedResponsesBody(body: unknown): boolean { * say WHY it refused: "the operator granted nothing" and "this request cannot be replayed" are * different operator problems, and folding them together is what made the old refusal a single * undifferentiated no. + * + * `selfContained` is a predicate rather than a body, and the getter below is why: a provider + * that never opted in must not pay to walk the input array, and the caller memoizes one answer + * across every leg of the request. */ export function ambiguousResendAllowanceFor( provider: Pick, - inboundBody: unknown, + requestIsSelfContained: () => boolean, claim: (limit: number) => boolean, ): AmbiguousResendAllowance | undefined { const policy = resetReplayPolicyFor(provider); if (policy === null) return undefined; return { - selfContained: selfContainedResponsesBody(inboundBody), + get selfContained(): boolean { return requestIsSelfContained(); }, claim: () => claim(policy.replacements), }; } From 50d9033734df0381daadd381630f48381b68ba34 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 22:37:31 +0900 Subject: [PATCH 6/9] test(responses): declare reset-replay.ts as an extracted owner responses-core-modules.test.ts derives the owner graph from the source imports and compares it to the inventory. A new sibling under src/server/responses/ has to be in one of the two lists or the comparison fails, which is the point: a new owner must not disappear from source-oracle coverage by being absent. It belongs in the inventory rather than the separately-owned boundary set, because structure/transports/responses.md already lists it in the per-request core-module ownership table. The 2000-line coverage now applies to it too, and passthrough-dispatch.ts remains the largest owner at 1762. --- tests/helpers/responses-core-source.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/responses-core-source.ts b/tests/helpers/responses-core-source.ts index 6b1aa1e5d63..36b44511c96 100644 --- a/tests/helpers/responses-core-source.ts +++ b/tests/helpers/responses-core-source.ts @@ -37,6 +37,7 @@ export const RESPONSES_CORE_MODULES = [ "request-spend.ts", "passthrough-execution.ts", "passthrough-dispatch.ts", + "reset-replay.ts", "passthrough-delivery.ts", "sidecar-execution.ts", "completion-policy.ts", From 125fb87d9874c244922ec1b96b3a9c6afaff9d1a Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 23:11:55 +0900 Subject: [PATCH 7/9] test(responses): pin what a committed stream actually does at a read error The new case asserted that a read error after output commits reports `semantic-output`. It cannot: `preflightComboStreamResponse` returns the body as `accepted` the moment output commits, so the error happens on the caller's side of the boundary and no stage is ever reported. Assert that instead, which is the stronger safety statement -- a committed stream never reaches the resend gate at all, rather than reaching it and being refused there -- and keep the prefix and the original error observable to whoever reads the returned body. The stage helper stays total, with a note that its committed branches exist so a later change to that loop cannot promote a committed stream by omission. --- src/server/responses/combo-stream-preflight.ts | 6 ++++++ tests/routing/combo-stream-preflight.test.ts | 16 ++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index 7e76fc43ffb..801946fb571 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -133,6 +133,12 @@ export function comboStreamPayloadCommitsOutput(payload: unknown): boolean { * A terminal that settled carrying no output is `protocol-prelude`, not `terminal`. That is * the failure model's own rule: `terminal` means the answer was delivered, and an empty * completion delivered none. + * + * Only the two nothing-observed stages actually reach a read error today: the loop below hands + * the body back as `accepted` the moment output commits or a terminal arrives, so a stream + * that committed anything never reports a stage at all. The committed branches stay because + * this has to be total for any other caller, and because a later change to that loop must not + * be able to promote a committed stream into a replaceable one by omission. */ function observedResponsesStage(state: { readonly outputCommitted: boolean; diff --git a/tests/routing/combo-stream-preflight.test.ts b/tests/routing/combo-stream-preflight.test.ts index 36aa183538a..760431eddb5 100644 --- a/tests/routing/combo-stream-preflight.test.ts +++ b/tests/routing/combo-stream-preflight.test.ts @@ -567,7 +567,7 @@ describe("combo stream preflight", () => { expect(source.cancelSpy()!.mock.calls).toHaveLength(0); }); - test("a read error before any event is headers-only, and after output is committed", async () => { + test("a read error before any event is headers-only, and a committed stream never reports one", async () => { const readError = new Error("preflight-read-reset"); const bare = await preflightComboStreamResponse( prefixThenReadError(new TextEncoder().encode(""), readError).response, @@ -581,6 +581,10 @@ describe("combo stream preflight", () => { expect(stageCommitment(bare.stage)).toBe("nothing-observed"); } + // Once output commits the preflight stops buffering and hands the body back, so the read + // error that follows happens on the caller's side of the boundary and no stage is ever + // reported. That is the stronger statement: a committed stream does not reach the resend + // gate at all, rather than reaching it and being refused there. const outputPrefix = new TextEncoder().encode(`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hi", })}\n\n`); @@ -590,11 +594,11 @@ describe("combo stream preflight", () => { undefined, { replayReadErrors: true }, ); - expect(committed.kind).toBe("read-error"); - if (committed.kind === "read-error") { - expect(committed.stage).toBe("semantic-output"); - expect(stageCommitment(committed.stage)).not.toBe("nothing-observed"); - } + expect(committed.kind).toBe("accepted"); + // The prefix is still relayed and the error still reaches whoever reads it. + const reader = committed.response.body!.getReader(); + expect((await reader.read()).value).toEqual(outputPrefix); + await expect(reader.read()).rejects.toBe(readError); }); test("a response.created carrying output is not a prelude", () => { From 0f3d27a5cad56d17f3b0faafa49618a7e87e528b Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 23:17:38 +0900 Subject: [PATCH 8/9] test(responses): prove the resend boundary where it is enforced The stage a read error is reported at is only half the guarantee. What decides permission is that a stream which committed output never gets a replacement offered at all, and the seam that decides it is the deferred wrapper rather than the preflight. Assert it there: a prelude-only stream consults the recovery callback exactly once and at a stage whose `stageCommitment` is `nothing-observed`, and an output-bearing stream never consults it. The commitment is read from the failure model instead of compared against a written-out stage name, so a stage added to the model later cannot pass this by being unlisted. --- tests/routing/combo-stream-preflight.test.ts | 43 +++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/routing/combo-stream-preflight.test.ts b/tests/routing/combo-stream-preflight.test.ts index 760431eddb5..29e6d34c3e7 100644 --- a/tests/routing/combo-stream-preflight.test.ts +++ b/tests/routing/combo-stream-preflight.test.ts @@ -1,9 +1,10 @@ import { describe, expect, spyOn, test } from "bun:test"; import { comboStreamPayloadCommitsOutput, + deferProtocolSafeResetRecovery, preflightComboStreamResponse, } from "../../src/server/responses/combo-stream-preflight"; -import { stageCommitment } from "../../src/lib/request-failure-model"; +import { stageCommitment, type RequestFailureStage } from "../../src/lib/request-failure-model"; import type { RequestLogContext } from "../../src/server/request-log"; import { MAX_CLIENT_SSE_FRAME_BYTES } from "../../src/server/sse-frame-buffer"; @@ -601,6 +602,46 @@ describe("combo stream preflight", () => { await expect(reader.read()).rejects.toBe(readError); }); + /** + * The boundary that decides resend permission, asserted where it is actually enforced. + * + * The stage a read error is reported at is only half the guarantee. What matters is that a + * stream which committed output never gets a replacement offered at all, and the seam that + * decides it is the deferred wrapper, not the preflight. The commitment is read from + * `stageCommitment` rather than compared against a written-out stage name, so a stage added + * to the model later cannot pass this by being unlisted. + */ + test("a replacement is offered only for a stage the caller observed nothing at", async () => { + const readError = new Error("preflight-read-reset"); + const logCtx: RequestLogContext = { model: "m1", provider: "a" }; + const seen: RequestFailureStage[] = []; + const recover = async (_error: unknown, stage: RequestFailureStage): Promise => { + seen.push(stage); + return null; + }; + + const prelude = deferProtocolSafeResetRecovery( + prefixThenReadError(createdPrefix, readError).response, logCtx, recover); + const preludeReader = prelude.body!.getReader(); + expect((await preludeReader.read()).value).toEqual(createdPrefix); + await expect(preludeReader.read()).rejects.toBe(readError); + expect(seen).toHaveLength(1); + expect(stageCommitment(seen[0]!)).toBe("nothing-observed"); + + seen.length = 0; + const outputPrefix = new TextEncoder().encode(`data: ${JSON.stringify({ + type: "response.output_text.delta", delta: "hi", + })}\n\n`); + const committed = deferProtocolSafeResetRecovery( + prefixThenReadError(outputPrefix, readError).response, logCtx, recover); + const committedReader = committed.body!.getReader(); + expect((await committedReader.read()).value).toEqual(outputPrefix); + await expect(committedReader.read()).rejects.toBe(readError); + // Never consulted. A turn whose output the caller already saw cannot be replaced, and it + // does not get as far as asking. + expect(seen).toEqual([]); + }); + test("a response.created carrying output is not a prelude", () => { expect(comboStreamPayloadCommitsOutput({ type: "response.created", response: { id: "r1", output: [] }, From fd3cc808c97e4d401c2317bb721a4b81bd562a59 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 20 Sep 2026 23:33:54 +0900 Subject: [PATCH 9/9] test(layout): name the gate test so its seed does not contradict its domain The layout map's regex seeds place a new test file on the day it is added, and the tooling oracle fails when a seed disagrees with the explicit entry, because that seed would put the next similarly named file in the wrong directory. `request-` seeds to `usage`, so `request-resend-gate.test.ts` pointed there while the explicit table said `lib`. Renamed rather than pinned: `pinnedOverrides` is for the historical files whose name says one thing and whose imports say another, not a place to park a file added today. `ambiguous-resend-gate` matches no seed, which is the case the oracle tolerates, and it says what the gate is about -- the ambiguous row of the stage table, which is precisely not the transient one. Updates both layout maps and the INV-RESEND-02 binding in structure/overview.md. --- scripts/test-layout/layout.json | 4 ++-- structure/overview.md | 2 +- tests/fixtures/test-layout-expected.json | 4 ++-- ...uest-resend-gate.test.ts => ambiguous-resend-gate.test.ts} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename tests/lib/{request-resend-gate.test.ts => ambiguous-resend-gate.test.ts} (100%) diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index acbb0ee4381..979493010c1 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1611,9 +1611,9 @@ "claude-desktop-first-party.test.ts": "claude-integration", "claude-desktop-mode-explanation.test.ts": "claude-integration", "claude-intercept-integration.test.ts": "server", - "request-resend-gate.test.ts": "lib", "management-provider-reset-replay.test.ts": "server", - "responses-reset-replay.test.ts": "responses" + "responses-reset-replay.test.ts": "responses", + "ambiguous-resend-gate.test.ts": "lib" }, "migrated": [ "adapters", diff --git a/structure/overview.md b/structure/overview.md index ac1fb372792..67fb90bfaa1 100644 --- a/structure/overview.md +++ b/structure/overview.md @@ -151,7 +151,7 @@ still cover the rule, which is a judgement only review makes. and a stage the caller observed something at refuses without spending it. The grant never widens a send budget: an authorised replacement still has to fit the allowance the leg already had. - Enforced by `tests/lib/request-resend-gate.test.ts`. + Enforced by `tests/lib/ambiguous-resend-gate.test.ts`. CI enumerates that domain layout through `scripts/ci/run-bun-test-batches.sh`. Its default general scope and 12-file/120-second process shape leave the dedicated Linux storage-policy and api-usage diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index bed9ce79b2e..dd374a930c9 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1443,7 +1443,7 @@ "claude-desktop-first-party.test.ts": "claude-integration", "claude-desktop-mode-explanation.test.ts": "claude-integration", "claude-intercept-integration.test.ts": "server", - "request-resend-gate.test.ts": "lib", "management-provider-reset-replay.test.ts": "server", - "responses-reset-replay.test.ts": "responses" + "responses-reset-replay.test.ts": "responses", + "ambiguous-resend-gate.test.ts": "lib" } diff --git a/tests/lib/request-resend-gate.test.ts b/tests/lib/ambiguous-resend-gate.test.ts similarity index 100% rename from tests/lib/request-resend-gate.test.ts rename to tests/lib/ambiguous-resend-gate.test.ts