From 23a3694803348735f8cb5478ef33672c0af31f66 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 20 Sep 2026 22:00:13 +0900 Subject: [PATCH 1/8] fix(codex): bind scoped quota suppression to alternate --- src/codex/quota-rejection.ts | 12 +++-- src/server/responses/core-codex-account.ts | 44 +++++++++++++++---- src/server/responses/core.ts | 5 ++- structure/providers/openai-tiers.md | 5 ++- structure/transports/responses.md | 13 +++--- .../codex-quota-rejection.test.ts | 31 ++++++++++--- 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/src/codex/quota-rejection.ts b/src/codex/quota-rejection.ts index cde0a8272d1..d4559c1187c 100644 --- a/src/codex/quota-rejection.ts +++ b/src/codex/quota-rejection.ts @@ -349,9 +349,10 @@ export async function codexScopedExhaustionCode( * Status alone and message text are intentionally insufficient. The broad * alternate-account retry remains eligible for 429/402 to preserve #584. * - * The one carve-out from that breadth is an organization- or project-scoped exhaustion - * ({@link SCOPED_EXHAUSTION_CODE_VALUES}), which reports `alternateRetryEligible: false` - * because every credential inside the refusing limit would be refused by the same counter. + * Organization- or project-scoped exhaustion ({@link SCOPED_EXHAUSTION_CODE_VALUES}) remains + * alternate-retry eligible here because the response does not identify the refusing scope. The + * account-rotation path may suppress the send later when the resolved alternate carries binding + * evidence that it shares an organization-level counter. */ export async function classifyCodexPreStreamRejection( response: Response, @@ -377,7 +378,10 @@ export async function classifyCodexPreStreamRejection( }); } if (scoped) { - return rejection(status, "scoped-quota-exhaustion", { scopedExhaustionCode: scoped }); + return rejection(status, "scoped-quota-exhaustion", { + alternateRetryEligible: true, + scopedExhaustionCode: scoped, + }); } return rejection( status, diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 99796846d8f..688793da947 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -276,15 +276,11 @@ export async function shouldRetryCodexPoolAccountQuota( // body carries no quota evidence either, but the marker is the contract, not the prose. if (isNonReplayableResponse(response)) return false; if (response.status === 402 || response.status === 429) { - // Status alone used to authorize the move, which is right for a limit the ACCOUNT owns and - // wrong for one it merely belongs to. An organization- or project-scoped exhaustion refuses - // every credential inside that organization, so the second account meets the same counter - // and the only thing the rotation buys is a second cold prompt prefix (#4546). Positive - // evidence is required to withhold it: the helper fails closed, so an unreadable or - // ambiguous body keeps the broad #584 behaviour unchanged, and `rate_limit_exceeded`, - // `slow_down` and plan-level exhaustion still rotate exactly as before. - const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection"); - return await codexScopedExhaustionCode(response, { signal }) === undefined; + // The response does not identify the organization or project whose quota was exhausted. + // Resolve the alternate before deciding whether its known workspace identity proves that an + // organization-scoped retry would be futile. Until then, preserve the broad #584 behaviour. + void signal; + return true; } if (response.status < 500 || response.status >= 600) return false; try { @@ -301,6 +297,20 @@ export async function shouldRetryCodexPoolAccountQuota( } +export async function shouldRetryCodexScopedQuotaOnAlternate( + response: Response, + firstWorkspaceAccountId: string, + alternateWorkspaceAccountId: string | undefined, + signal?: AbortSignal, +): Promise { + if (!firstWorkspaceAccountId || firstWorkspaceAccountId !== alternateWorkspaceAccountId) return true; + const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection"); + const code = await codexScopedExhaustionCode(response, { signal }); + // Workspace identity binds organization-level limits, but the response supplies no project id. + return code === undefined || code === "project_spend_limit_exceeded"; +} + + /** * A pre-stream upstream 5xx another Codex account may still be able to serve. * @@ -644,6 +654,22 @@ export async function retryCodexPoolOnAlternateAccount( return { kind: "no-alternate" }; } + if ( + (outcomeStatus === 429 || outcomeStatus === 402) + && !await shouldRetryCodexScopedQuotaOnAlternate( + firstResponse, + firstAuthCtx.chatgptAccountId, + retryAuthCtx.kind === "pool" || retryAuthCtx.kind === "main-pool" + ? retryAuthCtx.chatgptAccountId + : undefined, + options.abortSignal, + ) + ) { + accountMovePermit?.release(); + releaseCodexAuthContextProbeLease(retryAuthCtx); + return { kind: "no-alternate" }; + } + const quotaMeta = { ...codexQuotaOutcomeMeta(firstResponse), ...(await codexDenialOutcomeMeta(firstResponse)) }; if (outcomeStatus === 429 || outcomeStatus === 402) { const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 89bf34dad3a..19216befddb 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -189,7 +189,10 @@ export { readDisplaySafeErrorText } from "./core-errors"; export { usesCodexForwardPoolAuth } from "./core-codex-account"; export { preAuthUpstreamHostCircuitKey } from "./core-codex-account"; export { upstreamHostCircuitOpenResponse } from "./core-codex-account"; -export { shouldRetryCodexPoolAccountQuota } from "./core-codex-account"; +export { + shouldRetryCodexPoolAccountQuota, + shouldRetryCodexScopedQuotaOnAlternate, +} from "./core-codex-account"; export { shouldRetryCodexPoolAccountTransient } from "./core-codex-account"; export { codexAccountGatedCanonicalWireModel } from "./core-codex-account"; export { codexForwardTerminalOutcomeRecorder } from "./core-codex-account"; diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index c9479b1b051..374ff4fe941 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -149,11 +149,12 @@ and credential/transport failures retain their ordinary handling. `credit_balance_exhausted`, `organization_spend_limit_exceeded`, `project_spend_limit_exceeded` and `organization_usage_limit_exceeded` name a balance or cap held by the organization or project, so `classifyCodexPreStreamRejection` reports `scoped-quota-exhaustion` with `alternateRetryEligible` -false and `scopedExhaustionCode` set, and never `resetCreditEligible` — a reset credit reconciles a +true and `scopedExhaustionCode` set, and never `resetCreditEligible` — a reset credit reconciles a ChatGPT plan window and cannot pay an organization's bill. The two sets are disjoint and share one parser, so a `code`/`type` pair that disagrees, a duplicate key at any depth, or a case or whitespace near-miss yields no code at all. `codexScopedExhaustionCode` exposes the scoped answer -alone for the rotation gate and fails closed, so only positive evidence changes a routing decision. +alone for the post-resolution rotation gate and fails closed. A code by itself cannot bind the +refusal to every credential in a heterogeneous pool. `pausedCodexAccountIds` is a persisted Pool eligibility boundary. A paused added account or the stable `__main__` alias remains visible for maintenance and quota reads, but is excluded from new diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 031f1d5417f..622c089885f 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -406,11 +406,14 @@ bridge. It uses the existing account quorum, cooldown and three-rotation request the complete credential/transport/replay identity, and attributes usage to the serving account. Single-account installs do not retry; a missing alternate credential preserves the original error. -`shouldRetryCodexPoolAccountQuota` withholds that rotation when the 429 or 402 body names an -organization- or project-scoped exhaustion (`codexScopedExhaustionCode` in -`src/codex/quota-rejection.ts`). Every credential inside the refusing organization meets the same -counter, so the move would pay a second cold prompt prefix for no new capacity. Withholding the -move does not withhold the accounting: `src/server/responses/passthrough-delivery.ts` applies the +`shouldRetryCodexPoolAccountQuota` admits that rotation when the 429 or 402 body names an +organization- or project-scoped exhaustion because the response does not identify the refusing +scope. After resolving an alternate, the rotation path uses `codexScopedExhaustionCode` from +`src/codex/quota-rejection.ts` to withhold organization-level retries only when both credentials +have the same known workspace account id. Project exhaustion remains retryable because no project +identity is available. Credentials in distinct or unknown workspaces therefore retain failover, +while a proven same-workspace move cannot pay a second cold prompt prefix for no new capacity. +Withholding the move does not withhold the accounting: `src/server/responses/passthrough-delivery.ts` applies the response's quota headers to the serving account and records the 429 outcome on the ordinary delivery path, so the account still earns its cooldown and leaves the selection pool. The gate fails closed — an empty, truncated, unparseable, duplicate-keyed or aborted body keeps the broad diff --git a/tests/codex-integration/codex-quota-rejection.test.ts b/tests/codex-integration/codex-quota-rejection.test.ts index 9327b33db23..cce7e812422 100644 --- a/tests/codex-integration/codex-quota-rejection.test.ts +++ b/tests/codex-integration/codex-quota-rejection.test.ts @@ -4,6 +4,7 @@ import { BOUNDED_BODY_MAX_BYTES } from "../../src/lib/bounded-body"; import { consumeComboFailure, shouldRetryCodexPoolAccountQuota, + shouldRetryCodexScopedQuotaOnAlternate, shouldRetryCodexPoolAccountTransient, } from "../../src/server/responses/core"; import { markResponseNonReplayable } from "../../src/lib/upstream-retry"; @@ -488,7 +489,8 @@ describe("Codex pre-stream quota rejection classification", () => { }); /** - * Rotating inside the limit that refused is the send amplification #4546 exists to stop. + * Rotating inside a proven-shared limit is the send amplification #4546 exists to stop. A code + * alone cannot prove that a prospective alternate belongs to the same organization or project. * * openai/codex #44492 and #45602 reclassified exactly these HTTP 429 codes as terminal quota * exhaustion while deliberately keeping `rate_limit_exceeded` and `slow_down` retryable, and @@ -499,7 +501,7 @@ describe("Codex pre-stream quota rejection classification", () => { * user-level rate limit stops failing over, and that regression would be invisible until a pool * stopped rotating in production. */ -describe("organization-scoped quota exhaustion withholds the account rotation (#4546)", () => { +describe("scoped quota exhaustion preserves unbound account rotation (#4546)", () => { const SCOPED_CODES = [ "credit_balance_exhausted", "organization_spend_limit_exceeded", @@ -512,7 +514,7 @@ describe("organization-scoped quota exhaustion withholds the account rotation (# expect(result).toEqual({ kind: "scoped-quota-exhaustion", status: 429, - alternateRetryEligible: false, + alternateRetryEligible: true, resetCreditEligible: false, scopedExhaustionCode: code, }); @@ -520,18 +522,33 @@ describe("organization-scoped quota exhaustion withholds the account rotation (# expect(result).not.toHaveProperty("semanticCode"); }); - test.each(SCOPED_CODES)("%s withholds the alternate-account send", async code => { + test.each(SCOPED_CODES)("%s keeps an unresolved alternate-account send eligible", async code => { await expect(shouldRetryCodexPoolAccountQuota(jsonRejection(429, { code }))) - .resolves.toBe(false); + .resolves.toBe(true); }); test("a root-level code and a 402 are read the same way", async () => { await expect(shouldRetryCodexPoolAccountQuota( jsonPayload(429, { code: "organization_spend_limit_exceeded" }), - )).resolves.toBe(false); + )).resolves.toBe(true); await expect(shouldRetryCodexPoolAccountQuota( jsonRejection(402, { code: "credit_balance_exhausted" }), - )).resolves.toBe(false); + )).resolves.toBe(true); + }); + + test("only proven shared organization scope withholds the resolved alternate", async () => { + const rejection = () => jsonRejection(429, { code: "organization_spend_limit_exceeded" }); + await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", "workspace-b")) + .resolves.toBe(true); + await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", undefined)) + .resolves.toBe(true); + await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", "workspace-a")) + .resolves.toBe(false); + await expect(shouldRetryCodexScopedQuotaOnAlternate( + jsonRejection(429, { code: "project_spend_limit_exceeded" }), + "workspace-a", + "workspace-a", + )).resolves.toBe(true); }); test.each([ From 559db1649e29645eed784dce14d9242d9812492c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:08:51 +0000 Subject: [PATCH 2/8] fix(ci): restore core.ts to file-size ratchet cap The scoped-quota re-export grew src/server/responses/core.ts past its committed 210-line cap (213). Collapse the two-name re-export back to one line; the file's export list already carries longer single-line statements. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/server/responses/core.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 19216befddb..9905ed7dc5f 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -189,10 +189,7 @@ export { readDisplaySafeErrorText } from "./core-errors"; export { usesCodexForwardPoolAuth } from "./core-codex-account"; export { preAuthUpstreamHostCircuitKey } from "./core-codex-account"; export { upstreamHostCircuitOpenResponse } from "./core-codex-account"; -export { - shouldRetryCodexPoolAccountQuota, - shouldRetryCodexScopedQuotaOnAlternate, -} from "./core-codex-account"; +export { shouldRetryCodexPoolAccountQuota, shouldRetryCodexScopedQuotaOnAlternate } from "./core-codex-account"; export { shouldRetryCodexPoolAccountTransient } from "./core-codex-account"; export { codexAccountGatedCanonicalWireModel } from "./core-codex-account"; export { codexForwardTerminalOutcomeRecorder } from "./core-codex-account"; From 52675933a402c5b006b04a8f00587d3fbcd74cb8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 02:05:38 +0000 Subject: [PATCH 3/8] fix(codex): record wrapped quota on suppressed moves and bind caller main - record the normalized 429/402 outcome before returning no-alternate on a suppressed same-workspace move, so a 5xx-wrapped quota refusal still cools the refused account instead of reading as transient - bind a request-owned `main` alternate by the caller credential's own workspace id (chatgpt-account-id header, else the bearer token's account claim) via callerCodexWorkspaceAccountId - apply the same scoped-quota workspace gate to the single bounded alternate send in the native /responses/compact path - cover all three in tests and update the transport doc Co-Authored-By: Epinephrine --- src/codex/auth-context.ts | 14 +++ src/server/responses/compact.ts | 21 +++- src/server/responses/core-codex-account.ts | 39 +++++--- structure/transports/responses.md | 17 ++-- .../responses-compaction-routing.test.ts | 40 ++++++++ tests/server/server-auth.test.ts | 95 +++++++++++++++++++ 6 files changed, 204 insertions(+), 22 deletions(-) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index db100fd6c26..206036cbbe1 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -598,6 +598,20 @@ function selectedCodexToken(headers: Headers): { accessToken: string; chatgptAcc }; } +/** + * The workspace account id a request-owned `main` credential materializes under, or + * `undefined` when the caller's headers carry none. This is the `chatgpt-account-id` + * `materializeCodexUpstreamAuth` would set for a caller-owned `{ kind: "main" }` context, + * read here without touching a credential store so a rotation gate can compare workspace + * scope before a send is ever built. + */ +export function callerCodexWorkspaceAccountId(headers: Headers): string | undefined { + const explicit = headers.get("chatgpt-account-id"); + if (explicit) return explicit; + const bearer = headers.get("authorization")?.replace(/^Bearer\s+/i, "").trim(); + return bearer ? extractAccountId(undefined, bearer) : undefined; +} + function assertMaterializedReserve(headers: Headers, ctx: CodexAuthContext, options: CodexAuthMaterializationOptions): void { if (!requiresReserveAuthorization(options.config, options.modelId, options.admission)) return; assertReserveAdmission(options.config!); diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 89e40123789..ce9a74c1d52 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -51,6 +51,7 @@ import { describeImagesInPlace, planVisionSidecar, shouldResolveOpenAiVisionSide import { createAdapterEventQueue, preflightAdapterEvents } from "../../adapters/run-turn-queue"; import { applyCodexAuthContextToProvider, + callerCodexWorkspaceAccountId, createCodexReserveDispatchGuard, unwrapUpstreamRetryEvidenceError, CodexMainProfileDrainingError, @@ -186,6 +187,7 @@ import { handleResponses, preAuthUpstreamHostCircuitKey, poolCredentialRefreshIncompleteResponse, + shouldRetryCodexScopedQuotaOnAlternate, upstreamHostCircuitOpenResponse, usesCodexForwardPoolAuth, } from "./core"; @@ -1213,7 +1215,24 @@ export async function handleResponsesCompact( recordCompactPoolOutcome(outcomeCtx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } - if (alternate) { + // The same scope binding the regular path applies: an organization-scoped + // exhaustion refuses every credential in that workspace, so a proven + // same-workspace alternate pays a cold prompt prefix for no new capacity. + // Suppression is not silence — the buffered recorder below still attributes + // the 429/402 to the account that produced it. + const sharedWorkspaceScope = alternate != null + && !await shouldRetryCodexScopedQuotaOnAlternate( + upstream, + authCtx.chatgptAccountId, + alternate.authCtx.kind === "pool" || alternate.authCtx.kind === "main-pool" + ? alternate.authCtx.chatgptAccountId + : callerCodexWorkspaceAccountId(req.headers), + req.signal, + ); + if (alternate && sharedWorkspaceScope) { + releaseCodexAuthContextProbeLease(alternate.authCtx); + } + if (alternate && !sharedWorkspaceScope) { // Same order the regular path uses (core.ts:349-357): a 429/402 carries the // quota snapshot that produced it, so refresh A's cache before recording its // rejection. Skipping this leaves quota-strategy routing and the dashboard diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 688793da947..076bf14cbaa 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -41,6 +41,7 @@ import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup"; import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account"; import { slugsEquivalent } from "../../providers/slug-codec"; import { + callerCodexWorkspaceAccountId, codexProbeLeaseId, codexTransientProbeGrant, codexProbeQuotaScope, @@ -538,6 +539,22 @@ export async function retryCodexPoolOnAlternateAccount( writerGeneration: firstAuthCtx.writerGeneration, }); }; + // A body-confirmed quota response may arrive under HTTP 5xx. A path that returns the + // first response without a move must still record the NORMALIZED outcome: the ordinary + // terminal recorder sees only that wire status and would misclassify it as transient, + // leaving the exhausted account immediately selectable next turn. + const recordWrappedQuotaOutcome = (): void => { + if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return; + recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...codexQuotaOutcomeMeta(firstResponse), + threadId: firstAuthCtx.affinityKey, + modelId: route.modelId, + probeLeaseId: codexProbeLeaseId(firstAuthCtx), + probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), + transientProbe: codexTransientProbeGrant(firstAuthCtx), + writerGeneration: firstAuthCtx.writerGeneration, + }); + }; if (outcomeStatus === 400 && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId)) { invalidateCodexModelEntitlementsForAccount(firstAuthCtx.accountId); let refreshed; @@ -634,20 +651,7 @@ export async function retryCodexPoolOnAlternateAccount( && retryAuthCtx?.kind !== "main-pool" && retryAuthCtx?.kind !== "main" ) { - // A body-confirmed quota response may arrive under HTTP 5xx. Without an alternate, - // the ordinary terminal recorder sees only that wire status and would misclassify it - // as transient, leaving the exhausted account immediately selectable next turn. - if (outcomeStatus !== firstResponse.status && (outcomeStatus === 429 || outcomeStatus === 402)) { - recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { - ...codexQuotaOutcomeMeta(firstResponse), - threadId: firstAuthCtx.affinityKey, - modelId: route.modelId, - probeLeaseId: codexProbeLeaseId(firstAuthCtx), - probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), - transientProbe: codexTransientProbeGrant(firstAuthCtx), - writerGeneration: firstAuthCtx.writerGeneration, - }); - } + recordWrappedQuotaOutcome(); // No usable alternate was resolved, so the reserved move never becomes a send. accountMovePermit?.release(); recordUnmovedTransientOutcome(); @@ -661,10 +665,15 @@ export async function retryCodexPoolOnAlternateAccount( firstAuthCtx.chatgptAccountId, retryAuthCtx.kind === "pool" || retryAuthCtx.kind === "main-pool" ? retryAuthCtx.chatgptAccountId - : undefined, + // A request-owned `main` alternate has no stored account id; its workspace + // identity is what the caller's own credential materializes upstream. + : callerCodexWorkspaceAccountId(callerAuthHeaders), options.abortSignal, ) ) { + // Suppressing the move is not suppressing the evidence: a same-workspace refusal + // still records its normalized quota outcome on the account that produced it. + recordWrappedQuotaOutcome(); accountMovePermit?.release(); releaseCodexAuthContextProbeLease(retryAuthCtx); return { kind: "no-alternate" }; diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 53925f47bb4..8284dcce763 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -420,12 +420,17 @@ Single-account installs do not retry; a missing alternate credential preserves t `shouldRetryCodexPoolAccountQuota` admits that rotation when the 429 or 402 body names an organization- or project-scoped exhaustion because the response does not identify the refusing -scope. After resolving an alternate, the rotation path uses `codexScopedExhaustionCode` from -`src/codex/quota-rejection.ts` to withhold organization-level retries only when both credentials -have the same known workspace account id. Project exhaustion remains retryable because no project -identity is available. Credentials in distinct or unknown workspaces therefore retain failover, -while a proven same-workspace move cannot pay a second cold prompt prefix for no new capacity. -Withholding the move does not withhold the accounting: `src/server/responses/passthrough-delivery.ts` applies the +scope. After resolving an alternate — on `/v1/responses` and on the single bounded send the +native `/responses/compact` path resolves — the rotation path uses `codexScopedExhaustionCode` +from `src/codex/quota-rejection.ts` to withhold organization-level retries only when both +credentials have the same known workspace account id. A stored Pool or main-pool alternate +supplies that id directly; a request-owned `main` alternate is bound by the caller credential's +own `chatgpt-account-id` via `callerCodexWorkspaceAccountId`. Project exhaustion remains +retryable because no project identity is available. Credentials in distinct or unknown +workspaces therefore retain failover, while a proven same-workspace move cannot pay a second +cold prompt prefix for no new capacity. A suppressed move still records the normalized 429/402 +on the refused account, so a 5xx-wrapped quota body cools it rather than letting its wire +status record as transient. `src/server/responses/passthrough-delivery.ts` applies the response's quota headers to the serving account and records the 429 outcome on the ordinary delivery path, so the account still earns its cooldown and leaves the selection pool. The gate fails closed — an empty, truncated, unparseable, duplicate-keyed or aborted body keeps the broad diff --git a/tests/responses/responses-compaction-routing.test.ts b/tests/responses/responses-compaction-routing.test.ts index f111aafe71b..bf2a4555991 100644 --- a/tests/responses/responses-compaction-routing.test.ts +++ b/tests/responses/responses-compaction-routing.test.ts @@ -1330,6 +1330,46 @@ describe("compact alternate-account attempt (#913)", () => { expect(getCodexUpstreamHealth("pool-b")).toBeNull(); }); }); + + test(`a same-workspace alternate is withheld for a scoped ${rejection} refusal`, async () => { + await withPoolEnv(`ocx-compact-same-scope-${rejection}-`, async config => { + // pool-b shares pool-a's workspace: an organization-scoped exhaustion binds + // every credential in that workspace, so the alternate send cannot pay. + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-access-token", + refreshToken: "pool-b-refresh-token", + expiresAt: Date.now() + 300_000, + chatgptAccountId: "pool_acc_a", + }); + const bearers: string[] = []; + const accountIds: string[] = []; + const body = JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }); + globalThis.fetch = (async (_url: string, init?: RequestInit) => { + const headers = new Headers(init?.headers); + bearers.push(headers.get("authorization") ?? ""); + accountIds.push(headers.get("chatgpt-account-id") ?? ""); + return new Response(body, { + status: rejection, + headers: { "content-type": "application/json", "retry-after": "42" }, + }); + }) as typeof fetch; + + const res = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + + expect(bearers).toEqual(["Bearer pool-a-access-token"]); + expect(accountIds).toEqual(["pool_acc_a"]); + expect(res.status).toBe(rejection); + }); + }); } test("a native-main drain starting between attempts preserves the first rejection", async () => { diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index aec00ee0be2..fd6abba09f8 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -3288,6 +3288,101 @@ describe("server local API auth", () => { { timeout: SERVER_BUDGET_MS }, ); + test.each([429, 402] as const)( + "a same-workspace caller main is bound by its workspace id and never sees a %i scoped refusal", + async rejection => { + // The alternate resolved here is the request's own main credential: it has no + // stored account id, so the scope gate can only bind it by the workspace id the + // caller credential would materialize upstream. + const model = "gpt-daybreak-blue-latest"; + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + { status: rejection, headers: { "content-type": "application/json", "retry-after": "60" } }, + ), { + secondAccount: false, + modelRosterByAccount: { "acct-pool-a": [model] }, + }); + try { + const response = await harness.request({ + model, + headers: { "chatgpt-account-id": "acct-pool-a" }, + }); + expect(response.status).toBe(rejection); + expect(harness.dispatches).toEqual(["acct-pool-a"]); + } finally { + await stopPoolRetryHarness(harness); + } + }, + { timeout: SERVER_BUDGET_MS }, + ); + + test("a same-workspace caller main is also bound by the bearer token's account claim", async () => { + const model = "gpt-daybreak-blue-latest"; + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + { status: 429, headers: { "content-type": "application/json", "retry-after": "60" } }, + ), { + secondAccount: false, + modelRosterByAccount: { "acct-pool-a": [model] }, + }); + try { + const response = await harness.request({ + model, + headers: { + authorization: `Bearer ${fakeChatGptJwt({ chatgpt_account_id: "acct-pool-a" })}`, + }, + }); + expect(response.status).toBe(429); + expect(harness.dispatches).toEqual(["acct-pool-a"]); + } finally { + await stopPoolRetryHarness(harness); + } + }, { timeout: SERVER_BUDGET_MS }); + + test("a suppressed 5xx-wrapped scoped refusal still records its normalized quota outcome", async () => { + // ChatGPT sometimes wraps quota exhaustion in a generic 5xx. Suppressing the + // same-workspace alternate must still record the normalized 429 on the refused + // account — otherwise it earns only a transient failure and stays selectable. + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + // No Retry-After: the send layer honours it as a real wait, so the cooldown must + // come from the normalized quota record's default, not the wire header. + { status: 502, headers: { "content-type": "application/json" } }, + )); + try { + // pool-b shares pool-a's workspace, so the resolved alternate is suppressed. + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-a", + }); + const response = await harness.request(); + expect(response.status).toBe(502); + expect(harness.dispatches).not.toContain("acct-pool-b"); + const health = getCodexUpstreamHealth("pool-a"); + expect(health).toMatchObject({ cooldownSource: "default" }); + expect(health?.cooldownUntil).toBeGreaterThan(Date.now()); + } finally { + await stopPoolRetryHarness(harness); + } + }, { timeout: SERVER_BUDGET_MS }); + test("#584: Retry-After cools the first account even when its account retry fails", async () => { const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" ? new Response(JSON.stringify({ error: { message: "rate limited" } }), { From b391c9939f600962de7ef3f7688b009490f029f2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 02:27:38 +0000 Subject: [PATCH 4/8] test(server): move scoped-quota auth cases into a sibling file under the size cap server-auth.test.ts grew to 4684 against a 4589 baseline cap, so the file-size ratchet failed shard 3/4. The three scoped-quota suppression cases move byte-for-byte into server-auth-scoped-quota.test.ts, and the pool-retry harness they share is extracted to tests/helpers/pool-retry-harness.ts (per-run OPENCODEX_HOME dir, so each importing file keeps its own module state under bun test --isolate). Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/test-layout/layout.json | 1 + tests/fixtures/test-layout-expected.json | 1 + tests/helpers/pool-retry-harness.ts | 245 +++++++++++++ tests/server/server-auth-scoped-quota.test.ts | 144 ++++++++ tests/server/server-auth.test.ts | 329 +----------------- 5 files changed, 403 insertions(+), 317 deletions(-) create mode 100644 tests/helpers/pool-retry-harness.ts create mode 100644 tests/server/server-auth-scoped-quota.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 6b8712e526f..d2f938ea361 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1342,6 +1342,7 @@ "self-launch-argv.test.ts": "lib", "server-403-permission-e2e.test.ts": "server", "server-agent-task-recovery-replay.test.ts": "server", + "server-auth-scoped-quota.test.ts": "server", "server-auth.test.ts": "server", "server-background-lifecycle.test.ts": "server", "server-clickjacking-headers.test.ts": "server", diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index c27b6771458..46c24adaea8 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1171,6 +1171,7 @@ "self-launch-argv.test.ts": "lib", "server-403-permission-e2e.test.ts": "server", "server-agent-task-recovery-replay.test.ts": "server", + "server-auth-scoped-quota.test.ts": "server", "server-auth.test.ts": "server", "server-background-lifecycle.test.ts": "server", "server-clickjacking-headers.test.ts": "server", diff --git a/tests/helpers/pool-retry-harness.ts b/tests/helpers/pool-retry-harness.ts new file mode 100644 index 00000000000..28f44589712 --- /dev/null +++ b/tests/helpers/pool-retry-harness.ts @@ -0,0 +1,245 @@ +import { existsSync, mkdirSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { + clearAccountNeedsReauth, + clearAccountQuota, + markAccountNeedsReauth, + updateAccountQuota, +} from "../../src/codex/auth-api"; +import { resetCodexModelEntitlementCacheForTests } from "../../src/codex/model-entitlements"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; +import { clearCodexWebSocketRegistry } from "../../src/codex/websocket-registry"; +import { saveConfig } from "../../src/config"; +import { startServer } from "../../src/server"; +import { clearRequestLogsForTests } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { removeTreeWithRetry } from "./remove-tree"; + +const originalGlobalFetch = globalThis.fetch; + +// A per-run directory, not a fixed path, for the same reason server-auth.test.ts gives: +// `bun test --isolate` gives each file its own module registry but all files share one +// filesystem, so a literal here would collide with whichever file imported this harness. +export const POOL_RETRY_TEST_DIR = mkdtempSync(join(tmpdir(), "ocx-pool-retry-")); + +export const canonicalDirect = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", +} as const; + +export function redirectCanonicalCodexTo(baseUrl: string): void { + const prefix = "/backend-api/codex"; + const currentWebSocket = globalThis.WebSocket; + // These fixtures serve HTTP/SSE only. Refuse the native upstream upgrade + // deterministically so its existing SSE fallback stays on the mocked fetch; + // downstream loopback WebSockets and other destinations remain real. + globalThis.WebSocket = new Proxy(currentWebSocket, { + construct(target, args, newTarget) { + const url = new URL(String(args[0])); + if (url.protocol === "wss:" && url.hostname === "chatgpt.com" + && (url.pathname === prefix || url.pathname.startsWith(`${prefix}/`))) { + throw new Error("HTTP-only Codex fixture rejects native upstream WebSocket"); + } + return Reflect.construct(target, args, newTarget); + }, + }); + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(requestUrl); + if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { + const target = new URL(`${url.pathname.slice(prefix.length)}${url.search}`, baseUrl); + return originalGlobalFetch(target, init); + } + return originalGlobalFetch(input, init); + }) as typeof fetch; +} + +export const POOL_RETRY_MODEL = "gpt-5.5"; + +export function unsupportedModelBody(model = POOL_RETRY_MODEL): string { + return JSON.stringify({ + detail: `The '${model}' model is not supported when using Codex with a ChatGPT account.`, + }); +} + +export type PoolRetryHarness = { + config: OcxConfig; + dispatches: string[]; + request: (init?: { + stream?: boolean; + signal?: AbortSignal; + model?: string; + path?: "/v1/responses" | "/v1/responses/compact"; + callerBearer?: boolean; + headers?: Record; + extraBody?: Record; + }) => Promise; + restoreFetch: () => void; + server: ReturnType; + upstream: ReturnType; +}; + +async function removeTestDirBestEffort(dir: string): Promise { + if (!existsSync(dir)) return; + // Windows can keep the prior harness's ACL/icacls handles for a beat after + // stop; a single EBUSY must not take down the rest of the file. + for (let attempt = 0; attempt < 8; attempt++) { + try { + removeTreeWithRetry(dir); + return; + } catch (err) { + const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; + if (code !== "EBUSY" && code !== "EPERM" && code !== "ENOTEMPTY") throw err; + await Bun.sleep(25 * (attempt + 1)); + } + } + removeTreeWithRetry(dir); +} + +export async function startPoolRetryHarness( + reply: (accountId: string, request: Request) => Response | Promise, + options: { + secondAccount?: boolean; + streamMode?: "legacy-tee" | "eager-relay"; + accountMode?: "direct" | "pool"; + activeAccountId?: string; + accountNamespaces?: Record; + noVisionModels?: string[]; + visionSidecarModel?: string; + websockets?: boolean; + forwardApiKey?: string; + pausedAccountIds?: string[]; + reauthAccountIds?: string[]; + omitCredentialAccountIds?: string[]; + combos?: OcxConfig["combos"]; + modelRosterByAccount?: Record; + } = {}, +): Promise { + await removeTestDirBestEffort(POOL_RETRY_TEST_DIR); + mkdirSync(POOL_RETRY_TEST_DIR, { recursive: true }); + process.env.OPENCODEX_HOME = POOL_RETRY_TEST_DIR; + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearAccountQuota(); + resetCodexModelEntitlementCacheForTests(); + clearRequestLogsForTests(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + // The registry is process-global and survives a harness teardown. WS-REBIND-01 + // asserts exact per-account socket counts, so a socket leaked by any earlier test + // in this file shifts its snapshots and fails it in milliseconds — which reads as + // a flake next to the timeouts, but is ordinary shared state. Reset it with the + // rest rather than leaving one of six kinds of state uncleaned. + clearCodexWebSocketRegistry(); + + const dispatches: string[] = []; + const upstream = Bun.serve({ + port: 0, + async fetch(request) { + const accountId = request.headers.get("chatgpt-account-id") ?? "missing"; + if (new URL(request.url).pathname === "/models") { + return Response.json({ + models: (options.modelRosterByAccount?.[accountId] ?? []).map(slug => ({ + slug, + supported_in_api: true, + visibility: "list", + })), + }); + } + dispatches.push(accountId); + return reply(accountId, request); + }, + }); + redirectCanonicalCodexTo(upstream.url.toString()); + const redirectedFetch = globalThis.fetch; + + const secondAccount = options.secondAccount ?? true; + const config = { + port: 0, + defaultProvider: "openai", + openaiProviderTierVersion: 2, + providers: { + openai: { + ...canonicalDirect, + codexAccountMode: options.accountMode ?? "pool", + ...(options.noVisionModels ? { noVisionModels: options.noVisionModels } : {}), + ...(options.forwardApiKey ? { apiKey: options.forwardApiKey } : {}), + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, + ...(secondAccount + ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] + : []), + ], + activeCodexAccountId: options.activeAccountId ?? "pool-a", + ...(options.accountNamespaces ? { codexAccountNamespaces: options.accountNamespaces } : {}), + ...(options.pausedAccountIds ? { pausedCodexAccountIds: options.pausedAccountIds } : {}), + ...(options.visionSidecarModel ? { visionSidecar: { model: options.visionSidecarModel } } : {}), + ...(options.websockets ? { websockets: true } : {}), + ...(options.streamMode ? { streamMode: options.streamMode } : {}), + ...(options.combos ? { combos: options.combos } : {}), + } as OcxConfig; + saveConfig(config); + if (!options.omitCredentialAccountIds?.includes("pool-a")) { + saveCodexAccountCredential("pool-a", { + accessToken: "pool-a-token", + refreshToken: "pool-a-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-a", + }); + } + updateAccountQuota("pool-a", 10); + if (secondAccount) { + if (!options.omitCredentialAccountIds?.includes("pool-b")) { + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + } + updateAccountQuota("pool-b", 20); + } + for (const accountId of options.reauthAccountIds ?? []) markAccountNeedsReauth(accountId); + + const server = startServer(0); + return { + config, + dispatches, + restoreFetch: () => { + if (globalThis.fetch === redirectedFetch) globalThis.fetch = originalGlobalFetch; + }, + server, + upstream, + request: ({ + stream = false, + signal, + model = POOL_RETRY_MODEL, + path = "/v1/responses", + callerBearer = true, + headers = {}, + extraBody = {}, + } = {}) => originalGlobalFetch(new URL(path, server.url), { + method: "POST", + headers: { + "content-type": "application/json", + ...(callerBearer ? { authorization: "Bearer inbound-token" } : {}), + ...headers, + }, + body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream, ...extraBody }), + signal, + }), + }; +} + +export async function stopPoolRetryHarness(harness: PoolRetryHarness): Promise { + harness.restoreFetch(); + await harness.server.stop(true); + await harness.upstream.stop(true); +} diff --git a/tests/server/server-auth-scoped-quota.test.ts b/tests/server/server-auth-scoped-quota.test.ts new file mode 100644 index 00000000000..297fdc0b9fd --- /dev/null +++ b/tests/server/server-auth-scoped-quota.test.ts @@ -0,0 +1,144 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearAccountNeedsReauth, clearAccountQuota } from "../../src/codex/auth-api"; +import { resetCodexModelEntitlementCacheForTests } from "../../src/codex/model-entitlements"; +import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth } from "../../src/codex/routing"; +import { resetDebugLogBufferForTests } from "../../src/lib/debug-log-buffer"; +import { resetDebugSettingsForTests } from "../../src/lib/debug-settings"; +import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { + POOL_RETRY_TEST_DIR, + startPoolRetryHarness, + stopPoolRetryHarness, +} from "../helpers/pool-retry-harness"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { SERVER_BUDGET_MS } from "../helpers/test-budget"; + +const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; +const previousOpencodexHome = process.env.OPENCODEX_HOME; +const originalGlobalFetch = globalThis.fetch; +const originalGlobalWebSocket = globalThis.WebSocket; +let isolatedCodexHome: IsolatedCodexHome | null = null; + +beforeEach(() => { + isolatedCodexHome = installIsolatedCodexHome("ocx-server-auth-codex-"); +}); + +afterEach(() => { + globalThis.fetch = originalGlobalFetch; + globalThis.WebSocket = originalGlobalWebSocket; + if (previousApiToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; + else process.env.OPENCODEX_API_AUTH_TOKEN = previousApiToken; + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + clearAccountQuota(); + resetCodexModelEntitlementCacheForTests(); + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); + if (existsSync(POOL_RETRY_TEST_DIR)) removeTreeWithRetry(POOL_RETRY_TEST_DIR); +}); + +describe("server local API auth", () => { + test.each([429, 402] as const)( + "a same-workspace caller main is bound by its workspace id and never sees a %i scoped refusal", + async rejection => { + // The alternate resolved here is the request's own main credential: it has no + // stored account id, so the scope gate can only bind it by the workspace id the + // caller credential would materialize upstream. + const model = "gpt-daybreak-blue-latest"; + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + { status: rejection, headers: { "content-type": "application/json", "retry-after": "60" } }, + ), { + secondAccount: false, + modelRosterByAccount: { "acct-pool-a": [model] }, + }); + try { + const response = await harness.request({ + model, + headers: { "chatgpt-account-id": "acct-pool-a" }, + }); + expect(response.status).toBe(rejection); + expect(harness.dispatches).toEqual(["acct-pool-a"]); + } finally { + await stopPoolRetryHarness(harness); + } + }, + { timeout: SERVER_BUDGET_MS }, + ); + + test("a same-workspace caller main is also bound by the bearer token's account claim", async () => { + const model = "gpt-daybreak-blue-latest"; + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + { status: 429, headers: { "content-type": "application/json", "retry-after": "60" } }, + ), { + secondAccount: false, + modelRosterByAccount: { "acct-pool-a": [model] }, + }); + try { + const response = await harness.request({ + model, + headers: { + authorization: `Bearer ${fakeChatGptJwt({ chatgpt_account_id: "acct-pool-a" })}`, + }, + }); + expect(response.status).toBe(429); + expect(harness.dispatches).toEqual(["acct-pool-a"]); + } finally { + await stopPoolRetryHarness(harness); + } + }, { timeout: SERVER_BUDGET_MS }); + + test("a suppressed 5xx-wrapped scoped refusal still records its normalized quota outcome", async () => { + // ChatGPT sometimes wraps quota exhaustion in a generic 5xx. Suppressing the + // same-workspace alternate must still record the normalized 429 on the refused + // account — otherwise it earns only a transient failure and stays selectable. + const harness = await startPoolRetryHarness(() => new Response( + JSON.stringify({ + error: { + code: "organization_spend_limit_exceeded", + message: "The usage limit has been reached", + }, + }), + // No Retry-After: the send layer honours it as a real wait, so the cooldown must + // come from the normalized quota record's default, not the wire header. + { status: 502, headers: { "content-type": "application/json" } }, + )); + try { + // pool-b shares pool-a's workspace, so the resolved alternate is suppressed. + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-a", + }); + const response = await harness.request(); + expect(response.status).toBe(502); + expect(harness.dispatches).not.toContain("acct-pool-b"); + const health = getCodexUpstreamHealth("pool-a"); + expect(health).toMatchObject({ cooldownSource: "default" }); + expect(health?.cooldownUntil).toBeGreaterThan(Date.now()); + } finally { + await stopPoolRetryHarness(harness); + } + }, { timeout: SERVER_BUDGET_MS }); +}); diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index fd6abba09f8..e07f119fb49 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -7,7 +7,7 @@ import { request as httpRequest } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; -import { clearCodexWebSocketRegistry, getTrackedCodexWebSocketCountForAccount } from "../../src/codex/websocket-registry"; +import { getTrackedCodexWebSocketCountForAccount } from "../../src/codex/websocket-registry"; import { INTERNAL_DEADLINE_MS, SERVER_BUDGET_MS } from "../helpers/test-budget"; import { clearAccountNeedsReauth, clearAccountQuota, getAccountQuota, isAccountNeedsReauth, markAccountNeedsReauth, updateAccountQuota } from "../../src/codex/auth-api"; import { @@ -55,6 +55,15 @@ import { resetDebugSettingsForTests, setDebugSettings } from "../../src/lib/debu import { watchdogMs } from "../helpers/ci-watchdog"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { deferredResetSseUpstream } from "../helpers/deferred-reset-sse-upstream"; +import { + POOL_RETRY_MODEL, + POOL_RETRY_TEST_DIR, + canonicalDirect, + redirectCanonicalCodexTo, + startPoolRetryHarness, + stopPoolRetryHarness, + unsupportedModelBody, +} from "../helpers/pool-retry-harness"; const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; const previousOpencodexHome = process.env.OPENCODEX_HOME; const originalGlobalFetch = globalThis.fetch; @@ -112,46 +121,12 @@ function managementHeaders(initial?: HeadersInit): Headers { return headers; } -const canonicalDirect = { - adapter: "openai-responses", - baseUrl: "https://chatgpt.com/backend-api/codex", - authMode: "forward", - codexAccountMode: "direct", -} as const; - function poolProviders(): OcxConfig["providers"] { return { openai: { ...canonicalDirect, codexAccountMode: "pool" }, }; } -function redirectCanonicalCodexTo(baseUrl: string): void { - const prefix = "/backend-api/codex"; - const currentWebSocket = globalThis.WebSocket; - // These fixtures serve HTTP/SSE only. Refuse the native upstream upgrade - // deterministically so its existing SSE fallback stays on the mocked fetch; - // downstream loopback WebSockets and other destinations remain real. - globalThis.WebSocket = new Proxy(currentWebSocket, { - construct(target, args, newTarget) { - const url = new URL(String(args[0])); - if (url.protocol === "wss:" && url.hostname === "chatgpt.com" - && (url.pathname === prefix || url.pathname.startsWith(`${prefix}/`))) { - throw new Error("HTTP-only Codex fixture rejects native upstream WebSocket"); - } - return Reflect.construct(target, args, newTarget); - }, - }); - globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { - const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; - const url = new URL(requestUrl); - if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { - const target = new URL(`${url.pathname.slice(prefix.length)}${url.search}`, baseUrl); - return originalGlobalFetch(target, init); - } - return originalGlobalFetch(input, init); - }) as typeof fetch; -} - function stubModelDiscoveryFor(...origins: string[]): void { const allowed = new Set(origins); globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { @@ -186,194 +161,9 @@ afterEach(() => { resetDebugSettingsForTests(); resetDebugLogBufferForTests(); if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR); + if (existsSync(POOL_RETRY_TEST_DIR)) removeTreeWithRetry(POOL_RETRY_TEST_DIR); }); -const POOL_RETRY_MODEL = "gpt-5.5"; - -function unsupportedModelBody(model = POOL_RETRY_MODEL): string { - return JSON.stringify({ - detail: `The '${model}' model is not supported when using Codex with a ChatGPT account.`, - }); -} - -type PoolRetryHarness = { - config: OcxConfig; - dispatches: string[]; - request: (init?: { - stream?: boolean; - signal?: AbortSignal; - model?: string; - path?: "/v1/responses" | "/v1/responses/compact"; - callerBearer?: boolean; - headers?: Record; - extraBody?: Record; - }) => Promise; - restoreFetch: () => void; - server: ReturnType; - upstream: ReturnType; -}; - -async function removeTestDirBestEffort(dir: string): Promise { - if (!existsSync(dir)) return; - // Windows can keep the prior harness's ACL/icacls handles for a beat after - // stop; a single EBUSY must not take down the rest of the file. - for (let attempt = 0; attempt < 8; attempt++) { - try { - removeTreeWithRetry(dir); - return; - } catch (err) { - const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; - if (code !== "EBUSY" && code !== "EPERM" && code !== "ENOTEMPTY") throw err; - await Bun.sleep(25 * (attempt + 1)); - } - } - removeTreeWithRetry(dir); -} - -async function startPoolRetryHarness( - reply: (accountId: string, request: Request) => Response | Promise, - options: { - secondAccount?: boolean; - streamMode?: "legacy-tee" | "eager-relay"; - accountMode?: "direct" | "pool"; - activeAccountId?: string; - accountNamespaces?: Record; - noVisionModels?: string[]; - visionSidecarModel?: string; - websockets?: boolean; - forwardApiKey?: string; - pausedAccountIds?: string[]; - reauthAccountIds?: string[]; - omitCredentialAccountIds?: string[]; - combos?: OcxConfig["combos"]; - modelRosterByAccount?: Record; - } = {}, -): Promise { - await removeTestDirBestEffort(TEST_DIR); - mkdirSync(TEST_DIR, { recursive: true }); - process.env.OPENCODEX_HOME = TEST_DIR; - clearCodexUpstreamHealth(); - clearThreadAccountMap(); - clearAccountQuota(); - resetCodexModelEntitlementCacheForTests(); - clearRequestLogsForTests(); - clearAccountNeedsReauth("pool-a"); - clearAccountNeedsReauth("pool-b"); - // The registry is process-global and survives a harness teardown. WS-REBIND-01 - // asserts exact per-account socket counts, so a socket leaked by any earlier test - // in this file shifts its snapshots and fails it in milliseconds — which reads as - // a flake next to the timeouts, but is ordinary shared state. Reset it with the - // rest rather than leaving one of six kinds of state uncleaned. - clearCodexWebSocketRegistry(); - - const dispatches: string[] = []; - const upstream = Bun.serve({ - port: 0, - async fetch(request) { - const accountId = request.headers.get("chatgpt-account-id") ?? "missing"; - if (new URL(request.url).pathname === "/models") { - return Response.json({ - models: (options.modelRosterByAccount?.[accountId] ?? []).map(slug => ({ - slug, - supported_in_api: true, - visibility: "list", - })), - }); - } - dispatches.push(accountId); - return reply(accountId, request); - }, - }); - redirectCanonicalCodexTo(upstream.url.toString()); - const redirectedFetch = globalThis.fetch; - - const secondAccount = options.secondAccount ?? true; - const config = { - port: 0, - defaultProvider: "openai", - openaiProviderTierVersion: 2, - providers: { - openai: { - ...canonicalDirect, - codexAccountMode: options.accountMode ?? "pool", - ...(options.noVisionModels ? { noVisionModels: options.noVisionModels } : {}), - ...(options.forwardApiKey ? { apiKey: options.forwardApiKey } : {}), - }, - }, - codexAccounts: [ - { id: "main", email: "main@example.test", isMain: true }, - { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, - ...(secondAccount - ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] - : []), - ], - activeCodexAccountId: options.activeAccountId ?? "pool-a", - ...(options.accountNamespaces ? { codexAccountNamespaces: options.accountNamespaces } : {}), - ...(options.pausedAccountIds ? { pausedCodexAccountIds: options.pausedAccountIds } : {}), - ...(options.visionSidecarModel ? { visionSidecar: { model: options.visionSidecarModel } } : {}), - ...(options.websockets ? { websockets: true } : {}), - ...(options.streamMode ? { streamMode: options.streamMode } : {}), - ...(options.combos ? { combos: options.combos } : {}), - } as OcxConfig; - saveConfig(config); - if (!options.omitCredentialAccountIds?.includes("pool-a")) { - saveCodexAccountCredential("pool-a", { - accessToken: "pool-a-token", - refreshToken: "pool-a-refresh", - expiresAt: Date.now() + 10 * 60_000, - chatgptAccountId: "acct-pool-a", - }); - } - updateAccountQuota("pool-a", 10); - if (secondAccount) { - if (!options.omitCredentialAccountIds?.includes("pool-b")) { - saveCodexAccountCredential("pool-b", { - accessToken: "pool-b-token", - refreshToken: "pool-b-refresh", - expiresAt: Date.now() + 10 * 60_000, - chatgptAccountId: "acct-pool-b", - }); - } - updateAccountQuota("pool-b", 20); - } - for (const accountId of options.reauthAccountIds ?? []) markAccountNeedsReauth(accountId); - - const server = startServer(0); - return { - config, - dispatches, - restoreFetch: () => { - if (globalThis.fetch === redirectedFetch) globalThis.fetch = originalGlobalFetch; - }, - server, - upstream, - request: ({ - stream = false, - signal, - model = POOL_RETRY_MODEL, - path = "/v1/responses", - callerBearer = true, - headers = {}, - extraBody = {}, - } = {}) => originalGlobalFetch(new URL(path, server.url), { - method: "POST", - headers: { - "content-type": "application/json", - ...(callerBearer ? { authorization: "Bearer inbound-token" } : {}), - ...headers, - }, - body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream, ...extraBody }), - signal, - }), - }; -} - -async function stopPoolRetryHarness(harness: PoolRetryHarness): Promise { - harness.restoreFetch(); - await harness.server.stop(true); - await harness.upstream.stop(true); -} - function rejectionResponse(body: BodyInit, headers: Record = {}): Response { return new Response(body, { status: 400, @@ -3288,101 +3078,6 @@ describe("server local API auth", () => { { timeout: SERVER_BUDGET_MS }, ); - test.each([429, 402] as const)( - "a same-workspace caller main is bound by its workspace id and never sees a %i scoped refusal", - async rejection => { - // The alternate resolved here is the request's own main credential: it has no - // stored account id, so the scope gate can only bind it by the workspace id the - // caller credential would materialize upstream. - const model = "gpt-daybreak-blue-latest"; - const harness = await startPoolRetryHarness(() => new Response( - JSON.stringify({ - error: { - code: "organization_spend_limit_exceeded", - message: "The usage limit has been reached", - }, - }), - { status: rejection, headers: { "content-type": "application/json", "retry-after": "60" } }, - ), { - secondAccount: false, - modelRosterByAccount: { "acct-pool-a": [model] }, - }); - try { - const response = await harness.request({ - model, - headers: { "chatgpt-account-id": "acct-pool-a" }, - }); - expect(response.status).toBe(rejection); - expect(harness.dispatches).toEqual(["acct-pool-a"]); - } finally { - await stopPoolRetryHarness(harness); - } - }, - { timeout: SERVER_BUDGET_MS }, - ); - - test("a same-workspace caller main is also bound by the bearer token's account claim", async () => { - const model = "gpt-daybreak-blue-latest"; - const harness = await startPoolRetryHarness(() => new Response( - JSON.stringify({ - error: { - code: "organization_spend_limit_exceeded", - message: "The usage limit has been reached", - }, - }), - { status: 429, headers: { "content-type": "application/json", "retry-after": "60" } }, - ), { - secondAccount: false, - modelRosterByAccount: { "acct-pool-a": [model] }, - }); - try { - const response = await harness.request({ - model, - headers: { - authorization: `Bearer ${fakeChatGptJwt({ chatgpt_account_id: "acct-pool-a" })}`, - }, - }); - expect(response.status).toBe(429); - expect(harness.dispatches).toEqual(["acct-pool-a"]); - } finally { - await stopPoolRetryHarness(harness); - } - }, { timeout: SERVER_BUDGET_MS }); - - test("a suppressed 5xx-wrapped scoped refusal still records its normalized quota outcome", async () => { - // ChatGPT sometimes wraps quota exhaustion in a generic 5xx. Suppressing the - // same-workspace alternate must still record the normalized 429 on the refused - // account — otherwise it earns only a transient failure and stays selectable. - const harness = await startPoolRetryHarness(() => new Response( - JSON.stringify({ - error: { - code: "organization_spend_limit_exceeded", - message: "The usage limit has been reached", - }, - }), - // No Retry-After: the send layer honours it as a real wait, so the cooldown must - // come from the normalized quota record's default, not the wire header. - { status: 502, headers: { "content-type": "application/json" } }, - )); - try { - // pool-b shares pool-a's workspace, so the resolved alternate is suppressed. - saveCodexAccountCredential("pool-b", { - accessToken: "pool-b-token", - refreshToken: "pool-b-refresh", - expiresAt: Date.now() + 10 * 60_000, - chatgptAccountId: "acct-pool-a", - }); - const response = await harness.request(); - expect(response.status).toBe(502); - expect(harness.dispatches).not.toContain("acct-pool-b"); - const health = getCodexUpstreamHealth("pool-a"); - expect(health).toMatchObject({ cooldownSource: "default" }); - expect(health?.cooldownUntil).toBeGreaterThan(Date.now()); - } finally { - await stopPoolRetryHarness(harness); - } - }, { timeout: SERVER_BUDGET_MS }); - test("#584: Retry-After cools the first account even when its account retry fails", async () => { const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" ? new Response(JSON.stringify({ error: { message: "rate limited" } }), { @@ -3774,7 +3469,7 @@ describe("server local API auth", () => { test("valid JSON wrong top-level shape never authorizes a pool retry", async () => { // One harness, five bodies — same reason as the sibling above. Each - // startPoolRetryHarness() wipes and recreates TEST_DIR, binds a server, and + // startPoolRetryHarness() wipes and recreates its OPENCODEX_HOME directory, binds a server, and // redirects global fetch; five of those did not fit Bun's 5s default on a // Windows runner, and the request still in flight when the budget expired // raced the next test through that same global fetch. From c52b64bb250fb4be6cc2a0c50ce0b77c2df9be21 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 02:31:31 +0000 Subject: [PATCH 5/8] fix(codex): re-check abort after the scoped-quota body read The workspace classification in shouldRetryCodexScopedQuotaOnAlternate reads the first response body asynchronously, so a client disconnect can land after the earlier abort check but before the branch records the first account, cancels its body, and sends the alternate. Re-check the abort signal immediately after the await in both paths: compact returns the 499 client_cancelled response after releasing the alternate lease, and the regular path releases its permit and lease and returns no-alternate while still recording the first account's real outcome. Co-Authored-By: Epinephrine --- src/server/responses/compact.ts | 8 ++++++++ src/server/responses/core-codex-account.ts | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index ce9a74c1d52..a84e78d5dd7 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1229,6 +1229,14 @@ export async function handleResponsesCompact( : callerCodexWorkspaceAccountId(req.headers), req.signal, ); + // The scope check reads the rejection body asynchronously — the same window the + // comment above covers. Re-check before the branch below records A, cancels its + // body, and sends B for a caller that is gone. + if (alternate && req.signal.aborted) { + releaseCodexAuthContextProbeLease(alternate.authCtx); + recordCompactPoolOutcome(outcomeCtx, 499); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } if (alternate && sharedWorkspaceScope) { releaseCodexAuthContextProbeLease(alternate.authCtx); } diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 076bf14cbaa..c78733dd943 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -679,6 +679,17 @@ export async function retryCodexPoolOnAlternateAccount( return { kind: "no-alternate" }; } + // The scope classification above reads the rejection body asynchronously, so the + // request may have been cancelled while it ran. Re-check before the send below + // mutates routing state or spends the alternate on a caller that is gone. + if (options.abortSignal?.aborted) { + recordWrappedQuotaOutcome(); + recordUnmovedTransientOutcome(); + accountMovePermit?.release(); + releaseCodexAuthContextProbeLease(retryAuthCtx); + return { kind: "no-alternate" }; + } + const quotaMeta = { ...codexQuotaOutcomeMeta(firstResponse), ...(await codexDenialOutcomeMeta(firstResponse)) }; if (outcomeStatus === 429 || outcomeStatus === 402) { const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); From 813efcc2e21a46b4f91a32c5643cbb89bcbe097e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 02:34:46 +0000 Subject: [PATCH 6/8] fix(codex): release the discarded compact rejection body on abort The 499 exits around the scoped-quota classification return a fresh response while the first rejection's body is still open; cancel it so the abandoned upstream connection and tee resources are released. Co-Authored-By: Epinephrine --- src/server/responses/compact.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index a84e78d5dd7..eae34333144 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1213,6 +1213,7 @@ export async function handleResponsesCompact( if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); recordCompactPoolOutcome(outcomeCtx, 499); + await upstream.body?.cancel().catch(() => undefined); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } // The same scope binding the regular path applies: an organization-scoped @@ -1235,6 +1236,7 @@ export async function handleResponsesCompact( if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); recordCompactPoolOutcome(outcomeCtx, 499); + await upstream.body?.cancel().catch(() => undefined); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } if (alternate && sharedWorkspaceScope) { From 2d1ee699c52b6e7a628916076d991672d2d353c3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 02:36:26 +0000 Subject: [PATCH 7/8] fix(codex): keep compact abort cleanup off the return path upstream.body.cancel() can wait on a custom or stalled source; awaiting it at the abort checkpoints would park the 499 reply on cleanup. Fire it with the request's abort reason and swallow rejection, the same best-effort shape bufferCompactResponse already uses. Co-Authored-By: Epinephrine --- src/server/responses/compact.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index eae34333144..195c26230e5 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1213,7 +1213,7 @@ export async function handleResponsesCompact( if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); recordCompactPoolOutcome(outcomeCtx, 499); - await upstream.body?.cancel().catch(() => undefined); + void upstream.body?.cancel(req.signal.reason).catch(() => undefined); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } // The same scope binding the regular path applies: an organization-scoped @@ -1236,7 +1236,7 @@ export async function handleResponsesCompact( if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); recordCompactPoolOutcome(outcomeCtx, 499); - await upstream.body?.cancel().catch(() => undefined); + void upstream.body?.cancel(req.signal.reason).catch(() => undefined); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } if (alternate && sharedWorkspaceScope) { From bce996a27359d2e5123892408582b7d7d9ea91d9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 09:25:30 +0000 Subject: [PATCH 8/8] ci: retrigger macos 1/2 (wedged spawn stall in codex-app-server-processes.test.ts) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>