From 154e8f8571514cb6110f842e2a2d997063f83735 Mon Sep 17 00:00:00 2001 From: Fred Amartey <43480311+FredAmartey@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:35:19 -0400 Subject: [PATCH 1/3] fix(responses): let retryOnReset replace a Codex WebSocket send that died unanswered A Codex socket that closed or errored under its create frame, before any Responses event, settles as a non-replayable 502 and nothing could send the turn again. That is the same unknown state as an HTTP reset before the head, so the operator's retryOnReset grant now answers it the same way: the exchange marks the settle with the stage it reached, and the passthrough dispatch asks the resend gate once, at the end of the recovery loop, and sends one HTTP replacement. The send budget is checked before the gate claims, and the replacement's answer is settled by the rule fetchWithResetRetry already used, now shared as settleOperatorReplacement. reasoningEffortRejectionText now skips a non-replayable answer, so a spent replacement's effort rejection no longer starts a downgrade send. Refs #4191 --- .../fr/reference/configuration/providers.md | 2 +- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/server.md | 12 +- .../ru/reference/configuration/providers.md | 2 +- .../tr/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- scripts/test-layout/layout.json | 1 + src/lib/request-resend-gate.ts | 7 +- src/lib/upstream-retry.ts | 32 +- src/server/responses/codex-ws-exchange.ts | 25 +- src/server/responses/codex-ws-wire.ts | 31 +- src/server/responses/core-opaque-recovery.ts | 5 +- src/server/responses/passthrough-dispatch.ts | 128 ++++--- structure/transports/responses-failover.md | 34 +- structure/transports/responses-wire-shapes.md | 11 +- tests/fixtures/test-layout-expected.json | 1 + tests/responses/ws-ambiguous-resend.test.ts | 340 ++++++++++++++++++ 20 files changed, 549 insertions(+), 94 deletions(-) create mode 100644 tests/responses/ws-ambiguous-resend.test.ts diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index 04bd23891eb..154bef591c0 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -142,7 +142,7 @@ sauvegarde dont le contenu diffère, puis réécrit en identifiants sans préfix | `responsesSnapshotRepair?` | `boolean` | Réparation côté client désactivée par défaut pour les instantanés du cycle de vie des réponses clairsemés dans SSE et JSON. Remplit les métadonnées d'état canonique, de sortie et d'outil manquantes tandis que l'inspection brute et la persistance restent inchangées. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Fournisseurs à clé API uniquement (`authMode: "key"`). Nouvelle tentative facultative sur la même cible après un 429 : lorsque `retryOn429` est absent, la fonctionnalité est désactivée ; la présence d'un objet l'active, sauf avec `enabled: false`. Après un 429, le proxy attend selon `Retry-After` reçu en amont ou selon l'intervalle fixe, puis relit la requête à l'identique avec la même clé avant tout basculement de clé. Ce comportement couvre la boucle principale de récupération d'un tour textuel, le protocole de transfert Responses, le pont d'images et de vidéos, le service auxiliaire de recherche Web et les continuations du terminal. Seules les réponses HTTP 429 reçues avant le début de la diffusion peuvent être relues ; les transports `runTurn` personnalisés ne font pas partie de la boucle de nouvelle tentative HTTP. `attempts` compte les relectures avec la même clé après le premier 429, soit `attempts` + 1 envois au total, et constitue un budget commun à toute la requête, partagé entre la boucle principale de récupération, la continuation de la garde du terminal et les nouvelles tentatives du pont. L'épuisement de `attempts` arrête uniquement les relectures supplémentaires avec la même clé : le basculement normal de clé ou la gestion de l'erreur finale s'applique ensuite selon les cibles disponibles. Sur le protocole de transfert authentifié par clé, aucun basculement n'est possible ; le 429 final est donc renvoyé sans modification. Codex ne retente jamais lui-même une requête après un 429 : cette option constitue ainsi la seule protection pour les fournisseurs à clé unique. Valeurs par défaut : `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (chaque attente est plafonnée à `maxIntervalMs`, lui-même plafonné à 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Fournisseurs `openai-chat` et `openai-responses` authentifiés par clé uniquement. Les fournisseurs `authMode: "forward"` (le pool de comptes ChatGPT) ne lisent jamais cette option et conservent l'échelle par défaut. Nouvelle tentative facultative pour les états transitoires reçus en amont avant le début de la diffusion (500, 502, 503, 504, 520, 521, 522) : l'absence de l'option la désactive ; la présence d'un objet l'active, sauf avec `enabled: false`. Ce comportement couvre la requête Responses initiale, la continuation de la garde du terminal, le point de terminaison natif `/v1/chat/completions` et les réémissions liées à la récupération après un 429 ou à la récupération de compte. `attempts` représente le nombre TOTAL d'envois en amont autorisés pour une requête, premier envoi compris (de 1 à 10, valeur par défaut : 3). Il constitue un budget commun à la requête, partagé avec la récupération après une réinitialisation de connexion ; ainsi, `3` signifie qu'au plus trois requêtes réelles atteignent le fournisseur. Les attentes utilisent une temporisation exponentielle à base fixe de 400 ms, plafonnée à 5 s, et respectent `Retry-After`. Cette option est distincte de `retryOn429`, qui traite la limitation de débit ; les échecs en cours de diffusion ne sont jamais relus. | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Fournisseurs `openai-responses` natifs uniquement, `authMode: "forward"` compris. Remplacement facultatif d'un envoi qui a échoué alors que l'appelant n'avait rien observé : l'absence de l'option la désactive ; la présence d'un objet l'active, sauf avec `enabled: false`. Couvre les deux étapes ambiguës — une connexion rompue avant tout en-tête de réponse, et un corps SSE rompu après l'en-tête alors qu'il ne portait que des événements de contrôle. Seule une requête autonome est remplacée : `store: false`, `input` complet, ni `previous_response_id`, ni `conversation`, ni `stream_id`, et uniquement des outils exécutés par le client. `replacements` est le nombre d'envois de remplacement qu'UNE requête logique peut effectuer, toutes étapes et tous enfants de combo confondus (de 1 à 2, valeur par défaut : 1). Ce n'est ni un nombre de tentatives par étape ni un budget d'envoi : un remplacement doit toujours tenir dans l'allocation d'envois dont l'étape disposait déjà. Une requête qui a déjà émis une sortie ou un appel d'outil n'est jamais remplacée, quelle que soit cette valeur. L'inférence de remplacement peut tout de même être facturée si l'origine avait déjà démarré la première, d'où la désactivation par défaut. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Fournisseurs `openai-responses` natifs uniquement, `authMode: "forward"` compris. Remplacement facultatif d'un envoi qui a échoué alors que l'appelant n'avait rien observé : l'absence de l'option la désactive ; la présence d'un objet l'active, sauf avec `enabled: false`. Couvre les deux étapes ambiguës — une connexion rompue avant tout en-tête de réponse, et un corps SSE rompu après l'en-tête alors qu'il ne portait que des événements de contrôle. Une WebSocket canonique ChatGPT en amont qui s'est fermée ou a échoué après l'envoi de sa trame de création, avant tout événement Responses, est couverte de la même façon ; son remplacement est envoyé en HTTP. Seule une requête autonome est remplacée : `store: false`, `input` complet, ni `previous_response_id`, ni `conversation`, ni `stream_id`, et uniquement des outils exécutés par le client. `replacements` est le nombre d'envois de remplacement qu'UNE requête logique peut effectuer, toutes étapes et tous enfants de combo confondus (de 1 à 2, valeur par défaut : 1). Ce n'est ni un nombre de tentatives par étape ni un budget d'envoi : un remplacement doit toujours tenir dans l'allocation d'envois dont l'étape disposait déjà. Une requête qui a déjà émis une sortie ou un appel d'outil n'est jamais remplacée, quelle que soit cette valeur. L'inférence de remplacement peut tout de même être facturée si l'origine avait déjà démarré la première, d'où la désactivation par défaut. | | `autoToolChoiceOnlyModels?` | `string[]` | Modèles dont `tool_choice` accepte uniquement `auto` ou `none` ; les choix forcés sont dévalorisés. | | `preserveReasoningContentModels?` | `string[]` | Modèles nécessitant un assistant préalable `reasoning_content` dans l'historique des discussions. Les enregistrements depuis le tableau de bord conservent la liste enregistrée, y compris `[]`. `PATCH /api/providers?name=` accepte un tableau ou `null` pour l'effacer. Une sauvegarde qui déplace le fournisseur vers un autre adaptateur, une autre URL de base ou un autre mode d'authentification ne la conserve pas (voir la section ci-dessous). | | `reasoningDetailsModels?` | `string[]` | Modèles dont le point de terminaison renvoie la réflexion sous forme de tableau structuré `reasoning_details` (MiniMax série M avec `reasoning_split`) ; les deltas de flux sont des instantanés cumulatifs comparés par préfixe, et la réflexion conservée est rejouée sous forme de tableau `reasoning_details` plutôt que de chaîne `reasoning_content`. | diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index e5f7025a0f8..5d49fe60aa3 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -134,7 +134,7 @@ account を削除しても mapping は保持され、同じ id を再追加す | `responsesSnapshotRepair?` | `boolean` | デフォルトで無効のクライアント向け修復です。SSE と JSON の Responses ライフサイクルで欠落した status、output、ツールメタデータを補完し、raw 検査と永続化は変更しません。 | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。`attempts` を使い切っても同一キーでの再送が止まるだけで、通常のキー フェイルオーバーまたは最終エラー処理が利用可能なターゲットに応じて続きます — キー認証の passthrough ワイヤにはフェイルオーバーがないため、使い切った 429 はそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | キー認証の `openai-chat` および `openai-responses` プロバイダーのみ。`authMode: "forward"` のプロバイダー(ChatGPT アカウントプール)はこのオプションを読まず、既定の再試行段数を維持します。ストリーム開始前に上流から返される一時的なステータス(500、502、503、504、520、521、522)に対するオプトインの再試行です。設定がなければ無効で、オブジェクトを指定すると `enabled: false` でない限り有効になります。最初の Responses リクエスト、ターミナルガード継続、ネイティブの `/v1/chat/completions`、および 429/アカウント回復時の再取得が対象です。`attempts` は最初の送信を含め、1 回のリクエストで許可される上流への送信総数です(1~10、デフォルトは 3)。接続リセット回復と共有するリクエスト単位の単一予算であるため、`3` を指定した場合、プロバイダーに到達する実リクエストは最大 3 回です。待機には 400 ms を基準とする固定式の指数バックオフを使用し、上限は 5 秒で、`Retry-After` に従います。レート制限を扱う `retryOn429` とは別の機能であり、ストリーム開始後の失敗は再送されません。 | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | ネイティブ `openai-responses` プロバイダー専用で、`authMode: "forward"` も含みます。呼び出し側が何も観測しないまま失敗した送信を、オプトインで置き換えます。設定がなければ無効で、オブジェクトを指定すると `enabled: false` でない限り有効になります。レスポンスヘッダーが届く前に接続が切れた場合と、ヘッダー後に SSE 本文が制御イベントだけを運んだまま切れた場合の両方が対象です。置き換えるのは自己完結したリクエストだけで、`store: false`、完全な `input`、`previous_response_id` / `conversation` / `stream_id` がないこと、クライアントが実行するツールのみ、が条件です。`replacements` は、すべてのレッグとすべてのコンボ子リクエストを合わせて 1 つの論理リクエストが行える置き換え送信の回数です(1..2、デフォルトは 1)。レッグ単位の再試行回数でも送信予算でもないため、置き換え送信もそのレッグがすでに持つ送信許容量に収まる必要があります。すでに出力やツール呼び出しを送ったリクエストは、この値に関わらず置き換えません。元の送信がすでに開始されていた場合は置き換えた推論も課金される可能性があるため、既定では無効です。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | ネイティブ `openai-responses` プロバイダー専用で、`authMode: "forward"` も含みます。呼び出し側が何も観測しないまま失敗した送信を、オプトインで置き換えます。設定がなければ無効で、オブジェクトを指定すると `enabled: false` でない限り有効になります。レスポンスヘッダーが届く前に接続が切れた場合と、ヘッダー後に SSE 本文が制御イベントだけを運んだまま切れた場合の両方が対象です。canonical ChatGPT upstream WebSocket で create フレームの送信後、Responses イベントが届く前にソケットが閉じたかエラーになった場合も同じく対象で、その置き換えは HTTP で送信します。置き換えるのは自己完結したリクエストだけで、`store: false`、完全な `input`、`previous_response_id` / `conversation` / `stream_id` がないこと、クライアントが実行するツールのみ、が条件です。`replacements` は、すべてのレッグとすべてのコンボ子リクエストを合わせて 1 つの論理リクエストが行える置き換え送信の回数です(1..2、デフォルトは 1)。レッグ単位の再試行回数でも送信予算でもないため、置き換え送信もそのレッグがすでに持つ送信許容量に収まる必要があります。すでに出力やツール呼び出しを送ったリクエストは、この値に関わらず置き換えません。元の送信がすでに開始されていた場合は置き換えた推論も課金される可能性があるため、既定では無効です。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` が `auto` または `none` のみを受け入れるモデル。強制的な選択は格下げされます。 | | `preserveReasoningContentModels?` | `string[]` |チャット履歴に以前のアシスタント `reasoning_content` が必要なモデル。ダッシュボードから保存しても、保存済みのリスト(`[]` を含む)は保持されます。`PATCH /api/providers?name=` は配列、または消去するための `null` を受け付けます。 アダプター、ベース URL、または認証モードを変えて別の宛先に移す保存では保持されません(下の節を参照)。 | | `reasoningDetailsModels?` | `string[]` | thinking を構造化された `reasoning_details` 配列で返すモデル(`reasoning_split` 使用の MiniMax M シリーズ)。ストリーム差分は累積スナップショットとして prefix-diff され、保持された reasoning は `reasoning_content` 文字列ではなく `reasoning_details` 配列としてリプレイされます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 0662c2f6ebf..05969efb075 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -134,7 +134,7 @@ managed map을 활성화하면 privacy-safe selector를 만들고, 이후 계정 | `responsesSnapshotRepair?` | `boolean` | 기본값이 꺼진 클라이언트용 복구입니다. SSE와 JSON의 Responses 수명 주기에서 누락된 status, output, 도구 메타데이터를 채우며 raw 검사와 영속화는 변경하지 않습니다. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. `attempts`를 모두 소진해도 동일 키 재전송만 중단되며, 이후에는 일반 키 장애 조치 또는 최종 오류 처리가 사용 가능한 대상에 따라 진행됩니다 — 키 인증 passthrough 와이어에는 장애 조치가 없으므로 소진된 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 키 인증 `openai-chat` 및 `openai-responses` 프로바이더 전용입니다. `authMode: "forward"` 프로바이더(ChatGPT 계정 풀)는 이 옵션을 읽지 않고 기본 재시도 단계를 유지합니다. 스트림 시작 전의 일시적인 업스트림 상태(500, 502, 503, 504, 520, 521, 522)를 선택적으로 재시도합니다. 이 옵션이 없으면 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 최초 Responses 요청, 터미널 가드 연속 요청, 네이티브 `/v1/chat/completions`, 429/계정 복구 재조회를 포함합니다. `attempts`는 최초 전송을 포함하여 요청 하나에 허용되는 업스트림 전송의 총횟수(1..10, 기본값 3)입니다. 연결 재설정 복구와 요청 단위 예산 하나를 공유하므로 `3`이면 실제로 프로바이더에 도달하는 요청은 최대 세 번입니다. 대기에는 400ms로 고정된 지수 백오프를 사용하고 상한은 5초이며 `Retry-After`를 따릅니다. 속도 제한을 처리하는 `retryOn429`와는 별개이며, 스트림 도중의 실패는 절대 재전송하지 않습니다. | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 네이티브 `openai-responses` 프로바이더 전용이며 `authMode: "forward"`도 포함합니다. 호출자가 아무것도 관측하지 못한 채 실패한 전송을 선택적으로 대체합니다. 이 옵션이 없으면 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 응답 헤더가 오기 전에 연결이 끊어진 경우와, 헤더 이후 SSE 본문이 제어 이벤트만 실은 채 끊어진 경우를 모두 다룹니다. 자체 완결된 요청만 대체합니다. `store: false`, 완전한 `input`, `previous_response_id`·`conversation`·`stream_id` 없음, 클라이언트가 실행하는 도구만 해당합니다. `replacements`는 모든 구간과 모든 콤보 자식을 합쳐 논리 요청 하나가 만들 수 있는 대체 전송 횟수입니다(1..2, 기본값 1). 구간별 재시도 횟수도 전송 예산도 아니므로, 대체 전송도 해당 구간이 이미 가진 전송 허용량 안에 들어가야 합니다. 이미 출력이나 도구 호출을 내보낸 요청은 이 값과 무관하게 대체하지 않습니다. 원본 전송이 이미 시작됐다면 대체한 추론도 과금될 수 있어서 기본값은 꺼짐입니다. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 네이티브 `openai-responses` 프로바이더 전용이며 `authMode: "forward"`도 포함합니다. 호출자가 아무것도 관측하지 못한 채 실패한 전송을 선택적으로 대체합니다. 이 옵션이 없으면 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 응답 헤더가 오기 전에 연결이 끊어진 경우와, 헤더 이후 SSE 본문이 제어 이벤트만 실은 채 끊어진 경우를 모두 다룹니다. canonical ChatGPT 업스트림 WebSocket에서 create 프레임을 보낸 뒤 Responses 이벤트가 오기 전에 소켓이 닫히거나 오류가 난 경우도 같은 방식으로 다루며, 이때 대체 전송은 HTTP로 보냅니다. 자체 완결된 요청만 대체합니다. `store: false`, 완전한 `input`, `previous_response_id`·`conversation`·`stream_id` 없음, 클라이언트가 실행하는 도구만 해당합니다. `replacements`는 모든 구간과 모든 콤보 자식을 합쳐 논리 요청 하나가 만들 수 있는 대체 전송 횟수입니다(1..2, 기본값 1). 구간별 재시도 횟수도 전송 예산도 아니므로, 대체 전송도 해당 구간이 이미 가진 전송 허용량 안에 들어가야 합니다. 이미 출력이나 도구 호출을 내보낸 요청은 이 값과 무관하게 대체하지 않습니다. 원본 전송이 이미 시작됐다면 대체한 추론도 과금될 수 있어서 기본값은 꺼짐입니다. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. 대시보드에서 저장해도 저장된 목록(`[]` 포함)은 유지됩니다. `PATCH /api/providers?name=`는 배열 또는 지우기 위한 `null`을 받습니다. 어댑터, 기본 URL, 인증 모드를 바꿔 다른 목적지로 옮기는 저장에서는 유지되지 않습니다(아래 절 참고). | | `reasoningDetailsModels?` | `string[]` | thinking을 구조화된 `reasoning_details` 배열로 반환하는 모델(`reasoning_split` 사용 MiniMax M 시리즈). 스트림 델타는 누적 스냅샷이라 prefix-diff로 처리하고, 보존된 reasoning은 `reasoning_content` 문자열 대신 `reasoning_details` 배열로 리플레이합니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 3f296a66d87..e45789240c7 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -267,7 +267,7 @@ Providers can expose a built-in shorthand, such as `agy` for `google-antigravity | `webSearchBridge?` | `{ enabled?: boolean; backend?: "ollama" \| "openai" \| "anthropic" \| "xai" \| "gemini" \| "exa"; maxSearches?: number; timeoutMs?: number; endpoint?: string }` | Key-auth `openai-responses` passthrough providers only. Off by default. Codex always declares the hosted `web_search` tool, and the passthrough relays it on the assumption the destination executes it. A gateway that does not run hosted search answers with a `function_call` named `web_search` that nothing runs, and the undeclared-tool guard ends the turn. With `enabled: true` and an explicit `backend` OpenCodex intercepts that call, runs the search itself, feeds the result back to the same upstream, and shows Codex a hosted `web_search_call` cell. Never armed for `authMode: "forward"` (ChatGPT already searches) or for a provider that executes hosted search upstream. `backend` is required; there is no implicit default and a missing credential for the named backend leaves the bridge disarmed rather than falling through to another paid search. `ollama` reuses this provider's own API key on `POST /api/web_search`, so the origin must be `https://ollama.com` unless the operator names `endpoint` explicitly. `openai` / `anthropic` / `xai` / `gemini` / `exa` reuse the matching sidecar executor and that executor's own credential (`webSearchSidecar.exaApiKey` for Exa). The search model comes from `webSearchSidecar.model` only when `webSearchSidecar.backend` resolves to the same backend this bridge names; otherwise the bridge runs that backend's own default, because a model chosen for one vendor is rejected by another. An unset `webSearchSidecar.backend` resolves to `openai`, so an unset-backend model reaches an `openai` bridge and no other. There is no per-provider bridge model override. Streaming turns only. A turn that mixes `web_search` with another client tool call still fails closed rather than dropping the client's call. Assistant text such as XML-like `` prose is not executed. Defaults: `maxSearches: 3` (1..10), `timeoutMs: 60000` (1000..600000). | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries. Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Key-auth `openai-chat` and `openai-responses` providers only. `authMode: "forward"` providers (the ChatGPT account pool) never read this option and keep the default ladder. Opt-in retry for pre-stream transient upstream statuses (500, 502, 503, 504, 520, 521, 522): absent means off, object presence enables it unless `enabled: false`. Covers the initial Responses request, the Responses passthrough lane and each of its recovery legs (OAuth-401 replay, same-target 429 replay, validated rebuild), the terminal-guard continuation, and native `/v1/chat/completions`. `attempts` is the TOTAL number of upstream sends allowed for one request including the first (1..10, default 3) — it is one budget shared with connection-reset recovery, so `3` means at most three real requests reach the provider. On the Responses passthrough lane the configured value is additionally intersected with the request-wide send allowance, so a value below that allowance narrows the ladder exactly while a value above it does not raise the bound. Waits use a fixed 400 ms exponential backoff capped at 5 s and honor `Retry-After`. Separate from `retryOn429`, which handles rate limiting; mid-stream failures are never replayed. | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Native `openai-responses` providers, including `authMode: "forward"`. Opt-in replacement of a send that failed while the caller had observed nothing: absent means off, object presence enables it unless `enabled: false`. Covers both ambiguous stages — a connection that died before any response header, and an SSE body that died after the header while carrying only control events. Only a self-contained request is ever replaced: `store: false`, complete `input`, no `previous_response_id`, `conversation` or `stream_id`, and only client-executed tools. `replacements` is the number of replacement sends ONE logical request may make across every leg and every combo child (1..2, default 1) — not a per-leg retry count and not a send budget, so a replacement still has to fit inside the send allowance the leg already had. A request that already emitted output or a tool call is never replaced, whatever this is set to. The replacement inference may still be billed if the origin had already started the first one, which is why this is off by default. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Native `openai-responses` providers, including `authMode: "forward"`. Opt-in replacement of a send that failed while the caller had observed nothing: absent means off, object presence enables it unless `enabled: false`. Covers both ambiguous stages — a connection that died before any response header, and an SSE body that died after the header while carrying only control events. A canonical ChatGPT upstream WebSocket that closed or errored after its create frame left, before any Responses event, is covered the same way, and its replacement is sent over HTTP. Only a self-contained request is ever replaced: `store: false`, complete `input`, no `previous_response_id`, `conversation` or `stream_id`, and only client-executed tools. `replacements` is the number of replacement sends ONE logical request may make across every leg and every combo child (1..2, default 1) — not a per-leg retry count and not a send budget, so a replacement still has to fit inside the send allowance the leg already had. A request that already emitted output or a tool call is never replaced, whatever this is set to. The replacement inference may still be billed if the origin had already started the first one, which is why this is off by default. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. A provider save that keeps the destination keeps the stored list, including `[]`; see [What a provider save keeps](#what-a-provider-save-keeps). `PATCH /api/providers?name=` accepts an array or `null` to clear it. | | `inlineThinkTagModels?` | `string[]` | Opt-in recovery for `openai-chat` gateways without a server-side reasoning parser. A leading `` / `` / `` block (optionally after whitespace) activates splitting in streamed and buffered replies. All answer whitespace is preserved. Subsequent tags are delimiters anywhere, including same-line interleaving and code fences; this mode does not interpret Markdown. Ordinary text or a code fence before the first tag keeps the whole reply untouched. Off by default; prefer structured upstream reasoning or `reasoningSplitModels` where supported. | diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 0e406b44a1e..0394a17d5f3 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -54,9 +54,10 @@ HTTP 504 with an `upstream_no_response` error. A slow but alive origin therefore client's own deadline or for `connectTimeoutMs` (default 200s), whichever comes first; a connect timeout that fires after the create frame was sent settles as the same 504. A socket that closes or errors before the first Responses event settles as an HTTP 502 with -`upstream_closed_before_response`. These statuses are never retried inside the proxy — the -frame may already be executing upstream, so the client applies its own retry policy exactly as -it would when connected to the backend directly. Once the response has started, a later drop +`upstream_closed_before_response`. The proxy does not retry either status on its own: the frame +may already be executing upstream, so the client applies its own retry policy exactly as it would +when connected to the backend directly. The one exception is that 502 on a provider that opted +into `retryOnReset`, described below. Once the response has started, a later drop surfaces inside the stream as before. `stallTimeoutSec` is unrelated to this window. An ordinary HTTP send has a third case. When the connection dies before any response header @@ -78,7 +79,10 @@ duplicate a turn. A native Responses provider can opt into replacing that send with [`retryOnReset`](/reference/configuration/providers/#provider-entries-ocxproviderconfig). The same grant covers the case where the connection survives the header and the SSE body then dies carrying only control -events, because the caller has observed nothing in either one. A replacement happens only when +events, because the caller has observed nothing in either one. It also covers the canonical +ChatGPT WebSocket above: a socket that closes or errors before the first Responses event is +replaced by one HTTP send, never by a second socket. A silent socket keeps its 504, and a native +steering or injection turn is never replaced. A replacement happens only when the request is self-contained (`store: false`, complete input, client-executed tools only, no server-side continuation state), and one logical request gets the configured number of replacements in total — across every recovery leg and every combo child, not one each. The diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index d84233b89bf..26aa73be207 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -147,7 +147,7 @@ cross-route credential fallback не существует. Строки API GPT- | `responsesSnapshotRepair?` | `boolean` | По умолчанию выключенная клиентская repair для неполных lifecycle snapshot'ов Responses в SSE и JSON. Добавляет отсутствующие status, output и tool metadata, не меняя raw inspection и persistence. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста. Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Только для провайдеров `openai-chat` и `openai-responses` с аутентификацией по ключу. Провайдеры с `authMode: "forward"` (пул аккаунтов ChatGPT) никогда не читают эту настройку и сохраняют число повторов по умолчанию. Опциональный повтор при временных статусах апстрима до начала потока (500, 502, 503, 504, 520, 521, 522): если параметр отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. Покрывает исходный запрос `Responses`, продолжение терминального предохранителя, нативный `/v1/chat/completions`, а также повторные запросы при восстановлении после 429 или ошибки учётной записи. `attempts` — ОБЩЕЕ число разрешённых отправок в апстрим для одного запроса, включая первую (1..10, по умолчанию 3). Это единый бюджет на запрос, общий с восстановлением после сброса соединения, поэтому `3` означает, что до провайдера дойдут не более трёх реальных запросов. Ожидание использует экспоненциальную задержку с фиксированной начальной величиной 400 мс, ограниченную 5 с, и учитывает `Retry-After`. Параметр не связан с `retryOn429`, который обрабатывает ограничение частоты запросов; сбои после начала потока никогда не воспроизводятся. | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Только для нативных провайдеров `openai-responses`, включая `authMode: "forward"`. Необязательная замена отправки, которая завершилась неудачей, когда вызывающая сторона ещё ничего не наблюдала: если параметр отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. Покрывает обе неоднозначные стадии — обрыв соединения до любого заголовка ответа и обрыв тела SSE после заголовка, когда оно несло только управляющие события. Заменяется только самодостаточный запрос: `store: false`, полный `input`, отсутствие `previous_response_id`, `conversation` и `stream_id`, и только инструменты, исполняемые клиентом. `replacements` — число замещающих отправок, которые ОДИН логический запрос может сделать по всем участкам и всем дочерним запросам комбо (1..2, по умолчанию 1). Это не число повторов на участок и не бюджет отправок, поэтому замена всё равно должна поместиться в уже имеющийся у участка лимит отправок. Запрос, который уже выдал вывод или вызов инструмента, не заменяется никогда, каким бы ни было это значение. Замещающий вывод модели всё равно может быть оплачен, если источник уже начал первый, поэтому параметр выключен по умолчанию. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Только для нативных провайдеров `openai-responses`, включая `authMode: "forward"`. Необязательная замена отправки, которая завершилась неудачей, когда вызывающая сторона ещё ничего не наблюдала: если параметр отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. Покрывает обе неоднозначные стадии — обрыв соединения до любого заголовка ответа и обрыв тела SSE после заголовка, когда оно несло только управляющие события. Так же покрывается upstream WebSocket canonical ChatGPT, который закрылся или завершился ошибкой после отправки кадра create и до любого события Responses; замена в этом случае отправляется по HTTP. Заменяется только самодостаточный запрос: `store: false`, полный `input`, отсутствие `previous_response_id`, `conversation` и `stream_id`, и только инструменты, исполняемые клиентом. `replacements` — число замещающих отправок, которые ОДИН логический запрос может сделать по всем участкам и всем дочерним запросам комбо (1..2, по умолчанию 1). Это не число повторов на участок и не бюджет отправок, поэтому замена всё равно должна поместиться в уже имеющийся у участка лимит отправок. Запрос, который уже выдал вывод или вызов инструмента, не заменяется никогда, каким бы ни было это значение. Замещающий вывод модели всё равно может быть оплачен, если источник уже начал первый, поэтому параметр выключен по умолчанию. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. Сохранение в дашборде не меняет записанный список, в том числе `[]`. `PATCH /api/providers?name=` принимает массив или `null`, чтобы его очистить. Сохранение, которое переносит провайдера на другой адаптер, базовый URL или режим аутентификации, список не сохраняет (см. раздел ниже). | | `reasoningDetailsModels?` | `string[]` | Модели, чей endpoint возвращает thinking как структурированный массив `reasoning_details` (MiniMax M-series с `reasoning_split`); потоковые дельты — кумулятивные снимки, сравниваемые по префиксу, а сохранённый reasoning воспроизводится массивом `reasoning_details` вместо строки `reasoning_content`. | diff --git a/docs-site/src/content/docs/tr/reference/configuration/providers.md b/docs-site/src/content/docs/tr/reference/configuration/providers.md index 8fa358583c5..d1c8fb099a2 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/tr/reference/configuration/providers.md @@ -148,7 +148,7 @@ alanlı seçilmiş kimlikleri yalın kimliklere yeniden yazar. | `responsesSnapshotRepair?` | `boolean` | SSE ve JSON'daki seyrek Responses yaşam döngüsü anlık görüntüleri için varsayılan olarak devre dışı bırakılmış istemciye yönelik onarım. Ham inceleme ve kalıcılık değişmeden kalırken eksik kurallı durumu, çıktıyı ve araç meta verilerini doldurur. | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Yalnızca API anahtarı sağlayıcıları (`authMode: "key"`). İsteğe bağlı aynı hedef 429 yeniden denemesi: `retryOn429` olmadığında özellik kapalıdır; nesnenin varlığı `enabled: false` olmadığı sürece özelliği etkinleştirir. 429'da proxy bekler (yukarı akış `Retry-After` veya sabit aralık) ve herhangi bir anahtar yük devretmesinden önce aynı istek üzerinde aynı anahtarla aynı isteği yeniden oynatır — ana metin turu kurtarma döngüsü, Responses doğrudan geçiş hattı, görsel/video köprüsü, web araması sidecar'ı ve terminal devamları genelinde. Yalnızca akış öncesi HTTP 429 yanıtları yeniden oynatma için uygundur; özel `runTurn` aktarımları HTTP yeniden deneme döngüsünün dışındadır. `attempts`, ilk 429'dan sonraki aynı anahtar yeniden oynatmalarını sayar (toplam gönderim = `attempts` + 1) ve ana kurtarma döngüsü, terminal koruma devamı ve köprü yeniden denemeleri tarafından paylaşılan tek bir istek genelinde bütçedir. `attempts`'ı tüketmek yalnızca daha fazla aynı anahtar yeniden oynatmasını durdurur: normal anahtar yük devretmesi veya nihai hata işleme daha sonra kullanılabilir hedeflere göre geçerli olur — anahtar kimlik doğrulamalı doğrudan geçiş hattında yük devretme yoktur, bu nedenle tükenen 429 olduğu gibi görünür. Codex'in kendisi 429'u asla yeniden denemez, bu nedenle tek anahtarlı sağlayıcılar için tek savunma budur. Varsayılanlar: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (tek bir bekleme `maxIntervalMs` ile sınırlandırılır, kendisi de 600000 ile sınırlandırılır), `respectRetryAfter: true`. | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | Yalnızca anahtarla kimlik doğrulanan `openai-chat` ve `openai-responses` sağlayıcıları. `authMode: "forward"` sağlayıcıları (ChatGPT hesap havuzu) bu seçeneği hiç okumaz ve varsayılan merdiveni korur. Akış öncesi geçici yukarı akış durumları (500, 502, 503, 504, 520, 521, 522) için isteğe bağlı yeniden deneme: seçenek belirtilmezse kapalıdır; nesnenin varlığı, `enabled: false` olmadığı sürece özelliği etkinleştirir. İlk Responses isteğini, terminal koruma devamını, yerel `/v1/chat/completions` isteklerini ve 429/hesap kurtarma yeniden getirmelerini kapsar. `attempts`, bir istek için ilk gönderim dahil izin verilen yukarı akış gönderimlerinin TOPLAM sayısıdır (1..10, varsayılan 3) — bağlantı sıfırlama kurtarmasıyla paylaşılan, istek kapsamlı tek bütçedir; dolayısıyla `3`, sağlayıcıya en fazla üç gerçek isteğin ulaşması anlamına gelir. Beklemelerde 400 ms'lik sabit üstel geri çekilme uygulanır, süre 5 sn ile sınırlandırılır ve `Retry-After` dikkate alınır. Hız sınırlamasını işleyen `retryOn429` seçeneğinden ayrıdır; akış ortası hataları hiçbir zaman yeniden oynatılmaz. | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Yalnızca yerel `openai-responses` sağlayıcıları, `authMode: "forward"` dahil. Çağıranın hiçbir şey gözlemlemediği bir anda başarısız olan gönderimin isteğe bağlı olarak değiştirilmesi: seçenek belirtilmezse kapalıdır; nesnenin varlığı, `enabled: false` olmadığı sürece özelliği etkinleştirir. İki belirsiz aşamayı da kapsar: yanıt başlığı gelmeden kopan bağlantı ve başlıktan sonra yalnızca denetim olayları taşırken kopan SSE gövdesi. Yalnızca kendi kendine yeten bir istek değiştirilir: `store: false`, eksiksiz `input`, `previous_response_id`, `conversation` veya `stream_id` bulunmaması ve yalnızca istemcinin yürüttüğü araçlar. `replacements`, BİR mantıksal isteğin tüm bacaklar ve tüm combo alt istekleri boyunca yapabileceği değiştirme gönderimi sayısıdır (1..2, varsayılan 1). Bacak başına yeniden deneme sayısı da gönderim bütçesi de değildir; bu yüzden bir değiştirme gönderimi, ilgili bacağın hâlihazırda sahip olduğu gönderim payına sığmak zorundadır. Halihazırda çıktı veya araç çağrısı üretmiş bir istek, bu değer ne olursa olsun asla değiştirilmez. Kaynak ilk çıkarımı zaten başlatmışsa değiştirilen çıkarım yine ücretlendirilebilir; bu nedenle varsayılan olarak kapalıdır. | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Yalnızca yerel `openai-responses` sağlayıcıları, `authMode: "forward"` dahil. Çağıranın hiçbir şey gözlemlemediği bir anda başarısız olan gönderimin isteğe bağlı olarak değiştirilmesi: seçenek belirtilmezse kapalıdır; nesnenin varlığı, `enabled: false` olmadığı sürece özelliği etkinleştirir. İki belirsiz aşamayı da kapsar: yanıt başlığı gelmeden kopan bağlantı ve başlıktan sonra yalnızca denetim olayları taşırken kopan SSE gövdesi. Kanonik ChatGPT upstream WebSocket'i create çerçevesi gönderildikten sonra, herhangi bir Responses olayından önce kapanır veya hata verirse aynı şekilde kapsanır ve değiştirme gönderimi HTTP üzerinden yapılır. Yalnızca kendi kendine yeten bir istek değiştirilir: `store: false`, eksiksiz `input`, `previous_response_id`, `conversation` veya `stream_id` bulunmaması ve yalnızca istemcinin yürüttüğü araçlar. `replacements`, BİR mantıksal isteğin tüm bacaklar ve tüm combo alt istekleri boyunca yapabileceği değiştirme gönderimi sayısıdır (1..2, varsayılan 1). Bacak başına yeniden deneme sayısı da gönderim bütçesi de değildir; bu yüzden bir değiştirme gönderimi, ilgili bacağın hâlihazırda sahip olduğu gönderim payına sığmak zorundadır. Halihazırda çıktı veya araç çağrısı üretmiş bir istek, bu değer ne olursa olsun asla değiştirilmez. Kaynak ilk çıkarımı zaten başlatmışsa değiştirilen çıkarım yine ücretlendirilebilir; bu nedenle varsayılan olarak kapalıdır. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`'u yalnızca `auto` veya `none` kabul eden modeller; zorunlu seçimlerin derecesi düşürülür. | | `preserveReasoningContentModels?` | `string[]` | Sohbet geçmişinde önceki asistan `reasoning_content`'ini gerektiren modeller. Dashboard üzerinden yapılan kayıtlar, `[]` dahil saklanan listeyi korur. `PATCH /api/providers?name=`, bir dizi veya temizlemek için `null` kabul eder. Sağlayıcıyı başka bir bağdaştırıcıya, temel URL'ye veya kimlik doğrulama moduna taşıyan bir kayıt listeyi korumaz (aşağıdaki bölüme bakın). | | `reasoningDetailsModels?` | `string[]` | Thinking'i yapılandırılmış bir `reasoning_details` dizisi olarak döndüren modeller (`reasoning_split` ile MiniMax M-serisi); akış deltaları önek farkıyla işlenen kümülatif anlık görüntülerdir ve korunan reasoning, `reasoning_content` dizesi yerine `reasoning_details` dizisi olarak yeniden oynatılır. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index e0e0502826e..5230ad5dfc1 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -134,7 +134,7 @@ selector,而不是分配一个新名称。 | `responsesSnapshotRepair?` | `boolean` | 默认关闭的客户端修复,用于补全 SSE 与 JSON 中稀疏 Responses 生命周期快照缺失的 status、output 和工具元数据;原始检查与持久化保持不变。 | | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商(`authMode: "key"`)。可选的同目标 429 重试:未配置 `retryOn429` 时功能关闭;对象存在即启用,除非 `enabled: false`。收到 429 时等待(上游 `Retry-After` 或固定间隔)后在相同 key 上重放完全相同请求,再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应;自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数(总发送次数 = `attempts` + 1),是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算;`attempts` 耗尽只会停止进一步的同 key 重放:随后按可用目标进行正常的 key 故障转移或最终错误处理——key 认证的 passthrough 线路上没有故障转移,因此耗尽的 429 会原样透出。Codex 自身从不重试 429,因此这是单 key 提供商唯一的防线。默认值:`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(单次等待以 `maxIntervalMs` 为上限,其本身上限 600000)、`respectRetryAfter: true`。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 仅限使用 key 认证的 `openai-chat` 与 `openai-responses` 提供商。`authMode: "forward"` 的提供商(ChatGPT 账号池)从不读取此选项,保持默认重试次数。可选的流开始前上游瞬态状态码(500、502、503、504、520、521、522)重试:未配置时关闭;对象存在即启用,除非 `enabled: false`。覆盖初始 Responses 请求、终结守卫续接、原生 `/v1/chat/completions`,以及 429/账户恢复重新获取。`attempts` 是单个请求允许向上游发送的总次数,包含首次发送(1..10,默认 3);它是与连接重置恢复共享的按请求预算,因此 `3` 表示最多只有三个实际请求到达提供商。等待采用固定 400 毫秒的指数退避,上限为 5 秒,并遵循 `Retry-After`。此选项独立于处理速率限制的 `retryOn429`;流开始后的故障绝不会重放。 | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 仅限原生 `openai-responses` 提供商,包含 `authMode: "forward"`。可选地替换一次在调用方尚未观察到任何内容时就失败的发送:未配置时关闭;对象存在即启用,除非 `enabled: false`。涵盖两个不确定阶段——响应头到达前连接断开,以及响应头之后 SSE 正文只承载控制事件时断开。只有自包含的请求才会被替换:`store: false`、完整的 `input`、没有 `previous_response_id`/`conversation`/`stream_id`,且只使用由客户端执行的工具。`replacements` 是单个逻辑请求在所有环节和所有组合子请求中可以进行的替换发送次数(1..2,默认 1);它既不是按环节的重试次数,也不是发送预算,因此替换发送仍必须落在该环节已有的发送额度之内。已经产生输出或工具调用的请求,无论此值为何都不会被替换。如果上游已经开始了第一次推理,被替换的推理仍可能计费,因此该选项默认关闭。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 仅限原生 `openai-responses` 提供商,包含 `authMode: "forward"`。可选地替换一次在调用方尚未观察到任何内容时就失败的发送:未配置时关闭;对象存在即启用,除非 `enabled: false`。涵盖两个不确定阶段——响应头到达前连接断开,以及响应头之后 SSE 正文只承载控制事件时断开。canonical ChatGPT 上游 WebSocket 在 create 帧发出之后、任何 Responses 事件到达之前关闭或出错时,也按同样方式处理,其替换发送走 HTTP。只有自包含的请求才会被替换:`store: false`、完整的 `input`、没有 `previous_response_id`/`conversation`/`stream_id`,且只使用由客户端执行的工具。`replacements` 是单个逻辑请求在所有环节和所有组合子请求中可以进行的替换发送次数(1..2,默认 1);它既不是按环节的重试次数,也不是发送预算,因此替换发送仍必须落在该环节已有的发送额度之内。已经产生输出或工具调用的请求,无论此值为何都不会被替换。如果上游已经开始了第一次推理,被替换的推理仍可能计费,因此该选项默认关闭。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型;强制选择会被降级。 | | `preserveReasoningContentModels?` | `string[]` | 需要在聊天历史中保留先前 assistant `reasoning_content` 的模型。从仪表板保存时会保留已存储的列表(包括 `[]`)。`PATCH /api/providers?name=` 接受数组,或传入 `null` 清除该字段。将提供方改到其他适配器、base URL 或认证模式的保存不会保留该列表(见下文)。 | | `reasoningDetailsModels?` | `string[]` | 以结构化 `reasoning_details` 数组返回思考内容的模型(启用 `reasoning_split` 的 MiniMax M 系列);流式增量为累积快照,按前缀差分处理,保留的推理以 `reasoning_details` 数组而非 `reasoning_content` 字符串回放。 | diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md index 91bef990796..7b992e4a55e 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md @@ -106,7 +106,7 @@ ocx models provider openrouter on | `parallelToolCalls?` | `boolean` | 切換平行工具呼叫。OpenAI Chat 預設開啟;非 chat adapter 僅在明確 `true` 時廣告。 | | `responsesItemIdRepair?` | `{ message?: string[]; reasoning?: string[]; repairMissingTerminalIds?: boolean }` | 預設停用的下游 SSE 修復,用於精確佔位 id 與缺失的終端 id。Function-call id 永不被重寫。 | | `transientRetryOn5xx?` | `{ enabled?: boolean; attempts?: number }` | 僅限使用金鑰認證的 `openai-chat` 與 `openai-responses` 供應商。`authMode: "forward"` 的供應商(ChatGPT 帳號池)從不讀取此選項,維持預設重試次數。選擇性重試串流開始前的暫時性上游狀態(500、502、503、504、520、521、522):未設定時停用;只要有此物件即啟用,除非 `enabled: false`。涵蓋初始 `Responses` 請求、終止防護續接、原生 `/v1/chat/completions`,以及 429/帳號復原的重新擷取。`attempts` 是單一請求允許傳送至上游的總次數,包含第一次(1..10,預設 3);這是與連線重設復原共用的單一請求範圍預算,因此 `3` 表示最多只有三個實際請求會送達供應商。等待採固定 400 毫秒、上限 5 秒的指數退避,並遵循 `Retry-After`。此機制獨立於處理速率限制的 `retryOn429`;串流中的失敗絕不重播。 | -| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 僅限原生 `openai-responses` 供應商,包含 `authMode: "forward"`。可選擇性地替換一次在呼叫端尚未觀察到任何內容時就失敗的傳送:未設定時停用;只要有此物件即啟用,除非 `enabled: false`。涵蓋兩個不確定階段——回應標頭抵達前連線中斷,以及標頭之後 SSE 內文只載有控制事件時中斷。只有自我完備的請求才會被替換:`store: false`、完整的 `input`、沒有 `previous_response_id`/`conversation`/`stream_id`,且僅使用由用戶端執行的工具。`replacements` 是單一邏輯請求在所有環節與所有組合子請求中可進行的替換傳送次數(1..2,預設 1);它既不是各環節的重試次數,也不是傳送預算,因此替換傳送仍必須落在該環節既有的傳送額度之內。已經產生輸出或工具呼叫的請求,無論此值為何都不會被替換。若上游已經開始第一次推論,被替換的推論仍可能計費,因此此選項預設停用。 | +| `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | 僅限原生 `openai-responses` 供應商,包含 `authMode: "forward"`。可選擇性地替換一次在呼叫端尚未觀察到任何內容時就失敗的傳送:未設定時停用;只要有此物件即啟用,除非 `enabled: false`。涵蓋兩個不確定階段——回應標頭抵達前連線中斷,以及標頭之後 SSE 內文只載有控制事件時中斷。canonical ChatGPT upstream WebSocket 在 create 訊框送出之後、任何 Responses 事件抵達之前關閉或出錯時,也以相同方式處理,其替換傳送改走 HTTP。只有自我完備的請求才會被替換:`store: false`、完整的 `input`、沒有 `previous_response_id`/`conversation`/`stream_id`,且僅使用由用戶端執行的工具。`replacements` 是單一邏輯請求在所有環節與所有組合子請求中可進行的替換傳送次數(1..2,預設 1);它既不是各環節的重試次數,也不是傳送預算,因此替換傳送仍必須落在該環節既有的傳送額度之內。已經產生輸出或工具呼叫的請求,無論此值為何都不會被替換。若上游已經開始第一次推論,被替換的推論仍可能計費,因此此選項預設停用。 | | `autoToolChoiceOnlyModels?` | `string[]` | 其 `tool_choice` 僅接受 `auto` 或 `none` 的模型;強制選擇被降級。 | | `preserveReasoningContentModels?` | `string[]` | 需要在 chat 歷史中保留先前 assistant `reasoning_content` 的模型。從儀表板儲存時會保留已儲存的清單(包括 `[]`)。`PATCH /api/providers?name=` 接受陣列,或傳入 `null` 清除該欄位。將供應商改到其他轉接器、base URL 或驗證模式的儲存不會保留該清單(見下文)。 | | `reasoningDetailsModels?` | `string[]` | 以結構化 `reasoning_details` 陣列回傳思考內容的模型(啟用 `reasoning_split` 的 MiniMax M 系列);串流增量為累積快照,以前綴差分處理,保留的推理以 `reasoning_details` 陣列而非 `reasoning_content` 字串重播。 | diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 895510d88c4..2831c16ff33 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1702,6 +1702,7 @@ "winsw-stop-hardening.test.ts": "windows", "winsw.test.ts": "service", "workflow-budget.test.ts": "lib", + "ws-ambiguous-resend.test.ts": "responses", "ws-endpoint.test.ts": "responses", "ws-failure-stage.test.ts": "responses", "ws-native-injection.test.ts": "responses", diff --git a/src/lib/request-resend-gate.ts b/src/lib/request-resend-gate.ts index 0107aa648bb..eb75a2a0c9b 100644 --- a/src/lib/request-resend-gate.ts +++ b/src/lib/request-resend-gate.ts @@ -5,9 +5,10 @@ * asked it for a connection that died before any head; #4989 asked it for an SSE body that * died after the head while carrying only control events. Both are the same row of the stage * table: a stage the caller observed nothing at, with a cause that cannot prove the origin did - * not run the turn. `resendPermission` answers `refused-ambiguous` for both, and - * request-failure-model.ts already names the only thing that may override that answer -- a - * narrowly scoped recovery a maintainer opted into and bounded. + * not run the turn. A Codex WebSocket that dies under its create frame before any Responses + * event (#4191) is that row a third time. `resendPermission` answers `refused-ambiguous` for all + * of them, and request-failure-model.ts already names the only thing that may override that + * answer -- a narrowly scoped recovery a maintainer opted into and bounded. * * One override, not two. The reason this module exists rather than a boolean in each caller is * that a request which resets before the head and again after it would otherwise buy a diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 3419b8ffb8a..2b2b546388c 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -440,6 +440,25 @@ function invitesResendAfterReplacement(status: number): boolean { || status === 307 || status === 308 || status === 413 || status >= 500; } +/** + * The answer a request keeps once its one operator replacement has gone out. + * + * A status that invites another send settles as the refusal. Any other answer keeps its real + * status: no client retries it, and the caller needs the evidence (a 400 names the request + * defect). The marker still stops this process from using it as a recovery trigger, such as the + * opaque-blob rebuild of a 400 or a combo hop on a context overflow, because each of those checks + * it before sending again. + */ +export function settleOperatorReplacement(response: Response): Response { + if (response.ok) return response; + if (invitesResendAfterReplacement(response.status)) { + cancelResponseBodyBestEffort(response); + return replayRefusalResponse(); + } + markResponseNonReplayable(response); + return response; +} + export async function fetchWithAttemptDeadline( url: string, init: RequestInit, @@ -624,18 +643,7 @@ export async function fetchWithResetRetry( opts.onSendsConsumed?.(1); try { const response = await doFetch(attempt === 0 ? firstRecovery : "connection-reset"); - if (spentOperatorReplacement && !response.ok) { - if (invitesResendAfterReplacement(response.status)) { - cancelResponseBodyBestEffort(response); - return replayRefusalResponse(); - } - // Any other answer keeps its real status: no client retries it, and the caller needs the - // evidence (a 400 names the request defect). The marker still stops this process from - // using it as a recovery trigger, such as the opaque-blob rebuild of a 400 or a combo hop - // on a context overflow, because each of those checks it before sending again. - markResponseNonReplayable(response); - } - return response; + return spentOperatorReplacement ? settleOperatorReplacement(response) : response; } catch (err) { if (opts.abortSignal?.aborted) throw err; if (!isConnectionResetError(err)) { diff --git a/src/server/responses/codex-ws-exchange.ts b/src/server/responses/codex-ws-exchange.ts index 62129632cea..474bc2bb1f0 100644 --- a/src/server/responses/codex-ws-exchange.ts +++ b/src/server/responses/codex-ws-exchange.ts @@ -11,7 +11,7 @@ import type { CodexWsSession } from "./codex-ws-session"; import { UPGRADE_DEADLINE_MS, CODEX_WS_LIVENESS_PING_INTERVAL_MS, CODEX_WS_RESPONSE_PRELUDE_TIMEOUT_MS, MAX_CODEX_WS_FRAME_BYTES, MAX_CODEX_WS_QUEUE_BYTES, markCodexWsResponse, normalizeResponsesWsRelayEvent, closedBeforeTerminalMessage, codexWsCreateFrameExceedsLimit, codexWsFailureDetail, codexWsPreResponseFailure, markCodexWsStage, codexWsOcxVersion, - type CodexWsFailureStage, type CodexWsStageRecord } from "./codex-ws-wire"; + markCodexWsSocketDeath, type CodexWsFailureStage, type CodexWsStageRecord } from "./codex-ws-wire"; interface ExchangeOptions { nativeControl?: NativeResponseControl; @@ -213,14 +213,14 @@ export function codexWsExchange(options: ExchangeOptions): Promise { resolve(response); }; - const failStream = (error: unknown, status: 502 | 504 = 502) => { + const failStream = (error: unknown, status: 502 | 504 = 502, { socketDied = false } = {}) => { if (terminal) return; terminal = true; if (sent && !responseCommitted && metadata) { // Nothing has been promised to the client yet, so the honest answer is a gateway // status, not a 200 whose body then fails. The frame may already be executing - // upstream: the response is marked non-replayable so no layer of this process sends - // it again, and the client applies its own retry policy as it would on the direct + // upstream: the response is marked non-replayable so no retry layer of this process + // sends it again, and the client applies its own retry policy as it would on the direct // path. Same settle order as a refused create: snapshot, detach, close, dispose. const prelude = metadata.snapshot(); // Claim the commit slot so no later path can resolve a second, 200 Response. @@ -230,7 +230,13 @@ export function codexWsExchange(options: ExchangeOptions): Promise { session.dispose(); const message = error instanceof Error ? error.message : String(error); const failureResponse = codexWsPreResponseFailure(status, message, prelude); - markCodexWsStage(failureResponse, stageRecord(Buffer.byteLength(frameText, "utf8"))); + const stage = failureStage(); + markCodexWsStage(failureResponse, stageRecord(stage.requestBytes)); + // #4191: a socket that died under the send is the one settle the dispatch may replace, + // once, and only under the operator's `retryOnReset` grant. Native steering and injection + // are left out: their channel may already have sent continuation frames on this socket, so + // the create frame alone no longer describes the turn. + if (socketDied && !nativeControl) markCodexWsSocketDeath(failureResponse, stage); resolve(failureResponse); return; } @@ -549,7 +555,9 @@ export function codexWsExchange(options: ExchangeOptions): Promise { resolve(sseFallback(url, init)); return; } - if (sent && !terminal) failStream(closedBeforeTerminalMessage(event, failureStage())); + if (sent && !terminal) { + failStream(closedBeforeTerminalMessage(event, failureStage()), 502, { socketDied: true }); + } }; const onError = () => { @@ -560,7 +568,10 @@ export function codexWsExchange(options: ExchangeOptions): Promise { cleanup(); session.dispose(); resolve(sseFallback(url, init)); - } else failStream(`codex websocket transport error${codexWsFailureDetail(failureStage())}`); + } else { + const message = `codex websocket transport error${codexWsFailureDetail(failureStage())}`; + failStream(message, 502, { socketDied: true }); + } }; detachOwner = session.bindOwner(reason => cancelExchange(reason)); ws.addEventListener("open", onOpen); diff --git a/src/server/responses/codex-ws-wire.ts b/src/server/responses/codex-ws-wire.ts index e8edb1106c7..b2d904caa5e 100644 --- a/src/server/responses/codex-ws-wire.ts +++ b/src/server/responses/codex-ws-wire.ts @@ -214,10 +214,10 @@ export function classifyCodexWsFailure(stage: CodexWsFailureStage): CodexWsFailu * whose failures cannot be compared with anything else, which is the reported symptom -- every * such failure reached the user as one of two bare sentences. * - * It does not relax the transport's own rule. The no-replay-after-send contract in - * `codex-ws-exchange.ts` holds regardless of what this returns, and the stage below is - * deliberately not consulted as a fallback-eligibility signal; it reports where the exchange got - * to, and `resendPermission` happens to agree that everything past `before-send` is refused. + * It does not relax the transport's own rule. The stage below reports where the exchange got to, + * and `resendPermission` agrees that everything past `before-send` is refused. When a socket dies + * under the send, this stage is what the resend gate is asked with (#4191), so the operator's + * `retryOnReset` grant is the only way past that refusal, as it is for an HTTP reset. */ export const CODEX_WS_FAILURE_PROJECTION = { /** The create frame never left, so the origin provably never saw this turn. */ @@ -236,6 +236,29 @@ export function projectCodexWsFailure( return CODEX_WS_FAILURE_PROJECTION[classifyCodexWsFailure(stage)]; } +const socketDeathStages = new WeakMap(); + +/** + * Record that a pre-response settle came from the socket closing or failing under the send (#4191), + * rather than from silence, a refused frame or a local limit. + * + * A fact about how the exchange ended, not a grant. The settle is the same non-replayable 502 + * either way; whether the turn may go out once more is the resend gate's question, and only the + * operator's `retryOnReset` grant can answer it yes. + */ +export function markCodexWsSocketDeath(response: Response, stage: CodexWsFailureStage): void { + socketDeathStages.set(response, projectCodexWsFailure(stage).stage); +} + +/** + * Where the send stood when its socket died: `pre-header` when nothing came back and + * `protocol-prelude` when frames arrived but none was a Responses event. Undefined for every other + * response. + */ +export function codexWsSocketDeathStage(response: Response): RequestFailureStage | undefined { + return socketDeathStages.get(response); +} + /** * Render the stage as a suffix appended to an existing failure message. * diff --git a/src/server/responses/core-opaque-recovery.ts b/src/server/responses/core-opaque-recovery.ts index 788e2f2c1ab..b436ebdc9e6 100644 --- a/src/server/responses/core-opaque-recovery.ts +++ b/src/server/responses/core-opaque-recovery.ts @@ -311,14 +311,15 @@ export function shouldAttemptOpaqueBlobRecovery(args: { * Peek the upstream error body for the reasoning-effort downgrade. Only 400/403 are considered * and the body must be complete and display-safe, the same contract the other rejection peeks * use. The match is deliberately narrow: the upstream has to name reasoning effort, so an - * unrelated 400 never triggers a replay. + * unrelated 400 never triggers a replay. A non-replayable answer, such as one to a spent operator + * replacement, is never read: the first send may already have run the turn. */ export async function reasoningEffortRejectionText( response: Response, alreadyAttempted: boolean, signal: AbortSignal, ): Promise { - if (alreadyAttempted) return undefined; + if (alreadyAttempted || isNonReplayableResponse(response)) return undefined; if (response.status !== 400 && response.status !== 403) return undefined; try { const body = await readBoundedResponseBody(response.clone(), { signal }); diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 733081b0bdc..80854b58f6b 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -102,7 +102,7 @@ import { clearCodexModelDenialEvidence, recordCodexModelDenialEvidence, } from "../../codex/model-entitlements"; -import { isCodexWsUpstreamResponse, readCodexWsStage } from "./codex-ws-wire"; +import { codexWsSocketDeathStage, isCodexWsUpstreamResponse, readCodexWsStage } from "./codex-ws-wire"; import { linkAbortSignal } from "./core-lifetime"; import type { CodexAuthContext } from "../../codex/auth-context"; import { checkOutboundBodySize, describeOutboundBodyRefusal } from "./outbound-body-guard"; @@ -113,6 +113,8 @@ import { applyUpstreamRecoveryInit, isNonReplayableResponse, refetchAfterProtocolSafeReset, + replayRefusalResponse, + settleOperatorReplacement, prepareSameTarget429Wait, sleepWithAbort, } from "../../lib/upstream-retry"; @@ -751,6 +753,57 @@ export async function preparePassthroughExchange( */ const claimPreHeaderResend = (): boolean => authorizeResendForRecovery("pre-header", "connection-reset", ambiguousResend()).allowed; + /** + * The one replacement send the ambiguous rows at the end of the recovery loop may buy: an SSE + * body that died before any output, and a Codex WebSocket that died under its create frame + * (#4191). + * + * HTTP-only for both. A replacement HTTP body must not open a fresh WebSocket exchange: the SSE + * row replaces an HTTP stream, which a WS create frame is not, and the WebSocket row replaces + * the transport that just failed. + */ + const sendAmbiguousReplacement = ( + signal: AbortSignal = upstream.signal, + ): Promise => fetchWithHeaderTimeout( + request.url, + applyUpstreamRecoveryInit({ + method: request.method, + headers: request.headers, + body: request.body, + }, "connection-reset"), + signal, + connectMs, + true, + providerFetch(route.provider, options.codexWsRuntimeIdentity, { + httpOnly: true, + providerName: route.providerName, + modelId: route.modelId, + dispatchOverride: oauthDispatch(request), + beforeDispatch: headers => { + if (signal.aborted) throw signal.reason; + if (!transportState.selectionIsCurrent(transportState.requestBindings.get(request))) { + throw new Error("Credential selection changed before pre-output stream recovery"); + } + if (isCanonicalOpenAiForwardProvider(route.provider)) { + createCodexReserveDispatchGuard( + admissionState.authCtx, + options.codexAuthPolicy ?? config, + route.modelId, + options.admission, + options.visionDescribeTerminal === true, + )?.(headers); + } + // Recorded with the kind the gate derived its cause from, at the moment the + // send actually leaves. One authorisation, one recorded reason, one send. + transportState.noteRoutedAttemptSend(passthroughEstimate, "connection-reset"); + // Charged to the SAME request counter every other send goes through. The + // replacement is bought here rather than by a nested retry helper, so there is + // one charge for one send and no per-layer counter to reconcile. + noteTransientSends(1); + }, + }), + route.provider.authMode === "forward", + ); /** * Refuse a built body that exceeds the operator's configured ceiling, before it is sent. * @@ -1637,6 +1690,36 @@ export async function preparePassthroughExchange( } } + // The WebSocket row of the same table (#4191). A Codex socket that closed or failed under its + // create frame, before any Responses event, settled as a non-replayable 502 and every leg above + // let it through. Whether the turn ran upstream is as unknown as after a reset before the head, + // so the same grant decides, asked with the stage the exchange reached. The replacement's answer + // then goes round the loop like any other, and once the grant is spent nothing may send the + // turn a third time. + const socketDeathStage = codexWsSocketDeathStage(upstreamResponse); + if ( + socketDeathStage + && !upstream.signal.aborted + // Asked before the gate, which claims last: a replacement the budget cannot fund must not + // spend the request's one grant. + && remainingTransientSendBudget(transientSendAttempts()) > 0 + && authorizeResendForRecovery(socketDeathStage, "connection-reset", ambiguousResend()).allowed + ) { + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + console.warn(`[upstream-retry] codex websocket died before any Responses event (${safeHostLabel(request.url)}); ` + + "using one replacement over HTTP"); + try { + const replacement = await sendAmbiguousReplacement().then(adoptObservedResponse); + upstreamResponse = settleOperatorReplacement(replacement); + } catch (err) { + if (upstream.signal.aborted) return transportFailureResponse(err); + // The first send may already have run the turn, so a replacement that failed settles as + // the refusal rather than as a transport error the client would retry. + upstreamResponse = replayRefusalResponse(); + } + continue passthroughRecovery; + } + // The post-header row of the same table. A native SSE body can die after the head with the // caller having observed nothing, which is the identical question the pre-header helper // answers -- and the identical grant, because both claim from this request's one allowance. @@ -1660,48 +1743,7 @@ export async function preparePassthroughExchange( upstreamResponse, { model: logCtx.model, provider: logCtx.provider }, (error, stage) => refetchAfterProtocolSafeReset( - (signal = upstream.signal) => fetchWithHeaderTimeout( - request.url, - applyUpstreamRecoveryInit({ - method: request.method, - headers: request.headers, - body: request.body, - }, "connection-reset"), - signal, - connectMs, - true, - providerFetch(route.provider, options.codexWsRuntimeIdentity, { - // A replacement HTTP body must not open a fresh WebSocket exchange: the turn it - // replaces was an HTTP stream, and a WS create frame is a different send. - httpOnly: true, - providerName: route.providerName, - modelId: route.modelId, - dispatchOverride: oauthDispatch(request), - beforeDispatch: headers => { - if (signal.aborted) throw signal.reason; - if (!transportState.selectionIsCurrent(transportState.requestBindings.get(request))) { - throw new Error("Credential selection changed before pre-output stream recovery"); - } - if (isCanonicalOpenAiForwardProvider(route.provider)) { - createCodexReserveDispatchGuard( - admissionState.authCtx, - options.codexAuthPolicy ?? config, - route.modelId, - options.admission, - options.visionDescribeTerminal === true, - )?.(headers); - } - // Recorded with the kind the gate derived its cause from, at the moment the - // send actually leaves. One authorisation, one recorded reason, one send. - transportState.noteRoutedAttemptSend(passthroughEstimate, "connection-reset"); - // Charged to the SAME request counter every other send goes through. The - // replacement is bought here rather than by a nested retry helper, so there is - // one charge for one send and no per-layer counter to reconcile. - noteTransientSends(1); - }, - }), - route.provider.authMode === "forward", - ).then(adoptObservedResponse), + (signal = upstream.signal) => sendAmbiguousReplacement(signal).then(adoptObservedResponse), error, { abortSignal: upstream.signal, diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index acbe91a6044..720b14807a9 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -41,11 +41,12 @@ abort/sleep helpers from this module. ## Ambiguous-resend gate -A model POST that fails with the caller having observed nothing is one question asked at two -points: before any response head, and after a head whose SSE body carried only control events. -`src/lib/request-resend-gate.ts` is the single answer. It derives stage, cause, permission and -send class from `src/lib/request-failure-model.ts` and adds exactly one thing the table names -but does not implement — the narrowly scoped operator override for `refused-ambiguous`. +A model POST that fails with the caller having observed nothing is one question asked at three +points. Two are HTTP: before any response head, and after a head whose SSE body carried only +control events. The third is a Codex WebSocket that closes or errors under its create frame before +any Responses event (#4191). `src/lib/request-resend-gate.ts` is the single answer. It derives +stage, cause, permission and send class from `src/lib/request-failure-model.ts` and adds exactly +one thing the table names but does not implement — the narrowly scoped operator override for `refused-ambiguous`. The override is bounded on three axes at once. The provider opts in with `providers..retryOnReset`; the request must be one @@ -72,6 +73,17 @@ output cannot drain the replacement a later ambiguous reset would have been enti cause is derived from the `AttemptRecoveryKind` the send will be recorded as, which is what keeps the reason in the log and the reason the gate weighed from being two different values. +The WebSocket row is asked once, at the end of the passthrough recovery loop, after every leg has +let the settled 502 through. The exchange marks only a socket that closed or errored +(`markCodexWsSocketDeath`) and records the stage it reached: `pre-header` when nothing came back, +`protocol-prelude` when frames arrived but none was a Responses event. Silence keeps its 504, and a native steering or +injection exchange is never marked, because its channel may already have sent continuation frames +on that socket. The send budget is asked before the gate, so a replacement the request cannot fund +leaves the grant unspent. The replacement is one HTTP send, never a second socket, and its answer +is sorted exactly like the pre-header row's (see +[ambiguous connection-reset replay boundary](#ambiguous-connection-reset-replay-boundary)) before +it goes round the recovery loop again. + ## Console upload rejection recovery `src/providers/opencode-zen-rate-limit.ts` recognizes the complete Console upload-rejection envelope only at the effective HTTPS opencode.ai Zen/Go generation endpoint. A provider row name cannot authorize another destination. The two recovery loops in `src/server/responses/core.ts` wait 800 ms and replay the captured serialized request once; cancellation, nonreplayable responses, other errors and a second upload rejection keep their failure semantics. The recovery kind is persisted as `console-go-upload-retry` and has a localized Logs label. @@ -250,7 +262,8 @@ this same refusal. Nothing on that path hands the client a status that invites t to be sent again. See [ambiguous-resend gate](#ambiguous-resend-gate). That includes what the replacement send itself answers. Once the grant is spent, the first send -may already have run the turn, so `fetchWithResetRetry` sorts the replacement's answer: +may already have run the turn, so `settleOperatorReplacement` sorts the replacement's answer, for +the pre-header row in `fetchWithResetRetry` and the WebSocket row alike: | Replacement answer | Result | | --- | --- | @@ -268,14 +281,19 @@ caller, and the marker stops every recovery loop that checks it, such as the opa of a 400 or the Codex pool's gated-model retry. A combo rebuilds a failed attempt as a new response, so `consumeComboFailure` records `nonReplayable` and the combo stops rather than hopping on, say, a context overflow. The cost is that a real 401, 402 or 429 on a replacement send is not -recorded against its credential on that request. +recorded against its credential on that request. One case is not covered yet: a 2xx replacement +whose stream then fails with zero output. The combo preflight projects that terminal into a fresh +5xx that carries no marker, so a failover combo can still send the turn to its next target after +the grant was spent. That applies to all three replacement rows and is left for a follow-up. **An upstream reset observed mid-stream or after a terminal keeps its existing behaviour.** The passthrough read path still settles a genuine upstream reset as a synthetic 502, and the Codex WebSocket transport still settles `upstream_closed_before_response` (socket closed after the create frame) and `upstream_no_response` (origin never produced an event) as 502 and 504. Those describe something the upstream did after our send, they are the contract the -public server reference already documents, and this release does not move them. +public server reference already documents, and this release does not move them. Since #4191 a +provider that opted into `retryOnReset` may replace that 502 once, over HTTP, when the socket +closed or errored; the 504 is never replaced. This reclassification is the recorded behaviour change: before it, the pre-header refusal borrowed `upstream_closed_before_response` and its 502, which multiplied the duplicate send diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index 2b7db0e29a4..558ccdcc7a1 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -412,15 +412,20 @@ committed. Later quota observations update only the captured serving account; they cannot retroactively change HTTP headers already sent to the client. Control frames remain bounded, and provider credential/cookie headers are not forwarded. Once a WS create may have been sent, a missing prelude, overflow or -disconnect settles as an errored SSE body rather than a retryable fetch failure, -so HTTP fallback cannot duplicate that inference. A standalone no-response +disconnect settles as a non-replayable gateway status before the first Responses +event, or as an errored SSE body after it, rather than as a retryable fetch +failure, so HTTP fallback cannot duplicate that inference. The one exception is a +socket that closed or errored before any Responses event on a provider that opted +into `retryOnReset`: the passthrough dispatch may spend the request's replacement +grant on one HTTP send (see [ambiguous-resend gate](responses-failover.md#ambiguous-resend-gate)). +A standalone no-response exchange has a 90-second prelude deadline in addition to the upgrade deadline. That prelude deadline is a ceiling, not a floor: the exchange runs under the caller's abort signal, so a `connectTimeoutMs` shorter than 90 seconds cancels an already-sent create before the prelude timer fires. These are transport-fidelity guarantees, not a provider-billing guarantee. -Every exchange also leaves a content-free stage record (`CodexWsStageRecord`, #4191): create-frame bytes (measured on failure only — the committed-success record keeps it null so the happy path never byte-counts a megabyte replay frame), send completion, numeric close code, elapsed and first-frame durations, frame counters, liveness ping/pong counts, pool reuse, and the OCX/Bun versions. The exchange pins the record on the resolved Response (`markCodexWsStage`, the same marker seam as `markCodexWsResponse`); `handleResponses` adopts it onto the serving attempt, and usage.jsonl persists it per attempt behind a drop-guard normalizer, so hand-edited rows cannot inject strings into the DTO. Later snapshots update the same response-local record in place, so an attempt holding the committed reference observes final success or failure counters. Each exchange supplies a complete fresh snapshot; separate responses keep distinct records. On eager-relay cancel-drain expiry, upstream cancellation finalizes the transport snapshot before the cancellation hook writes the usage row; an actual terminal observed within the drain still wins over cancellation. The record never carries conversation text, headers, close-reason text, or account identifiers, and it is not a fallback-eligibility signal: the no-replay-after-send contract stands regardless of what it says. +Every exchange also leaves a content-free stage record (`CodexWsStageRecord`, #4191): create-frame bytes (measured on failure only — the committed-success record keeps it null so the happy path never byte-counts a megabyte replay frame), send completion, numeric close code, elapsed and first-frame durations, frame counters, liveness ping/pong counts, pool reuse, and the OCX/Bun versions. The exchange pins the record on the resolved Response (`markCodexWsStage`, the same marker seam as `markCodexWsResponse`); `handleResponses` adopts it onto the serving attempt, and usage.jsonl persists it per attempt behind a drop-guard normalizer, so hand-edited rows cannot inject strings into the DTO. Later snapshots update the same response-local record in place, so an attempt holding the committed reference observes final success or failure counters. Each exchange supplies a complete fresh snapshot; separate responses keep distinct records. On eager-relay cancel-drain expiry, upstream cancellation finalizes the transport snapshot before the cancellation hook writes the usage row; an actual terminal observed within the drain still wins over cancellation. The record never carries conversation text, headers, close-reason text, or account identifiers, and it is not a fallback-eligibility signal: nothing it says permits a resend. The one replacement an operator can grant after a socket dies is the resend gate's decision (see [ambiguous-resend gate](responses-failover.md#ambiguous-resend-gate)). Eligible complete-input creates can retain a canonical upstream socket within one selected account, credential, thread and turn. Model/tier and immutable diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 729a15b53e6..34c3aaef092 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1534,6 +1534,7 @@ "winsw-stop-hardening.test.ts": "windows", "winsw.test.ts": "service", "workflow-budget.test.ts": "lib", + "ws-ambiguous-resend.test.ts": "responses", "ws-endpoint.test.ts": "responses", "ws-failure-stage.test.ts": "responses", "ws-native-injection.test.ts": "responses", diff --git a/tests/responses/ws-ambiguous-resend.test.ts b/tests/responses/ws-ambiguous-resend.test.ts new file mode 100644 index 00000000000..38b89e18a3e --- /dev/null +++ b/tests/responses/ws-ambiguous-resend.test.ts @@ -0,0 +1,340 @@ +import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test"; +import { handleResponses } from "../../src/server/responses"; +import { codexWsExchange } from "../../src/server/responses/codex-ws-exchange"; +import { CodexWsSession } from "../../src/server/responses/codex-ws-session"; +import { prepareCodexWsRequest } from "../../src/server/responses/codex-ws-request"; +import { + CODEX_WS_RESPONSE_PRELUDE_TIMEOUT_MS, + codexWsSocketDeathStage, + readCodexWsStage, +} from "../../src/server/responses/codex-ws-wire"; +import { + isNonReplayableResponse, + REPLAY_REFUSED_STATUS, + UPSTREAM_RESET_REPLAY_REFUSED_CODE, +} from "../../src/lib/upstream-retry"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; +import { BOUNDED_WS_RUNTIME, codexWsUpstreamFetch, streamingInit } from "../helpers/ws-upstream-fixtures"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; + +/** + * #4191: a Codex WebSocket that dies after its create frame left, before any Responses event, + * leaves the turn in the same unknown state as an HTTP connection that resets before the head. + * The HTTP rows already answer that with the operator's `retryOnReset` grant. These cases hold the + * WebSocket to the same answer: one replacement, over HTTP, only when the grant covers it, and + * never a third send of the turn. + */ + +const CODEX_URL = "https://chatgpt.com/backend-api/codex/responses"; + +type Listener = (event: unknown) => void; + +/** Minimal scriptable stand-in for Bun's WebSocket, mirroring `ws-failure-stage.test.ts`. */ +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + static script: (ws: FakeWebSocket) => void = () => {}; + url: string; + sent: string[] = []; + closed = false; + listeners = new Map(); + + constructor(url: string) { + this.url = url; + FakeWebSocket.instances.push(this); + queueMicrotask(() => FakeWebSocket.script(this)); + } + + addEventListener(type: string, listener: Listener) { + const list = this.listeners.get(type) ?? []; + list.push(listener); + this.listeners.set(type, list); + } + + removeEventListener(type: string, listener: Listener) { + this.listeners.set(type, (this.listeners.get(type) ?? []).filter(value => value !== listener)); + } + + emit(type: string, event: unknown = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + send(data: string) { + this.sent.push(data); + } + + close() { this.closed = true; } +} + +const RealWebSocket = globalThis.WebSocket; +const RealFetch = globalThis.fetch; +const PROXY_ENV_KEYS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const; +let savedProxyEnv: Record; +// A case that calls handleResponses directly never takes the writer lease startServer takes, +// so its dispatch is refused. Dropped in teardown so a throwing case cannot leave it behind. +let releaseSpendHome: (() => void) | undefined; +const takeSpendHome = (): void => { releaseSpendHome ??= acquireOwnedSpendHome(); }; + +beforeEach(() => { + savedProxyEnv = Object.fromEntries(PROXY_ENV_KEYS.map(key => [key, process.env[key]])); + for (const key of PROXY_ENV_KEYS) delete process.env[key]; + FakeWebSocket.instances = []; + FakeWebSocket.script = () => {}; +}); + +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.WebSocket = RealWebSocket; + globalThis.fetch = RealFetch; + FakeWebSocket.instances = []; + FakeWebSocket.script = () => {}; + for (const key of PROXY_ENV_KEYS) { + if (savedProxyEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedProxyEnv[key]; + } +}); + +function installFake(script: (ws: FakeWebSocket) => void) { + FakeWebSocket.script = script; + globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket; +} + +const QUOTA_FRAME = JSON.stringify({ + type: "codex.rate_limits", rate_limits: { primary: { used_percent: 10, window_minutes: 10080 } }, +}); + +/** The three ways a socket can die under the send before anything was promised to the client. */ +const SOCKET_DEATHS: Array<[string, (ws: FakeWebSocket) => void, "pre-header" | "protocol-prelude"]> = [ + ["nothing came back", ws => { + ws.emit("open", {}); + ws.emit("close", { code: 1006 }); + }, "pre-header"], + ["only quota came back", ws => { + ws.emit("open", {}); + ws.emit("message", { data: QUOTA_FRAME }); + ws.emit("close", { code: 1006 }); + }, "protocol-prelude"], + ["the transport errored", ws => { + ws.emit("open", {}); + ws.emit("error", {}); + }, "pre-header"], +]; + +const noFallback = (async () => { + throw new Error("fallback must not run after open"); +}) as unknown as typeof fetch; + +describe("the exchange records a socket that died under the send (#4191)", () => { + test.each(SOCKET_DEATHS)("when %s it settles the same 502, marked with the stage it reached", + async (_name, script, stage) => { + installFake(script); + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), noFallback); + expect(response.status).toBe(502); + expect(isNonReplayableResponse(response)).toBe(true); + expect(readCodexWsStage(response)?.sent).toBe(true); + expect(codexWsSocketDeathStage(response)).toBe(stage); + }); + + test("silence keeps its 504 and is not a socket death", async () => { + jest.useFakeTimers(); + const opened = Promise.withResolvers(); + try { + installFake(ws => { ws.emit("open", {}); opened.resolve(); }); + const pending = codexWsUpstreamFetch(CODEX_URL, streamingInit(), noFallback); + await opened.promise; + jest.advanceTimersByTime(CODEX_WS_RESPONSE_PRELUDE_TIMEOUT_MS); + const response = await pending; + expect(response.status).toBe(504); + expect(codexWsSocketDeathStage(response)).toBeUndefined(); + } finally { + jest.useRealTimers(); + } + }); + + test("a drop after the response started stays a failed body and is not a socket death", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.created", response: { id: "r1" } }) }); + ws.emit("close", { code: 1006 }); + }); + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), noFallback); + expect(response.status).toBe(200); + expect(codexWsSocketDeathStage(response)).toBeUndefined(); + await expect(response.text()).rejects.toThrow("closed before a Responses terminal event"); + }); + + test("a steering exchange's death is not offered: its channel may have sent more than the create", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("close", { code: 1006 }); + }); + const init = streamingInit(); + const prepared = prepareCodexWsRequest(CODEX_URL, init)!; + const session = new CodexWsSession("wss://chatgpt.com/backend-api/codex/responses", prepared.headers, true); + const nativeControl = { + kind: "steering" as const, + relayActive: false, + attached: false, + ended: false, + attach() { return () => {}; }, + observe() { return false; }, + steer() {}, + continue() { return false; }, + }; + try { + expect(session.reserve()).toBe(true); + const response = await codexWsExchange({ session, url: CODEX_URL, init, prepared, nativeControl, sseFallback: noFallback }); + expect(response.status).toBe(502); + expect(codexWsSocketDeathStage(response)).toBeUndefined(); + } finally { session.dispose(); } + }); +}); + +describe("handleResponses replaces a dead socket's send once under retryOnReset (#4191)", () => { + function forwardConfig(provider: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "openai", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "direct", + ...provider, + }, + }, + } as OcxConfig; + } + + /** A turn whose second send can only repeat the inference: nothing stored, no hosted tools. */ + function turn(body: Record = {}): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer test" }, + body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: true, store: false, ...body }), + }); + } + + function completed(): Response { + return new Response(`event: response.completed\ndata: ${JSON.stringify({ + type: "response.completed", + response: { id: "r-http", status: "completed", output: [] }, + })}\n\n`, { status: 200, headers: { "content-type": "text/event-stream" } }); + } + + /** Every HTTP send reaches upstream through here, so its length is the number of HTTP sends. */ + function stubHttp(answer: () => Response): string[] { + const bodies: string[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push(typeof init?.body === "string" ? init.body : ""); + return answer(); + }) as typeof fetch; + return bodies; + } + + async function send( + request: Request, + config: OcxConfig, + logCtx: RequestLogContext = { model: "", provider: "" }, + sendBudget = createRequestExecutionBudget(), + ): Promise { + takeSpendHome(); + return handleResponses(request, config, logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, sendBudget }); + } + + test.each(SOCKET_DEATHS)("when %s, one HTTP send of the same turn serves it", async (_name, script) => { + installFake(script); + const http = stubHttp(completed); + const logCtx: RequestLogContext = { model: "", provider: "" }; + const response = await send(turn(), forwardConfig({ retryOnReset: {} }), logCtx); + + expect(response.status).toBe(200); + expect(await response.text()).toContain("response.completed"); + // Over HTTP: the transport that just failed is not asked again. + expect(FakeWebSocket.instances).toHaveLength(1); + expect(http).toHaveLength(1); + const frame = JSON.parse(FakeWebSocket.instances[0]!.sent[0]!) as { input?: unknown }; + expect((JSON.parse(http[0]!) as { input?: unknown }).input).toEqual(frame.input); + // Both sends are on the record, and the dead socket's evidence stays beside its replacement. + expect(logCtx.activeAttempt?.sendCount).toBe(2); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["connection-reset"]); + expect(logCtx.activeAttempt?.codexWsStage?.sent).toBe(true); + }); + + test.each([ + ["the provider grants nothing", {}, {}], + ["the turn is stored upstream", { retryOnReset: {} }, { store: true }], + ])("the 502 stands and nothing else is sent when %s", async (_name, provider, body) => { + installFake(SOCKET_DEATHS[0]![1]); + const http = stubHttp(completed); + const response = await send(turn(body), forwardConfig(provider)); + + expect(response.status).toBe(502); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(http).toHaveLength(0); + }); + + test.each([ + ["a status the client would retry", () => new Response("busy", { status: 503 })], + ["a reset of its own", () => { throw Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }); }], + ])("a replacement that fails with %s settles as the refusal, with no third send", async (_name, answer) => { + installFake(SOCKET_DEATHS[0]![1]); + const http = stubHttp(answer); + const response = await send(turn(), forwardConfig({ retryOnReset: {} })); + + expect(response.status).toBe(REPLAY_REFUSED_STATUS); + expect(await response.json()).toMatchObject({ error: { code: UPSTREAM_RESET_REPLAY_REFUSED_CODE } }); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(http).toHaveLength(1); + }); + + test("a spent replacement's effort rejection does not start a downgrade send", async () => { + installFake(SOCKET_DEATHS[0]![1]); + const http = stubHttp(() => Response.json( + { error: { param: "reasoning.effort", message: "Unsupported reasoning effort" } }, + { status: 400 }, + )); + const config = forwardConfig({ retryOnReset: {}, reasoningEfforts: ["low", "high"] }); + const response = await send(turn({ reasoning: { effort: "high" } }), config); + + // The 400 answers the replacement, not the send that may already have run the turn. + expect(response.status).toBe(400); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(http).toHaveLength(1); + }); + + test("the grant is not spent on a replacement the send budget cannot fund", async () => { + installFake(SOCKET_DEATHS[0]![1]); + const http = stubHttp(completed); + const sendBudget = createRequestExecutionBudget(); + // Room for the socket's own send and nothing after it. + sendBudget.used = sendBudget.policy.baseSendAllowance - 1; + const response = await send(turn(), forwardConfig({ retryOnReset: {} }), undefined, sendBudget); + + expect(response.status).toBe(502); + expect(http).toHaveLength(0); + expect(sendBudget.claimAmbiguousResend?.(1)).toBe(true); + }); + + test("the SSE row cannot buy a second replacement after the socket's", async () => { + installFake(SOCKET_DEATHS[0]![1]); + const encoder = new TextEncoder(); + // The replacement's stream dies after its prelude with nothing written, the SSE row's case. + const http = stubHttp(() => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(`event: response.created\ndata: ${JSON.stringify({ + type: "response.created", response: { id: "r-http", status: "in_progress" }, + })}\n\n`)); + controller.error(Object.assign(new Error("socket hang up"), { code: "ECONNRESET" })); + }, + }), { status: 200, headers: { "content-type": "text/event-stream" } })); + const response = await send(turn(), forwardConfig({ retryOnReset: {} })); + await response.text().catch(() => ""); + + expect(FakeWebSocket.instances).toHaveLength(1); + expect(http).toHaveLength(1); + }); +}); From a0142d77cc5c9c310b62222e6d764cd7225f5906 Mon Sep 17 00:00:00 2001 From: Fred Amartey <43480311+FredAmartey@users.noreply.github.com> Date: Wed, 23 Sep 2026 03:53:37 -0400 Subject: [PATCH 2/3] docs(structure): state the WebSocket replacement rule once The replay boundary paragraph said the release does not move the 502 and then that an opted-in provider may replace it. It now says one thing: the 504 and a drop after the response started are never replaced, and only the 502 of a socket that closed or errored before any Responses event may be replaced once over HTTP. The open-gap sentence points at #5646. --- structure/transports/responses-failover.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 720b14807a9..fdfb81d2dca 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -284,16 +284,16 @@ on, say, a context overflow. The cost is that a real 401, 402 or 429 on a replac recorded against its credential on that request. One case is not covered yet: a 2xx replacement whose stream then fails with zero output. The combo preflight projects that terminal into a fresh 5xx that carries no marker, so a failover combo can still send the turn to its next target after -the grant was spent. That applies to all three replacement rows and is left for a follow-up. +the grant was spent. That applies to all three replacement rows; #5646 closes it. **An upstream reset observed mid-stream or after a terminal keeps its existing behaviour.** The passthrough read path still settles a genuine upstream reset as a synthetic 502, and the Codex WebSocket transport still settles `upstream_closed_before_response` (socket closed after the create frame) and `upstream_no_response` (origin never produced an event) as 502 -and 504. Those describe something the upstream did after our send, they are the contract the -public server reference already documents, and this release does not move them. Since #4191 a -provider that opted into `retryOnReset` may replace that 502 once, over HTTP, when the socket -closed or errored; the 504 is never replaced. +and 504. Those describe something the upstream did after our send, and they are the contract +the public server reference already documents. The 504 and a drop after the response started +are never replaced. Only the 502 of a socket that closed or errored before any Responses event +may be replaced, once, over HTTP, when the provider opted into `retryOnReset` (#4191). This reclassification is the recorded behaviour change: before it, the pre-header refusal borrowed `upstream_closed_before_response` and its 502, which multiplied the duplicate send From 6593874499b040dfa81347c725cf0614f2f1c49c Mon Sep 17 00:00:00 2001 From: Fred Amartey <43480311+FredAmartey@users.noreply.github.com> Date: Wed, 23 Sep 2026 03:53:38 -0400 Subject: [PATCH 3/3] test(responses): cover pool accounts for the WebSocket replacement In pool mode three account ladders could send the turn a third time after the HTTP replacement: quota rotation on the refusal's 429, the unsupported model retry on a kept 400, and the transient rotation on the dead socket's own 502. Each case pins one socket and one HTTP send, both on the first account, with nothing sent under the second account's credentials. --- tests/responses/ws-ambiguous-resend.test.ts | 117 +++++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/tests/responses/ws-ambiguous-resend.test.ts b/tests/responses/ws-ambiguous-resend.test.ts index 38b89e18a3e..d32c7009a81 100644 --- a/tests/responses/ws-ambiguous-resend.test.ts +++ b/tests/responses/ws-ambiguous-resend.test.ts @@ -1,4 +1,12 @@ import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearAccountNeedsReauth } from "../../src/codex/auth-api"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { clearAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; import { handleResponses } from "../../src/server/responses"; import { codexWsExchange } from "../../src/server/responses/codex-ws-exchange"; import { CodexWsSession } from "../../src/server/responses/codex-ws-session"; @@ -18,6 +26,7 @@ import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { BOUNDED_WS_RUNTIME, codexWsUpstreamFetch, streamingInit } from "../helpers/ws-upstream-fixtures"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; /** * #4191: a Codex WebSocket that dies after its create frame left, before any Responses event, @@ -36,12 +45,14 @@ class FakeWebSocket { static instances: FakeWebSocket[] = []; static script: (ws: FakeWebSocket) => void = () => {}; url: string; + headers: Headers; sent: string[] = []; closed = false; listeners = new Map(); - constructor(url: string) { + constructor(url: string, options?: { headers?: HeadersInit }) { this.url = url; + this.headers = new Headers(options?.headers); FakeWebSocket.instances.push(this); queueMicrotask(() => FakeWebSocket.script(this)); } @@ -245,6 +256,110 @@ describe("handleResponses replaces a dead socket's send once under retryOnReset return handleResponses(request, config, logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, sendBudget }); } + describe("in pool mode", () => { + const ACCOUNT_ID = "work"; + const OTHER_ACCOUNT_ID = "other"; + const HOME_KEYS = ["HOME", "OPENCODEX_HOME", "CODEX_HOME"] as const; + let home = ""; + let previousHomes: Array; + + function clearPoolState(): void { + clearAccountNeedsReauth(ACCOUNT_ID); + clearAccountNeedsReauth(OTHER_ACCOUNT_ID); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(); + clearAccountQuota(); + } + + beforeEach(() => { + previousHomes = HOME_KEYS.map(key => process.env[key]); + home = mkdtempSync(join(tmpdir(), "ocx-ws-ambiguous-pool-")); + for (const key of HOME_KEYS) process.env[key] = home; + takeSpendHome(); + clearPoolState(); + // A primed pool does not issue unrelated background usage requests during the turn. + for (const id of [ACCOUNT_ID, OTHER_ACCOUNT_ID]) { + setAccountQuotaFromParsed(id, { weeklyPercent: 10 }); + } + writeFileSync(join(home, "codex-accounts.json"), JSON.stringify(Object.fromEntries( + [ACCOUNT_ID, OTHER_ACCOUNT_ID].map(id => [id, { + credential: { + accessToken: `${id}-access`, + refreshToken: `${id}-grant`, + expiresAt: Date.now() + 3_600_000, + chatgptAccountId: `acc-${id}`, + }, + generation: 1, + refreshGrantFingerprint: createHash("sha256") + .update(`codex-refresh-grant:${id}-grant`).digest("hex"), + }]), + ))); + }); + + afterEach(() => { + // Release the writer before removing its database or restoring the surrounding home. + releaseSpendHome?.(); + releaseSpendHome = undefined; + clearPoolState(); + for (const [index, key] of HOME_KEYS.entries()) { + if (previousHomes[index] === undefined) delete process.env[key]; + else process.env[key] = previousHomes[index]; + } + removeTreeWithRetry(home); + }); + + for (const [status, body] of [ + [429, JSON.stringify({ error: { message: "quota exhausted" } })], + [503, "busy"], + [400, JSON.stringify({ + detail: "The 'gpt-5.5' model is not supported when using Codex with a ChatGPT account.", + })], + ] as const) { + test(`a replacement ${status} cannot send the turn through the second account`, async () => { + installFake(SOCKET_DEATHS[0]![1]); + const http: Headers[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + http.push(new Headers(init?.headers)); + return new Response(body, { status }); + }) as typeof fetch; + const config: OcxConfig = { + ...forwardConfig({ codexAccountMode: "pool", retryOnReset: {} }), + activeCodexAccountId: ACCOUNT_ID, + autoSwitchThreshold: 0, + accountPoolStrategy: "round-robin", + codexAccounts: [{ id: ACCOUNT_ID, label: "work" }, { id: OTHER_ACCOUNT_ID, label: "other" }], + }; + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: true, store: false }), + }); + const response = await send(request, config); + + // Both transports count: the dead socket's 502 must not rotate before the HTTP row. + const credentials = [...FakeWebSocket.instances.map(ws => ws.headers), ...http]; + expect(credentials.map(headers => headers.get("authorization"))).not.toContain("Bearer other-access"); + expect(FakeWebSocket.instances).toHaveLength(1); + const socket = FakeWebSocket.instances[0]!; + expect(socket.headers.get("authorization")).toBe("Bearer work-access"); + expect(socket.sent).toHaveLength(1); + expect(JSON.parse(socket.sent[0]!)).toMatchObject({ type: "response.create" }); + expect(http).toHaveLength(1); + expect(http[0]!.get("authorization")).toBe("Bearer work-access"); + expect(http[0]!.get("chatgpt-account-id")).toBe("acc-work"); + if (status === 400) { + expect(response.status).toBe(400); + expect(await response.text()).toBe(body); + } else { + expect(response.status).toBe(REPLAY_REFUSED_STATUS); + expect(response.headers.get("x-should-retry")).toBe("false"); + expect(await response.json()).toMatchObject({ error: { code: UPSTREAM_RESET_REPLAY_REFUSED_CODE } }); + } + }); + } + }); + test.each(SOCKET_DEATHS)("when %s, one HTTP send of the same turn serves it", async (_name, script) => { installFake(script); const http = stubHttp(completed);