From 3a122d09d58ffef5d7115f1f0ca0a2886810fedf Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:51:14 +0900 Subject: [PATCH 1/2] Keep malformed UTF-8 from erasing a cyber-policy stop --- src/lib/bounded-body.ts | 25 +++++++++++++++++++ src/server/responses/core-combo-failure.ts | 7 +++--- .../codex-quota-rejection.test.ts | 2 +- .../cyber-policy-error-fidelity.test.ts | 13 ++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/lib/bounded-body.ts b/src/lib/bounded-body.ts index 0b3769eaac..225ddfd1de 100644 --- a/src/lib/bounded-body.ts +++ b/src/lib/bounded-body.ts @@ -15,6 +15,8 @@ export interface BoundedBodyOptions { * Reader cancellation and lock release still run. Defaults to false. */ fatalUtf8?: boolean; + /** Report UTF-8 validity without rejecting malformed bodies. */ + reportUtf8Validity?: boolean; /** * Byte ceiling for retained body data. Defaults to BOUNDED_BODY_MAX_BYTES (64 KiB), * which suits error bodies; callers materializing whole success payloads (e.g. a @@ -44,6 +46,8 @@ export interface BoundedBodyResult { oversized: boolean; /** False means callers should use a status-only fallback, not `text`. */ displaySafe: boolean; + /** Present when reportUtf8Validity was requested and the retained body reached EOF. */ + utf8Valid?: boolean; } export interface BoundedBytesOptions { @@ -238,6 +242,14 @@ function decodeUtf8(chunks: readonly Uint8Array[], fatal: boolean, timedOut = fa } } +function decodeUtf8WithValidity(bytes: Uint8Array): { text: string; utf8Valid: boolean } { + try { + return { text: decodeUtf8([bytes], true), utf8Valid: true }; + } catch { + return { text: decodeUtf8([bytes], false), utf8Valid: false }; + } +} + /** * Consume the original response body under strict memory and time bounds. * @@ -326,6 +338,19 @@ export async function readBoundedResponseBody( const { value, done } = outcome as ReadableStreamReadResult; if (done) { + if (options.reportUtf8Validity && options.fatalUtf8 !== true) { + const decoded = decodeUtf8WithValidity(retained.subarray(0, retainedBytes)); + return { + text: decoded.text, + truncated: false, + timedOut: false, + totalTimedOut: false, + inactivityTimedOut: false, + oversized: false, + displaySafe: true, + utf8Valid: decoded.utf8Valid, + }; + } return { text: decodeUtf8([retained.subarray(0, retainedBytes)], options.fatalUtf8 === true), truncated: false, diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index c35fc90a3d..5178cac10c 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -44,13 +44,14 @@ export async function consumeComboFailure( try { const body = await readBoundedResponseBody(response, { signal, - // Match shouldRetryCodexPoolAccountQuota before treating a 5xx body as quota evidence. - fatalUtf8: response.status >= 500 && response.status < 600, + // Quota evidence requires valid UTF-8, while other classifications still use the + // bounded replacement-decoded text (notably cyber-policy failures, which must stop). + reportUtf8Validity: response.status >= 500 && response.status < 600, }); usage = usageFromComboFailureText(body.text); if ( response.status >= 500 && response.status < 600 - && body.displaySafe && !body.truncated + && body.displaySafe && !body.truncated && body.utf8Valid === true ) { const quotaMessage = codexQuotaFailureMessage(body.text); quotaConfirmedByBody = quotaMessage !== undefined diff --git a/tests/codex-integration/codex-quota-rejection.test.ts b/tests/codex-integration/codex-quota-rejection.test.ts index 9327b33db2..945f3a1e7c 100644 --- a/tests/codex-integration/codex-quota-rejection.test.ts +++ b/tests/codex-integration/codex-quota-rejection.test.ts @@ -187,7 +187,7 @@ describe("Codex pre-stream quota rejection classification", () => { expect(failure.classificationText).toContain("The usage limit has been reached"); } else { expect(failure.resetAt).toBeUndefined(); - expect(failure.classificationText).toBe("Provider error 503"); + expect(failure.classificationText).toContain("The usage limit has been reached"); } expect(response.bodyUsed).toBe(true); }); diff --git a/tests/providers/cyber-policy-error-fidelity.test.ts b/tests/providers/cyber-policy-error-fidelity.test.ts index 7adc39f3f2..8738320235 100644 --- a/tests/providers/cyber-policy-error-fidelity.test.ts +++ b/tests/providers/cyber-policy-error-fidelity.test.ts @@ -180,6 +180,19 @@ describe("cyber_policy error fidelity", () => { }); }); + test("malformed UTF-8 does not erase a cyber-policy stop", async () => { + const bytes = new Uint8Array([ + ...new TextEncoder().encode(JSON.stringify(CYBER_ERROR_BODY)), + 0xff, + ]); + const failure = await consumeComboFailure(new Response(bytes, { status: 502 })); + expect(failure.response.status).toBe(400); + expect(failure.upstreamCode).toBe(CYBER_POLICY_ERROR_CODE); + expect(comboFailureDecision(502, failure.classificationText, { + code: failure.upstreamCode, + })).toBe("stop"); + }); + test("drops Codex reset headers as well as Retry-After for a cyber-policy failure", async () => { const upstream = new Response(JSON.stringify(CYBER_ERROR_BODY), { status: 429, From f7f5b4b138ae60f77de67361d4d7dde2f889cee8 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:49:39 +0900 Subject: [PATCH 2/2] fix(bounded-body): report utf8Valid on the fatal decode EOF path A caller combining reportUtf8Validity with fatalUtf8 got no utf8Valid field for a valid body at EOF, breaking the BoundedBodyResult contract. The reporting branch now runs whenever reporting is requested: a successful fatal decode already proved validity, while malformed input still throws. --- src/lib/bounded-body.ts | 9 +++++++-- tests/server/bounded-body.test.ts | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/lib/bounded-body.ts b/src/lib/bounded-body.ts index 225ddfd1de..effb247fb7 100644 --- a/src/lib/bounded-body.ts +++ b/src/lib/bounded-body.ts @@ -338,8 +338,13 @@ export async function readBoundedResponseBody( const { value, done } = outcome as ReadableStreamReadResult; if (done) { - if (options.reportUtf8Validity && options.fatalUtf8 !== true) { - const decoded = decodeUtf8WithValidity(retained.subarray(0, retainedBytes)); + if (options.reportUtf8Validity) { + const bytes = retained.subarray(0, retainedBytes); + // A fatal decode that returned already proved the bytes valid; still + // honour the reporting contract instead of dropping utf8Valid. + const decoded = options.fatalUtf8 === true + ? { text: decodeUtf8([bytes], true), utf8Valid: true } + : decodeUtf8WithValidity(bytes); return { text: decoded.text, truncated: false, diff --git a/tests/server/bounded-body.test.ts b/tests/server/bounded-body.test.ts index 2a142719ae..54bbe6ddcd 100644 --- a/tests/server/bounded-body.test.ts +++ b/tests/server/bounded-body.test.ts @@ -22,6 +22,22 @@ function responseFromChunks(...chunks: Uint8Array[]): Response { } describe("readBoundedResponseBody", () => { + test("reportUtf8Validity is honoured on the fatal decode path at EOF", async () => { + const valid = await readBoundedResponseBody(responseFromChunks(encoder.encode('{"ok":true}')), { + fatalUtf8: true, + reportUtf8Validity: true, + }); + expect(valid.utf8Valid).toBe(true); + let caught: unknown; + try { + await readBoundedResponseBody(responseFromChunks(new Uint8Array([0xff])), { + fatalUtf8: true, + reportUtf8Validity: true, + }); + } catch (error) { caught = error; } + expect(boundedBodyDecodeFailure(caught)).toBe("invalid_utf8"); + }); + test("only actual decoder exceptions carry the decode discriminator", async () => { for (const bytes of [new Uint8Array([0xff]), new Uint8Array([0xe2, 0x82])]) { let caught: unknown;