Keep malformed UTF-8 from erasing a cyber-policy stop

src/lib/bounded-body.ts

@@ -15,6 +15,8 @@ export interface BoundedBodyOptions {
 	 * Reader cancellation and lock release still run. Defaults to false.
 	 */
 	fatalUtf8?: boolean;
+	/** Report UTF-8 validity without rejecting malformed bodies. */
+	reportUtf8Validity?: boolean;
 	/**
 	 * Byte ceiling for retained body data. Defaults to BOUNDED_BODY_MAX_BYTES (64 KiB),
 	 * which suits error bodies; callers materializing whole success payloads (e.g. a
@@ -44,6 +46,8 @@ export interface BoundedBodyResult {
 	oversized: boolean;
 	/** False means callers should use a status-only fallback, not `text`. */
 	displaySafe: boolean;
+	/** Present when reportUtf8Validity was requested and the retained body reached EOF. */
+	utf8Valid?: boolean;
 }
 
 export interface BoundedBytesOptions {
@@ -238,6 +242,14 @@ function decodeUtf8(chunks: readonly Uint8Array[], fatal: boolean, timedOut = fa
 	}
 }
 
+function decodeUtf8WithValidity(bytes: Uint8Array): { text: string; utf8Valid: boolean } {
+	try {
+		return { text: decodeUtf8([bytes], true), utf8Valid: true };
+	} catch {
+		return { text: decodeUtf8([bytes], false), utf8Valid: false };
+	}
+}
+
 /**
  * Consume the original response body under strict memory and time bounds.
  *
@@ -326,6 +338,24 @@ export async function readBoundedResponseBody(
 
 			const { value, done } = outcome as ReadableStreamReadResult<Uint8Array>;
 			if (done) {
+				if (options.reportUtf8Validity) {
+					const bytes = retained.subarray(0, retainedBytes);
+					// A fatal decode that returned already proved the bytes valid; still
+					// honour the reporting contract instead of dropping utf8Valid.
+					const decoded = options.fatalUtf8 === true
+						? { text: decodeUtf8([bytes], true), utf8Valid: true }
+						: decodeUtf8WithValidity(bytes);
+					return {
+						text: decoded.text,
+						truncated: false,
+						timedOut: false,
+						totalTimedOut: false,
+						inactivityTimedOut: false,
+						oversized: false,
+						displaySafe: true,
+						utf8Valid: decoded.utf8Valid,
+					};
+				}
 				return {
 					text: decodeUtf8([retained.subarray(0, retainedBytes)], options.fatalUtf8 === true),
 					truncated: false,

src/server/responses/core-combo-failure.ts

@@ -44,13 +44,14 @@ export async function consumeComboFailure(
   try {
     const body = await readBoundedResponseBody(response, {
       signal,
-      // Match shouldRetryCodexPoolAccountQuota before treating a 5xx body as quota evidence.
-      fatalUtf8: response.status >= 500 && response.status < 600,
+      // Quota evidence requires valid UTF-8, while other classifications still use the
+      // bounded replacement-decoded text (notably cyber-policy failures, which must stop).
+      reportUtf8Validity: response.status >= 500 && response.status < 600,
     });
     usage = usageFromComboFailureText(body.text);
     if (
       response.status >= 500 && response.status < 600
-      && body.displaySafe && !body.truncated
+      && body.displaySafe && !body.truncated && body.utf8Valid === true
     ) {
       const quotaMessage = codexQuotaFailureMessage(body.text);
       quotaConfirmedByBody = quotaMessage !== undefined

tests/codex-integration/codex-quota-rejection.test.ts

@@ -187,7 +187,7 @@ describe("Codex pre-stream quota rejection classification", () => {
       expect(failure.classificationText).toContain("The usage limit has been reached");
     } else {
       expect(failure.resetAt).toBeUndefined();
-      expect(failure.classificationText).toBe("Provider error 503");
+      expect(failure.classificationText).toContain("The usage limit has been reached");
     }
     expect(response.bodyUsed).toBe(true);
   });

tests/providers/cyber-policy-error-fidelity.test.ts

@@ -180,6 +180,19 @@ describe("cyber_policy error fidelity", () => {
     });
   });
 
+  test("malformed UTF-8 does not erase a cyber-policy stop", async () => {
+    const bytes = new Uint8Array([
+      ...new TextEncoder().encode(JSON.stringify(CYBER_ERROR_BODY)),
+      0xff,
+    ]);
+    const failure = await consumeComboFailure(new Response(bytes, { status: 502 }));
+    expect(failure.response.status).toBe(400);
+    expect(failure.upstreamCode).toBe(CYBER_POLICY_ERROR_CODE);
+    expect(comboFailureDecision(502, failure.classificationText, {
+      code: failure.upstreamCode,
+    })).toBe("stop");
+  });
+
   test("drops Codex reset headers as well as Retry-After for a cyber-policy failure", async () => {
     const upstream = new Response(JSON.stringify(CYBER_ERROR_BODY), {
       status: 429,

tests/server/bounded-body.test.ts

@@ -22,6 +22,22 @@ function responseFromChunks(...chunks: Uint8Array[]): Response {
 }
 
 describe("readBoundedResponseBody", () => {
+	test("reportUtf8Validity is honoured on the fatal decode path at EOF", async () => {
+		const valid = await readBoundedResponseBody(responseFromChunks(encoder.encode('{"ok":true}')), {
+			fatalUtf8: true,
+			reportUtf8Validity: true,
+		});
+		expect(valid.utf8Valid).toBe(true);
+		let caught: unknown;
+		try {
+			await readBoundedResponseBody(responseFromChunks(new Uint8Array([0xff])), {
+				fatalUtf8: true,
+				reportUtf8Validity: true,
+			});
+		} catch (error) { caught = error; }
+		expect(boundedBodyDecodeFailure(caught)).toBe("invalid_utf8");
+	});
+
 	test("only actual decoder exceptions carry the decode discriminator", async () => {
 		for (const bytes of [new Uint8Array([0xff]), new Uint8Array([0xe2, 0x82])]) {
 			let caught: unknown;