Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -732,19 +732,26 @@ export async function executeComboResponses(
);
const failureNow = Date.now();
const attemptedTargets = pick.attempted;
const failureCooldownScope = comboFailureCooldownScope(failure.response.status, failure.classificationText, {
code: failure.upstreamCode,
});
const nextPick = advanceComboAfterFailure(config, pick, {
retryAfter: failure.retryAfter,
resetAt: failure.resetAt,
cooldownMs: combo.cooldownMs,
now: failureNow,
cooldownScope: comboFailureCooldownScope(failure.response.status, failure.classificationText, {
code: failure.upstreamCode,
}),
cooldownScope: failureCooldownScope,
eligible: targetEligible,
status: failure.response.status,
code: failure.upstreamCode,
message: failure.classificationText,
});
// Cooldown state is shared by every request using this target, so a concurrent failure
// can put it in cooldown while THIS failure recorded none. Scope "none" means the
// refusal described this request's shape rather than the target's health — only a
// cooldown this failure produced itself may arm the single-target retry below.
const failedTargetCooled = failureCooldownScope !== "none"
&& isComboTargetInCooldown(comboId, pick.target, failureNow);
// Same target selector as the exclusionary pick below, minus `exclude`: the only
// difference is deliberate and is the whole point of the single-target retry.
const retryAfterCooldown = () =>
Expand Down Expand Up @@ -775,6 +782,7 @@ export async function executeComboResponses(
&& combo.targets.length === 1
&& combo.waitForCooldownMs > 0
&& comboTargetsDispatched <= 1
&& failedTargetCooled
&& !options.abortSignal?.aborted
) {
pick = await retryAfterCooldown();
Expand Down
4 changes: 4 additions & 0 deletions tests/adapters/openai/openai-chat-dangling-toolcalls.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ const provider: OcxProviderConfig = {
baseUrl: "https://example.test/v1",
apiKey: "sk-test",
authMode: "key",
// The wire role folds to `system` unless a destination is recorded as accepting
// `developer`; this suite is about tool-result repair ordering, so it declares the
// destination rather than asserting the default.
foldDeveloperRoleToSystem: false,
};

interface ChatMsg {
Expand Down
4 changes: 4 additions & 0 deletions tests/responses/chat-inline-document-bytes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ const chatProvider: OcxProviderConfig = {
adapter: "openai-chat",
baseUrl: "https://gateway.example.internal/v1",
apiKey: "k",
// The wire role folds to `system` unless a destination is recorded as accepting
// `developer`; the document test asserts the role a turn keeps, so it declares the
// destination rather than asserting the default.
foldDeveloperRoleToSystem: false,
};
const anthropicProvider = {
adapter: "anthropic",
Expand Down
58 changes: 58 additions & 0 deletions tests/server/server-combo-failover-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2249,6 +2249,64 @@ describe("server combo failover 030 activation matrix", () => {
expect(hits).toBe(1);
});

test("single-target wait does not retry a request-local refusal", async () => {
let hits = 0;
const upstream = serve(() => {
hits += 1;
return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 });
});
const response = await post(comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [
{ provider: "a", model: "m1" },
], { cooldownMs: 50, waitForCooldownMs: 500 }), { user: "synthetic-client" });
expect(response.status).toBe(400);
expect(hits).toBe(1);
});

test("a concurrent cooldown does not retry a request-local refusal", async () => {
// Two requests share one target: the first stays in flight on a gate while the second
// fails hot and writes the SHARED cooldown. The first request's own failure records no
// cooldown (scope "none"), so the foreign entry alone must not arm the retry gate —
// it would wait out the sibling's cooldown and replay the refused request.
let markHeld!: () => void;
let releaseHeld!: () => void;
const heldRequest = new Promise<void>(resolve => { markHeld = resolve; });
const gate = new Promise<void>(resolve => { releaseHeld = resolve; });
let hits = 0;
const upstream = serve(async () => {
hits += 1;
if (hits === 1) {
markHeld();
await gate;
return Response.json({ error: { type: "invalid_request_error", message: "Unsupported parameter: user" } }, { status: 400 });
}
// 429 rather than 5xx so the failure reaches the combo layer directly:
// fetchWithTransientRetry would absorb a 503 before it could cool the target.
return hits === 2
? Response.json({ error: { message: "rate limited" } }, { status: 429 })
: chatSuccess("single target recovered", "m1");
});
const config = comboConfig({ a: provider("openai-responses", baseUrl(upstream), "key-a") }, [
{ provider: "a", model: "m1" },
], { cooldownMs: 500, waitForCooldownMs: 2_000 });

const refused = post(config, { user: "synthetic-client" });
await heldRequest;
const cooling = post(config);
const target = { provider: "a", model: "m1" };
const deadline = Date.now() + 5_000;
while (!isComboTargetInCooldown("free", target)) {
if (Date.now() > deadline) throw new Error("sibling request never cooled the target");
await Bun.sleep(5);
}
releaseHeld();
const [refusal, cooled] = await Promise.all([refused, cooling]);
expect(refusal.status).toBe(400);
expect(cooled.status).toBe(200);
// The cooling request hits twice (failure, then its own post-cooldown retry); the
// refused request must hit exactly once.
expect(hits).toBe(3);
});

test("a past Retry-After date remains immediate through response consumption", async () => {
const now = Date.parse("2026-07-18T00:00:00.000Z");
const failure = await consumeComboFailure(Response.json({ error: { message: "rate limited" } }, {
Expand Down
Loading