From 81e63636fcfeeb85c41ec6b5c87eb4973be425c7 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 12 Sep 2026 20:04:47 +0800 Subject: [PATCH 1/3] feat(reasoning): derive routed effort ladders from models.dev metadata Adds a provider-scoped models.dev snapshot (reasoning_options -> effort/toggle/budget_tokens), a lazy fallback in configuredReasoningEfforts() for models with no hand-written ladder, and a learned-refusal filter applied to every ladder source so a rung the upstream rejected is dropped even when the ladder is pinned in the registry. --- src/providers/reasoning-metadata.ts | 454 ++++++++++++++++++++++++++++ src/reasoning-effort.ts | 21 +- 2 files changed, 473 insertions(+), 2 deletions(-) create mode 100644 src/providers/reasoning-metadata.ts diff --git a/src/providers/reasoning-metadata.ts b/src/providers/reasoning-metadata.ts new file mode 100644 index 0000000000..5c184db02a --- /dev/null +++ b/src/providers/reasoning-metadata.ts @@ -0,0 +1,454 @@ +/** + * Data-driven reasoning ladders for routed providers (2026-09-12 local patch). + * + * Routed providers rarely publish per-model effort ladders: OpenCode Zen Go answers /models + * with ids only (id/object/created/owned_by), so opencodex had to hardcode ladders in + * registry.ts and synthesise max/ultra for codex-rs catalog membership. The public models.dev + * catalogue DOES publish them per model: + * reasoning: true + * reasoning_options: [{type:"effort",values:["low","high","max"]}, {type:"toggle"}, + * {type:"budget_tokens"}] + * This module snapshots that catalogue to disk and hands configuredReasoningEfforts() a + * fallback ladder, so the Codex catalog AND the wire clamp agree with the model instead of a + * hand-written guess. + * + * Failure policy: the network is never on the critical path. A missing, stale or corrupt + * snapshot yields undefined, which leaves every hand-written contract untouched. The second + * cache records rungs the upstream actually rejected (400/403 naming reasoning_effort), so an + * entitlement gap (muse-spark max needs an active Muse Code subscription) costs one rejected + * request instead of failing every turn that selects that rung. + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +// Leaf modules on purpose: this file is imported from reasoning-effort.ts, which combos/types.ts +// already imports. Going through the ../config barrel closes a cycle back into account-namespaces.ts +// and leaves COMBO_NAMESPACE in its temporal dead zone for entry points that start at combos/types.ts. +import { atomicWriteFile } from "../config/atomic-write"; +import { getConfigDir } from "../config/paths"; +import type { OcxProviderConfig } from "../types"; + +const FILENAME = "reasoning-metadata-cache.json"; +const SUPPORT_FILENAME = "reasoning-support-cache.json"; +const SOURCE_URL = "https://models.dev/api.json"; +const USER_AGENT = "opencodex-reasoning-metadata/1.0 (+https://github.com/lidge-jun/opencodex)"; +/** Snapshot age that triggers a background refresh. Older snapshots still serve reads. */ +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +/** A learned "this rung is refused" fact expires: entitlements change. */ +const SUPPORT_TTL_MS = 30 * 24 * 60 * 60 * 1000; +const PERSIST_DEBOUNCE_MS = 250; + +/** Canonical Codex ladder order; mirrors reasoning-effort.ts CODEX_REASONING_LEVELS. */ +const LADDER_ORDER = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]; +/** Ranked rungs used for downgrade planning; ultra is client-only and folds to max. */ +const RANKED = ["low", "medium", "high", "xhigh", "max"]; +/** Mirror of registry.ts THINKING_TOGGLE_EFFORTS / THINKING_BUDGET_EFFORTS. */ +const CLASSIFIED_STYLE_EFFORTS = ["low", "medium", "high", "xhigh", "max"]; + +/** + * models.dev provider key for a provider config. OcxProviderConfig carries no id, so the + * destination URL is the stable handle. Only destinations this patch has evidence for are + * listed; an unlisted provider simply keeps its current behaviour. + */ +const BASE_URL_TO_METADATA_PROVIDER: Record = { + "https://opencode.ai/zen/go/v1": "opencode-go", + "https://opencode.ai/zen/v1": "opencode", +}; + +/** + * Both sides of the mapping are compared after this normalisation, so a trailing slash or a + * `/v1` suffix never decides whether a destination resolves. models.dev publishes each + * provider's own `api` URL; the snapshot keeps it (v2) so the mapping can be checked against + * published data instead of trusted blindly. + */ +export function normalizeDestinationUrl(url: string | undefined): string | undefined { + if (typeof url !== "string" || url.trim() === "") return undefined; + try { + const parsed = new URL(url.trim()); + const path = parsed.pathname.replace(/\/+$/, "").replace(/\/v1$/i, ""); + return (parsed.protocol + "//" + parsed.host + path).toLowerCase(); + } catch { + return undefined; + } +} + +export type ReasoningMetadataOption = { type: string; values?: string[] }; +export type ReasoningMetadataModel = { reasoning: boolean; options: ReasoningMetadataOption[] }; + +interface MetadataSnapshot { + version: 1 | 2; + fetchedAt: number; + source: string; + providers: Record>; + /** + * v2: models.dev provider key -> that provider's published api URL (normalised). v1 snapshots + * predate the field and keep working through BASE_URL_TO_METADATA_PROVIDER. + */ + apis?: Record; +} + +interface SupportSnapshot { + version: 1; + rows: Record; +} + +let snapshotMemo: MetadataSnapshot | null | undefined; +let supportMemo: Map | undefined; +let persistTimer: ReturnType | null = null; +let refreshInFlight: Promise | null = null; + +function readJsonFile(filename: string): T | null { + try { + const path = join(getConfigDir(), filename); + if (!existsSync(path)) return null; + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch { + // A corrupt cache must never break routing, the catalog, or the dashboard. + return null; + } +} + +/** Canonical order + dedupe. Local mirror of sanitizeCodexReasoningEfforts (import cycle). */ +function sanitizeLadder(values: readonly string[] | undefined): string[] | undefined { + if (!Array.isArray(values)) return undefined; + const seen = new Set(values.filter((value): value is string => typeof value === "string")); + const ordered = LADDER_ORDER.filter(effort => seen.has(effort)); + return ordered.length > 0 ? ordered : undefined; +} + +function metadataProviderKey(provider: OcxProviderConfig): string | undefined { + const normalized = normalizeDestinationUrl(typeof provider.baseUrl === "string" ? provider.baseUrl : undefined); + if (!normalized) return undefined; + for (const [destination, key] of Object.entries(BASE_URL_TO_METADATA_PROVIDER)) { + if (normalizeDestinationUrl(destination) === normalized) return key; + } + return undefined; +} + +/** Opaque row key: providerKey|modelId|effort. None of the three may contain a pipe. */ +const KEY_SEP = "|"; + +function supportKey(providerKey: string, modelId: string, effort: string): string { + return providerKey + KEY_SEP + modelId + KEY_SEP + effort; +} + +function loadSnapshot(): MetadataSnapshot | null { + if (snapshotMemo !== undefined) return snapshotMemo; + const parsed = readJsonFile(FILENAME); + snapshotMemo = parsed && (parsed.version === 1 || parsed.version === 2) && parsed.providers && typeof parsed.providers === "object" + ? parsed + : null; + return snapshotMemo; +} + +/** + * Mapping report for diagnostics and tests: every gated destination, the models.dev provider it + * resolves to, and whether the snapshot (v2) publishes an `api` URL that confirms it. The gate is + * deliberate -- 36 of the registry's 83 destinations match a models.dev provider, so resolving by + * URL alone would silently move ladders for providers this change has no evidence for. + */ +export function reasoningMetadataMapping(): Array<{ + destination: string; + provider: string; + publishedApi?: string; + confirmed?: boolean; + models: number; +}> { + const snapshot = loadSnapshot(); + return Object.entries(BASE_URL_TO_METADATA_PROVIDER).map(([destination, provider]) => { + const normalized = normalizeDestinationUrl(destination); + const publishedApi = snapshot?.apis?.[provider]; + const row = { + destination, + provider, + ...(publishedApi ? { publishedApi } : {}), + ...(publishedApi ? { confirmed: publishedApi === normalized } : {}), + models: Object.keys(snapshot?.providers?.[provider] ?? {}).length, + }; + return row; + }); +} + +function loadSupport(): Map { + if (supportMemo) return supportMemo; + const rows = new Map(); + const parsed = readJsonFile(SUPPORT_FILENAME); + if (parsed && parsed.version === 1 && parsed.rows && typeof parsed.rows === "object") { + const now = Date.now(); + for (const [key, row] of Object.entries(parsed.rows)) { + if (!row || typeof row.at !== "number") continue; + if (now - row.at > SUPPORT_TTL_MS) continue; + rows.set(key, row.at); + } + } + supportMemo = rows; + return rows; +} + +/** Snapshot health for ocx status / diagnostics. */ +export function reasoningMetadataStatus(): { fetchedAt?: number; ageMs?: number; stale: boolean; models: number } { + const snapshot = loadSnapshot(); + if (!snapshot) return { stale: false, models: 0 }; + const ageMs = Date.now() - snapshot.fetchedAt; + let models = 0; + for (const provider of Object.values(snapshot.providers)) models += Object.keys(provider).length; + return { fetchedAt: snapshot.fetchedAt, ageMs, stale: ageMs > CACHE_TTL_MS, models }; +} + +export function reasoningMetadataModel(provider: OcxProviderConfig, modelId: string): ReasoningMetadataModel | undefined { + const key = metadataProviderKey(provider); + if (!key) return undefined; + const models = loadSnapshot()?.providers?.[key]; + if (!models) return undefined; + const model = models[modelId]; + return model && typeof model === "object" ? model : undefined; +} + +/** Raw models.dev effort values for a model, canonicalised; undefined when not published. */ +export function metadataEffortValues(provider: OcxProviderConfig, modelId: string): string[] | undefined { + const model = reasoningMetadataModel(provider, modelId); + if (!model) return undefined; + const options = Array.isArray(model.options) ? model.options : []; + const effort = options.find(option => option && option.type === "effort"); + const ladder = sanitizeLadder(effort?.values); + // none/minimal are sentinels, not picker rungs (mapReasoningEffort folds minimal to low), and + // advertising them would trip the Codex runtime clamp for no user-visible gain. + const rungs = ladder?.filter(value => value !== "none" && value !== "minimal"); + return rungs && rungs.length > 0 ? rungs : undefined; +} + +/** True when models.dev publishes the named option type (toggle / budget_tokens) for a model. */ +export function metadataDeclaresType(provider: OcxProviderConfig, modelId: string, type: string): boolean { + const model = reasoningMetadataModel(provider, modelId); + if (!model) return false; + const options = Array.isArray(model.options) ? model.options : []; + return options.some(option => option?.type === type); +} + +export function isReasoningEffortLearnedUnsupported(provider: OcxProviderConfig, modelId: string, effort: string): boolean { + const key = metadataProviderKey(provider); + if (!key) return false; + return loadSupport().has(supportKey(key, modelId, effort)); +} + +/** + * After the ladder is chosen (registry config or models.dev metadata), remove the rungs this + * account actually had refused. Applied at the configuredReasoningEfforts() exit so a + * registry-pinned ladder learns exactly like a metadata-derived one; without it a pinned rung + * the upstream rejects would replay-and-fail on every request. An all-refused ladder keeps the + * original list: turning "some rungs" into "no effort control" would silently drop the picker. + */ +export function dropLearnedUnsupportedReasoningEfforts( + provider: OcxProviderConfig, + modelId: string, + efforts: readonly string[], +): string[] { + if (efforts.length === 0) return [...efforts]; + const key = metadataProviderKey(provider); + if (!key) return [...efforts]; + const support = loadSupport(); + if (support.size === 0) return [...efforts]; + const kept = efforts.filter(effort => !support.has(supportKey(key, modelId, effort))); + return kept.length === 0 ? [...efforts] : kept; +} + +/** + * Metadata fallback ladder for a provider/model. + * + * - Published effort values win. + * - A model the provider already classifies as thinking-toggle / thinking-budget keeps the + * provider's own effort list; a toggle-only entry never invents wire semantics here. + * - Rungs the upstream actually refused are removed; a ladder emptied by that learning + * returns undefined (status quo) rather than advertising "no effort control". + */ +export function reasoningEffortsFromMetadata(provider: OcxProviderConfig, modelId: string): string[] | undefined { + const published = metadataEffortValues(provider, modelId); + let ladder = published; + if (!ladder) { + const classified = (provider.thinkingToggleModels ?? []).includes(modelId) + || (provider.thinkingBudgetModels ?? []).includes(modelId); + ladder = classified ? CLASSIFIED_STYLE_EFFORTS : undefined; + } + if (!ladder || ladder.length === 0) return undefined; + const kept = ladder.filter(effort => !isReasoningEffortLearnedUnsupported(provider, modelId, effort)); + if (kept.length === 0) return undefined; + return kept; +} + +const supportEvidence = new Map(); + +/** + * Record that the upstream refused a rung. Persisted (debounced) so the next catalog sync and + * every later request clamp before dispatch. Returns true when this is new information. + */ +export function recordUnsupportedReasoningEffort( + provider: OcxProviderConfig, + modelId: string, + effort: string, + evidence?: string, +): boolean { + const key = metadataProviderKey(provider); + if (!key || !effort) return false; + const rowKey = supportKey(key, modelId, effort); + const rows = loadSupport(); + if (rows.has(rowKey)) return false; + rows.set(rowKey, Date.now()); + if (evidence) supportEvidence.set(rowKey, evidence.slice(0, 240)); + if (persistTimer) clearTimeout(persistTimer); + persistTimer = setTimeout(() => { + persistTimer = null; + try { + const out: SupportSnapshot["rows"] = {}; + for (const [rowKey, at] of rows) { + const parts = rowKey.split(KEY_SEP); + const evidenceText = supportEvidence.get(rowKey); + out[rowKey] = { + effort: parts[2] ?? "", + at, + ...(evidenceText ? { evidence: evidenceText } : {}), + }; + } + atomicWriteFile(join(getConfigDir(), SUPPORT_FILENAME), JSON.stringify({ version: 1, rows: out }) + "\n"); + } catch { + // Best-effort persistence only. + } + }, PERSIST_DEBOUNCE_MS); + return true; +} + +/** Test seam: flush a pending support write so a script sees the snapshot immediately. */ +export function flushReasoningSupportCache(): void { + if (!persistTimer) return; + clearTimeout(persistTimer); + persistTimer = null; + try { + const rows = loadSupport(); + const out: SupportSnapshot["rows"] = {}; + for (const [rowKey, at] of rows) { + const parts = rowKey.split(KEY_SEP); + const evidenceText = supportEvidence.get(rowKey); + out[rowKey] = { effort: parts[2] ?? "", at, ...(evidenceText ? { evidence: evidenceText } : {}) }; + } + atomicWriteFile(join(getConfigDir(), SUPPORT_FILENAME), JSON.stringify({ version: 1, rows: out }) + "\n"); + } catch { + // Best-effort persistence only. + } +} + +/** Evidence test for a rejection body: does it blame reasoning effort? */ +export function isReasoningEffortRejection(text: string | undefined): boolean { + if (!text) return false; + if (text.includes("reasoning.effort") || text.includes("reasoning_effort")) return true; + return /reasoning effort|thinking budget|reasoning_parameters|unsupported.{0,24}effort/i.test(text); +} + +/** + * Plan a single-rung downgrade for a rejected request: records the refusal (so later turns + * clamp before dispatch) and returns the next lower rung the model does publish. + */ +export function planReasoningEffortDowngrade(args: { + provider: OcxProviderConfig; + modelId: string; + requested?: string; + rejectionText?: string; +}): { effort: string; recorded: boolean } | undefined { + const requested = args.requested === "ultra" ? "max" : args.requested; + if (!requested || !RANKED.includes(requested)) return undefined; + const recorded = recordUnsupportedReasoningEffort(args.provider, args.modelId, requested, args.rejectionText); + const effective = metadataEffortValues(args.provider, args.modelId) + ?? sanitizeLadder(args.provider.modelReasoningEfforts?.[args.modelId]) + ?? sanitizeLadder(args.provider.reasoningEfforts); + const ladder = (effective ?? []).filter(effort => RANKED.includes(effort)); + if (ladder.length === 0) return undefined; + const candidates = ladder + .filter(effort => RANKED.indexOf(effort) < RANKED.indexOf(requested)) + .filter(effort => !isReasoningEffortLearnedUnsupported(args.provider, args.modelId, effort)); + if (candidates.length === 0) return undefined; + return { effort: candidates[candidates.length - 1], recorded }; +} + +/** + * Refresh the models.dev snapshot. Best-effort and idempotent: never throws, never blocks a + * request, keeps the previous snapshot on failure. Ladders are stored for the gated destinations + * only (OpenCode Zen + Zen Go: about 130 models), while every published provider `api` URL is + * kept so the gate can be checked against real data and widened without another format change. + * Non-reasoning models carry no ladder and are dropped. + */ +export async function refreshReasoningMetadata(options: { force?: boolean } = {}): Promise<{ + ok: boolean; + reason: string; + providers?: number; + models?: number; +}> { + const snapshot = loadSnapshot(); + if (!options.force && snapshot && Date.now() - snapshot.fetchedAt <= CACHE_TTL_MS) { + return { ok: true, reason: "fresh" }; + } + if (refreshInFlight) { + await refreshInFlight; + return { ok: true, reason: "coalesced" }; + } + const job = (async () => { + const response = await fetch(SOURCE_URL, { + headers: { "user-agent": USER_AGENT, accept: "application/json" }, + // A hanging connection must not pin refreshInFlight for the life of the process. + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error("models.dev HTTP " + response.status); + const raw = await response.json() as Record }>; + const providers: MetadataSnapshot["providers"] = {}; + const apis: Record = {}; + const gated = new Set(Object.values(BASE_URL_TO_METADATA_PROVIDER)); + let models = 0; + for (const [providerKey, entry] of Object.entries(raw ?? {})) { + const api = normalizeDestinationUrl(typeof entry?.api === "string" ? entry.api : undefined); + if (api) apis[providerKey] = api; + if (!gated.has(providerKey)) continue; + const out: Record = {}; + for (const [modelId, value] of Object.entries(entry?.models ?? {})) { + const model = value as { reasoning?: unknown; reasoning_options?: unknown }; + if (model?.reasoning !== true) continue; + const options: ReasoningMetadataOption[] = []; + if (Array.isArray(model?.reasoning_options)) { + for (const option of model.reasoning_options) { + if (!option || typeof option !== "object") continue; + const type = (option as { type?: unknown }).type; + if (typeof type !== "string") continue; + const values = (option as { values?: unknown }).values; + options.push({ + type, + ...(Array.isArray(values) + ? { values: values.filter((v): v is string => typeof v === "string").slice(0, 12) } + : {}), + }); + } + } + out[modelId] = { reasoning: model?.reasoning === true, options }; + models += 1; + } + if (Object.keys(out).length === 0) continue; + providers[providerKey] = out; + } + const next: MetadataSnapshot = { version: 2, fetchedAt: Date.now(), source: SOURCE_URL, providers, apis }; + atomicWriteFile(join(getConfigDir(), FILENAME), JSON.stringify(next) + "\n"); + snapshotMemo = next; + return { ok: true, reason: "refreshed", providers: Object.keys(providers).length, models }; + })(); + refreshInFlight = job.catch(() => undefined).finally(() => { refreshInFlight = null; }); + try { + return await job; + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } +} + +/** + * Kick a background refresh when the snapshot is missing or stale. Called from the ladder read + * path so both the long-lived proxy and short-lived ocx sync self-heal without a new CLI + * surface. One refresh per process at a time; failures are ignored on purpose. + */ +export function ensureReasoningMetadataSnapshot(): void { + const snapshot = loadSnapshot(); + if (snapshot && Date.now() - snapshot.fetchedAt <= CACHE_TTL_MS) return; + if (refreshInFlight) return; + void refreshReasoningMetadata().catch(() => undefined); +} diff --git a/src/reasoning-effort.ts b/src/reasoning-effort.ts index 6342159d31..0c09748107 100644 --- a/src/reasoning-effort.ts +++ b/src/reasoning-effort.ts @@ -1,5 +1,6 @@ import type { OcxProviderConfig } from "./types"; import { modelInList } from "./types"; +import { dropLearnedUnsupportedReasoningEfforts, ensureReasoningMetadataSnapshot, reasoningEffortsFromMetadata } from "./providers/reasoning-metadata"; // Descriptions mirror the upstream bundled models.json canonical wording (openai/codex PR #31684). export const CODEX_REASONING_LEVELS: { effort: string; description: string }[] = [ @@ -148,8 +149,24 @@ export function sanitizeCodexReasoningEfforts(efforts: readonly string[] | undef export function configuredReasoningEfforts(provider: OcxProviderConfig, modelId: string): string[] | undefined { if (modelInList(provider.noReasoningModels, modelId)) return []; const modelEfforts = modelRecordValue(provider.modelReasoningEfforts, modelId); - if (modelEfforts !== undefined) return healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(modelEfforts) ?? []); - if (provider.reasoningEfforts !== undefined) return healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(provider.reasoningEfforts) ?? []); + // Rungs this account actually had refused are removed for every ladder source (registry + // config or models.dev), so a learned refusal is honoured even when the ladder is pinned in + // code; otherwise a rejected pinned rung would replay-and-fail on every request. + if (modelEfforts !== undefined) { + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(modelEfforts) ?? [])); + } + if (provider.reasoningEfforts !== undefined) { + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, sanitizeCodexReasoningEfforts(provider.reasoningEfforts) ?? [])); + } + // models.dev publishes the per-model ladder that routed providers never expose on /models. + // (OpenCode Zen Go answers ids only). Only consulted when nothing was configured for this + // model, so every hand-written contract stays authoritative. The snapshot refreshes itself in + // the background; no snapshot means the previous behaviour. + const fromMetadata = reasoningEffortsFromMetadata(provider, modelId); + if (fromMetadata !== undefined) { + ensureReasoningMetadataSnapshot(); + return dropLearnedUnsupportedReasoningEfforts(provider, modelId, healMappedTiers(provider, modelId, fromMetadata)); + } return undefined; } From 9b1e2e192d21d682a477ca379c601f2ed024f156 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 12 Sep 2026 20:31:43 +0800 Subject: [PATCH 2/3] test(reasoning): cover metadata ladders and learned refusals Adds tests/codex-integration/reasoning-metadata.test.ts (10 cases: published rungs, sentinel stripping, wire clamp, hand-written precedence, unknown destination, toggle-only entries, corrupt snapshot, learned-refusal filtering for both ladder sources, one-shot downgrade planning, and rejection classification) plus the cache reset seam and the devlog record. --- .../260912_reasoning_metadata/000_decision.md | 73 ++++++ scripts/test-layout/layout.json | 1 + src/providers/reasoning-metadata.ts | 13 +- .../reasoning-metadata.test.ts | 210 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + 5 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 devlog/_plan/260912_reasoning_metadata/000_decision.md create mode 100644 tests/codex-integration/reasoning-metadata.test.ts diff --git a/devlog/_plan/260912_reasoning_metadata/000_decision.md b/devlog/_plan/260912_reasoning_metadata/000_decision.md new file mode 100644 index 0000000000..dc0618be01 --- /dev/null +++ b/devlog/_plan/260912_reasoning_metadata/000_decision.md @@ -0,0 +1,73 @@ +# 260912 — Routed reasoning ladders come from models.dev + +## Decision + +For a routed provider whose destination models.dev publishes, the Codex catalog and the outbound +wire value prefer the published reasoning ladder over a hand-written one, and a rung the upstream +actually refused is dropped from every later ladder (registry config included). + +Layers: + +1. src/providers/reasoning-metadata.ts snapshots models.dev (reasoning + reasoning_options, the + effort / toggle / budget_tokens option types) into ~/.opencodex/reasoning-metadata-cache.json + (24h TTL, stale-but-readable offline, atomic write). The v2 snapshot stores ladders for the + gated destinations (OpenCode Zen + Zen Go, 133 models / ~20 KB) and the published `api` URL of + every provider models.dev lists, so the gate can be checked against real data. +2. configuredReasoningEfforts() consults that snapshot only when nothing was configured for the + model, so every hand-written contract stays authoritative; mapReasoningEffort() clamps through + the same function, which is what keeps the catalog and the wire in agreement. +3. reasoning-support-cache.json records (provider, model, effort) refusals; the filter at the + configuredReasoningEfforts() exit removes those rungs whether the ladder came from the snapshot + or from the registry. + +## Why the hand-written table was not enough + +OpenCode Zen Go answers GET https://opencode.ai/zen/go/v1/models with ids only (id, object, created, +owned_by — 37 models, verified 2026-09-12), so opencodex had to guess: + +- muse-spark-1.3-contributor was advertised up to ultra while the gateway refuses max with + 400 {"param":"reasoning.effort","type":"invalid_request_error","message":"Error from provider + (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an + active Muse Code subscription for model muse-spark-1.3-contributor."} ; xhigh answers 200. + models.dev publishes [minimal, low, medium, high, xhigh] for that model — the refusals were the + synthetic tiers, not the model. +- deepseek-v4.1-flash needs [low, high, max] before it advertises any control at all; models.dev + publishes exactly that. + +Verified after the change: the catalog lists [low, medium, high, xhigh] for muse-spark and +[low, high, max] for deepseek-v4.1-flash, max on muse-spark is sent as xhigh, and a refusal replays +once at the next lower published rung (usage.jsonl recovery kind reasoning-effort-downgrade) +instead of failing the turn. + +## Source resolution (2026-09-12 review follow-up) + +models.dev publishes each provider's own `api` URL (`opencode-go` -> `https://opencode.ai/zen/go/v1`, +`opencode` -> `https://opencode.ai/zen/v1`), so the destination is resolvable from data rather than from a +guess. Resolution stays gated: BASE_URL_TO_METADATA_PROVIDER is the authoritative list (both URLs are +compared normalised, so a trailing slash or a `/v1` suffix never decides), and reasoningMetadataMapping() +reports for each gated destination whether the snapshot confirms it against the published URL. + +Measured the same day: **36 of the registry's 83 destinations** match a models.dev provider, and 13 of a live +27-provider config do; 11 of those 13 already carry hand-written ladders (the metadata fallback is never +consulted) and the other 2 (`openrouter`, 4 models) would change catalog ladders. Resolving by URL alone +would therefore move ladders for providers this change has no evidence for, so widening the gate is a +separate decision with those numbers in hand -- the snapshot already carries the data it needs. + +## Learned refusals are credential-scoped in practice + +A refusal is recorded per `(provider, model, effort)`. Every destination that can reach this path is +`authKind: key`, i.e. one credential per provider entry, so that key already has the credential dimension; +the catalog is account-independent by construction (built once per process, not per request). Three +properties bound the rest: only the refused rung is dropped, the fact expires after 30 days, and the clamp +is visible as requestedEffort versus effectiveEffort in usage.jsonl. A credential-scoped key becomes +necessary only if opencodex ever pools several credentials behind one metadata-mapped provider entry. + +## Known follow-ups + +- Destination to models.dev provider id stays a gated table (two OpenCode destinations today). + Widening it to every URL match is measured above and is a maintainer call, not a mechanical edit. A + shared registry-side helper would replace the table itself, but importing providers/registry from this + module widened an unrelated supported_reasoning_levels literal type during development, so the naive + import was reverted. +- The snapshot refresh is triggered on first read with TTL and in-flight guards rather than from the + startup path, so a long-lived proxy refreshes at most daily. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 18fc2060c0..afe0b4af8b 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1054,6 +1054,7 @@ "raycast-detect.test.ts": "clients", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", + "reasoning-metadata.test.ts": "codex-integration", "reasoning-replay-identity.test.ts": "adapters", "reasoning-replay-robustness.test.ts": "adapters", "reasoning-replay-scope-source.test.ts": "lib", diff --git a/src/providers/reasoning-metadata.ts b/src/providers/reasoning-metadata.ts index 5c184db02a..14eded4aa0 100644 --- a/src/providers/reasoning-metadata.ts +++ b/src/providers/reasoning-metadata.ts @@ -1,5 +1,5 @@ /** - * Data-driven reasoning ladders for routed providers (2026-09-12 local patch). + * Data-driven reasoning ladders for routed providers. * * Routed providers rarely publish per-model effort ladders: OpenCode Zen Go answers /models * with ids only (id/object/created/owned_by), so opencodex had to hardcode ladders in @@ -96,6 +96,17 @@ let supportMemo: Map | undefined; let persistTimer: ReturnType | null = null; let refreshInFlight: Promise | null = null; +/** Test seam: drop the memoised snapshot/support caches so a suite can drive the load paths. */ +export function resetReasoningMetadataCachesForTests(): void { + snapshotMemo = undefined; + supportMemo = undefined; + if (persistTimer) { + clearTimeout(persistTimer); + persistTimer = null; + } + refreshInFlight = null; +} + function readJsonFile(filename: string): T | null { try { const path = join(getConfigDir(), filename); diff --git a/tests/codex-integration/reasoning-metadata.test.ts b/tests/codex-integration/reasoning-metadata.test.ts new file mode 100644 index 0000000000..6146212860 --- /dev/null +++ b/tests/codex-integration/reasoning-metadata.test.ts @@ -0,0 +1,210 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OcxProviderConfig } from "../../src/types"; + +/** + * Data-driven reasoning ladders (models.dev snapshot + learned refusals). + * + * OpenCode Zen Go publishes model ids only, so the catalog used to advertise whatever the + * registry hardcoded -- including rungs the upstream refuses (muse-spark max -> 400 "requires an + * active Muse Code subscription"). These cases pin the metadata fallback, the wire clamp and the + * learned-refusal filter that keeps a rejected rung out of every later request. + */ + +const ZEN_GO: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", +} as OcxProviderConfig; + +const MUSE_SPARK = "muse-spark-1.3-contributor"; +const DEEPSEEK_FLASH = "deepseek-v4.1-flash"; + +const REJECTION_BODY = JSON.stringify({ + model: MUSE_SPARK, + error: { + param: "reasoning.effort", + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.", + }, +}); + +const roots: string[] = []; + +function snapshotFile(providers: Record): Record { + return { version: 1, fetchedAt: Date.now(), source: "test", providers }; +} + +function sandbox(files: Record = {}): string { + const dir = mkdtempSync(join(tmpdir(), "ocx-reasoning-metadata-")); + roots.push(dir); + for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body); + process.env["OPENCODEX_HOME"] = dir; + return dir; +} + +async function load(files: Record = {}) { + sandbox(files); + const metadata = await import("../../src/providers/reasoning-metadata"); + const effort = await import("../../src/reasoning-effort"); + metadata.resetReasoningMetadataCachesForTests(); + return { metadata, effort }; +} + +function metadataFile(providers: Record): Record { + return { "reasoning-metadata-cache.json": JSON.stringify(snapshotFile(providers)) }; +} + +function metadataFileV2(providers: Record, apis: Record): Record { + return { + "reasoning-metadata-cache.json": JSON.stringify({ ...snapshotFile(providers), version: 2, apis }), + }; +} + +function supportFile(rows: Record): Record { + return { "reasoning-support-cache.json": JSON.stringify({ version: 1, rows }) }; +} + +afterEach(() => { + for (const dir of roots.splice(0)) rmSync(dir, { recursive: true, force: true }); + delete process.env["OPENCODEX_HOME"]; +}); + +describe("models.dev reasoning metadata", () => { + test("advertises the published effort rungs and strips the none/minimal sentinels", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { + [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["minimal", "low", "medium", "high", "xhigh"] }] }, + }, + })); + expect(effort.configuredReasoningEfforts(ZEN_GO, MUSE_SPARK)).toEqual(["low", "medium", "high", "xhigh"]); + }); + + test("clamps a rung the model does not publish instead of failing upstream", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { + [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["minimal", "low", "medium", "high", "xhigh"] }] }, + [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] }, + }, + })); + expect(effort.mapReasoningEffort(ZEN_GO, MUSE_SPARK, "max")).toBe("xhigh"); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "max")).toBe("max"); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "ultra")).toBe("max"); + }); + + test("a hand-written ladder stays authoritative and unknown models stay untouched", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "medium", "high", "xhigh"] }] } }, + })); + const pinned = { ...ZEN_GO, modelReasoningEfforts: { [MUSE_SPARK]: ["low", "high"] } } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(pinned, MUSE_SPARK)).toEqual(["low", "high"]); + expect(effort.configuredReasoningEfforts(ZEN_GO, "not-a-model")).toBeUndefined(); + }); + + test("a destination the snapshot does not describe keeps the previous behaviour", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "high"] }] } }, + })); + const elsewhere = { ...ZEN_GO, baseUrl: "https://api.deepseek.com/v1" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(elsewhere, MUSE_SPARK)).toBeUndefined(); + }); + + test("a toggle-only entry never invents wire semantics for an unclassified model", async () => { + const { effort } = await load(metadataFile({ + "opencode-go": { "minimax-m3": { reasoning: true, options: [{ type: "toggle" }] } }, + })); + expect(effort.configuredReasoningEfforts(ZEN_GO, "minimax-m3")).toBeUndefined(); + }); + + test("a corrupt or missing snapshot falls back to the status quo", async () => { + const { effort } = await load({ "reasoning-metadata-cache.json": "{not json" }); + expect(effort.configuredReasoningEfforts(ZEN_GO, MUSE_SPARK)).toBeUndefined(); + expect(effort.mapReasoningEffort(ZEN_GO, MUSE_SPARK, "max")).toBe("max"); + }); +}); + +describe("learned rung refusals", () => { + const refused = () => ({ + ["opencode-go|" + DEEPSEEK_FLASH + "|max"]: { effort: "max", at: Date.now() }, + }); + + test("drops a refused rung from a metadata-derived ladder", async () => { + const { effort } = await load({ + ...metadataFile({ "opencode-go": { [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] } } }), + ...supportFile(refused()), + }); + expect(effort.configuredReasoningEfforts(ZEN_GO, DEEPSEEK_FLASH)).toEqual(["low", "high"]); + expect(effort.mapReasoningEffort(ZEN_GO, DEEPSEEK_FLASH, "max")).toBe("high"); + }); + + test("drops a refused rung from a ladder pinned in the registry too", async () => { + const { effort } = await load(supportFile(refused())); + const pinned = { ...ZEN_GO, modelReasoningEfforts: { [DEEPSEEK_FLASH]: ["low", "high", "max"] } } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(pinned, DEEPSEEK_FLASH)).toEqual(["low", "high"]); + }); + + test("records the refusal and plans the next lower published rung once", async () => { + const { metadata } = await load({ + ...metadataFile({ "opencode-go": { [DEEPSEEK_FLASH]: { reasoning: true, options: [{ type: "effort", values: ["low", "high", "max"] }] } } }), + }); + const first = metadata.planReasoningEffortDowngrade({ + provider: ZEN_GO, modelId: DEEPSEEK_FLASH, requested: "max", rejectionText: REJECTION_BODY, + }); + expect(first).toEqual({ effort: "high", recorded: true }); + metadata.flushReasoningSupportCache(); + const second = metadata.planReasoningEffortDowngrade({ + provider: ZEN_GO, modelId: DEEPSEEK_FLASH, requested: "max", rejectionText: REJECTION_BODY, + }); + expect(second).toEqual({ effort: "high", recorded: false }); + }); + + test("classifies only reasoning-effort refusals", async () => { + const { metadata } = await load(); + expect(metadata.isReasoningEffortRejection(REJECTION_BODY)).toBe(true); + expect(metadata.isReasoningEffortRejection(JSON.stringify({ error: { message: "Invalid upload request." } }))).toBe(false); + expect(metadata.isReasoningEffortRejection(undefined)).toBe(false); + }); +}); + +describe("destination resolution", () => { + const ZEN = { ...ZEN_GO, baseUrl: "https://opencode.ai/zen/v1" } as OcxProviderConfig; + const MODEL = { [MUSE_SPARK]: { reasoning: true, options: [{ type: "effort", values: ["low", "high"] }] } }; + + test("resolves the OpenCode family and tolerates a trailing slash", async () => { + const { effort } = await load(metadataFile({ "opencode": MODEL, "opencode-go": MODEL })); + expect(effort.configuredReasoningEfforts(ZEN, MUSE_SPARK)).toEqual(["low", "high"]); + const trailingSlash = { ...ZEN_GO, baseUrl: "https://opencode.ai/zen/go/v1/" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(trailingSlash, MUSE_SPARK)).toEqual(["low", "high"]); + }); + + // 36 of the registry's 83 destinations match a models.dev provider, so resolving by URL alone + // would move ladders for providers this change has no evidence for. The gate stays explicit. + test("a destination that only matches by URL stays gated out", async () => { + const { effort } = await load(metadataFileV2( + { "some-upstream": MODEL }, + { "some-upstream": "https://api.some-upstream.example/v1" }, + )); + const provider = { ...ZEN_GO, baseUrl: "https://api.some-upstream.example/v1" } as OcxProviderConfig; + expect(effort.configuredReasoningEfforts(provider, MUSE_SPARK)).toBeUndefined(); + }); + + test("the v2 snapshot confirms the gate against each provider's published api url", async () => { + const { metadata } = await load(metadataFileV2( + { "opencode-go": MODEL }, + { "opencode-go": "https://opencode.ai/zen/go" }, + )); + expect(metadata.reasoningMetadataMapping()).toEqual([ + { destination: "https://opencode.ai/zen/go/v1", provider: "opencode-go", publishedApi: "https://opencode.ai/zen/go", confirmed: true, models: 1 }, + { destination: "https://opencode.ai/zen/v1", provider: "opencode", models: 0 }, + ]); + }); + + test("a v1 snapshot without published api urls still resolves through the table", async () => { + const { metadata } = await load(metadataFile({ "opencode-go": MODEL })); + const [zenGo] = metadata.reasoningMetadataMapping(); + expect(zenGo.publishedApi).toBeUndefined(); + expect(zenGo.confirmed).toBeUndefined(); + expect(zenGo.models).toBe(1); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 5e57040565..f8dd090adf 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -886,6 +886,7 @@ "raycast-detect.test.ts": "clients", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", + "reasoning-metadata.test.ts": "codex-integration", "reasoning-replay-identity.test.ts": "adapters", "reasoning-replay-robustness.test.ts": "adapters", "reasoning-replay-scope-source.test.ts": "lib", From 7de03fb066895cbeec4febba4f62db6b2436b7e4 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 12 Sep 2026 20:59:55 +0800 Subject: [PATCH 3/3] feat(reasoning): learn a refused effort rung and replay once A routed upstream can refuse a rung the catalog advertises because the ladder describes the model, not the account (max on muse-spark-1.3-contributor needs an active Muse Code subscription). The request path now detects that refusal in the 400/403 error body, records it, and replays once at the next lower published rung in both recovery loops; later turns clamp before dispatch. The attempt is logged with recovery kind reasoning-effort-downgrade and the same-target cache is invalidated so the replay carries the downgraded effort. --- .../000_decision.md | 41 +++++ src/server/responses/core.ts | 84 ++++++++++ src/usage/log.ts | 4 +- ...sponses-reasoning-effort-downgrade.test.ts | 158 ++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md create mode 100644 tests/responses/responses-reasoning-effort-downgrade.test.ts diff --git a/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md b/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md new file mode 100644 index 0000000000..67499186af --- /dev/null +++ b/devlog/_plan/260912_reasoning_effort_downgrade/000_decision.md @@ -0,0 +1,41 @@ +# 260912 — A refused reasoning rung is learned and replayed once + +## Decision + +When a routed upstream answers 400/403 and names reasoning effort in the body, the pipeline records +that (provider, model, effort) as refused, replays the request once at the next lower published rung, +and keeps the rung out of every later ladder (see the metadata record in +devlog/_plan/260912_reasoning_metadata/). The attempt is logged with recovery kind +reasoning-effort-downgrade, so requestedEffort and effectiveEffort stay distinguishable in usage. + +## Why the ladder is not enough + +The published ladder describes the model, not the account. Live 2026-09-12: +muse-spark-1.3-contributor answered 400 for max with + + Error from provider (Console Go): Upstream request failed: [invalid_request_error] + reasoning_effort max requires an active Muse Code subscription for model + muse-spark-1.3-contributor. + +while xhigh answered 200. Clamping against the published ladder removes that case before dispatch, +but any entitlement-driven refusal for a published rung would otherwise fail the turn outright. + +## Shape + +- Detection is narrow on purpose: 400/403 only, the body must be complete and display-safe (the same + contract as the other rejection peeks), and the text has to name reasoning effort. An unrelated 400 + never triggers a replay, which keeps the single extra send honest. +- One replay per request, guarded per recovery loop. The streamed passthroughRecovery loop and the + non-streamed recovery loop both carry the same block, matching the file's existing convention that + recovery kinds stay in sync across the two. +- Before the rebuild the parsed effort is replaced and the same-target cache is invalidated + (invalidateSameTargetRequest), because that cache keys on parsed identity and would otherwise + replay the original body byte-for-byte. +- No new failure surface: when the refusal is the only rung (or every lower rung is known-refused), + the original error is returned untouched. + +## Evidence + +tests/responses/responses-reasoning-effort-downgrade.test.ts (4 cases, mocked upstream): +pre-dispatch clamp, learn-then-replay on the non-streamed path, learn-then-replay on the streamed +path, and no replay for an unrelated 400. tests/responses runs 2040 pass / 0 fail with the change. diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 8b499d5111..dcc50a4094 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -291,6 +291,7 @@ import { } from "../lifecycle"; import { redactSecretString, sanitizeLogMetadataString } from "../../lib/redact"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +import { isReasoningEffortRejection, planReasoningEffortDowngrade } from "../../providers/reasoning-metadata"; import { ENCRYPTED_FUNCTION_OUTPUT_REJECTION, isRateLimitOrQuotaFailureMessage, @@ -830,6 +831,28 @@ export function shouldAttemptOpaqueBlobRecovery(args: { && isSelfIdentifiedOpaqueBlobRejection(args.errorBody); } +/** + * Peek the upstream error body for the reasoning-effort downgrade. Only 400/403 are considered + * and the body must be complete and display-safe, the same contract the other rejection peeks + * use. The match is deliberately narrow: the upstream has to name reasoning effort, so an + * unrelated 400 never triggers a replay. + */ +async function reasoningEffortRejectionText( + response: Response, + alreadyAttempted: boolean, + signal: AbortSignal, +): Promise { + if (alreadyAttempted) return undefined; + if (response.status !== 400 && response.status !== 403) return undefined; + try { + const body = await readBoundedResponseBody(response.clone(), { signal }); + if (!body.displaySafe || body.truncated) return undefined; + return isReasoningEffortRejection(body.text) ? body.text : undefined; + } catch { + return undefined; + } +} + async function opaqueBlobRejectionBodyForRecovery( response: Response, outboundBody: string | undefined, @@ -5334,6 +5357,8 @@ async function handleResponsesInner( } const opaqueBlobRecoveryGuard: OpaqueBlobRecoveryGuard = { attempted: false }; + // At most one reasoning-effort downgrade per request. + const reasoningEffortDowngradeGuard: { attempted: boolean } = { attempted: false }; let oauth401ReplayAttempted = false; let codex401ReplayKind: "main" | "stored" | null = null; const rateLimitPolicy = rateLimitRetryPolicyFor(route.provider); @@ -5887,6 +5912,35 @@ async function handleResponsesInner( logCtx.terminalIncompleteReason = preflightLog.terminalIncompleteReason; } } + // Reasoning-effort downgrade: a rung the catalog still advertises can be refused upstream -- + // the metadata records the model's ladder, not this account's entitlement (a Muse Code + // subscription gates max on muse-spark-1.3-contributor, for example). Learn the refusal so + // later turns clamp before dispatch, then replay once at the next lower published rung + // instead of failing the turn; requestedEffort/effectiveEffort keep both values in usage. + if (!reasoningEffortDowngradeGuard.attempted) { + const rejectionText = await reasoningEffortRejectionText( + upstreamResponse, + reasoningEffortDowngradeGuard.attempted, + upstream.signal, + ); + const downgrade = rejectionText === undefined + ? undefined + : planReasoningEffortDowngrade({ + provider: route.provider, + modelId: parsed.modelId, + requested: parsed.options.reasoning, + rejectionText, + }); + if (downgrade) { + reasoningEffortDowngradeGuard.attempted = true; + parsed.options.reasoning = downgrade.effort; + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + const result = await rebuildAndRefetch("reasoning-effort-downgrade"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue passthroughRecovery; + } + } break; } const headers = sanitizePassthroughHeaders(upstreamResponse.headers); @@ -7482,6 +7536,8 @@ async function handleResponsesInner( }; // Keep recovery kinds in sync with the native Responses `passthroughRecovery:` loop above. recovery: for (;;) { + // At most one reasoning-effort downgrade per request. + const reasoningEffortDowngradeGuard: { attempted: boolean } = { attempted: false }; if ( upstreamResponse.status === 401 && isOAuth401ReplayProvider @@ -7760,6 +7816,34 @@ async function handleResponsesInner( upstreamResponse = result; continue recovery; } + // Reasoning-effort downgrade, mirroring the passthroughRecovery loop above: learn the + // refused rung, then replay once at the next published one. + if (!reasoningEffortDowngradeGuard.attempted) { + const rejectionText = await reasoningEffortRejectionText( + upstreamResponse, + reasoningEffortDowngradeGuard.attempted, + upstream.signal, + ); + const downgrade = rejectionText === undefined + ? undefined + : planReasoningEffortDowngrade({ + provider: route.provider, + modelId: parsed.modelId, + requested: parsed.options.reasoning, + rejectionText, + }); + if (downgrade) { + reasoningEffortDowngradeGuard.attempted = true; + parsed.options.reasoning = downgrade.effort; + // The same-target cache keys on parsed identity, so a mutated effort needs a token bump. + invalidateSameTargetRequest(); + try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } + const result = await rebuildAndRefetch("reasoning-effort-downgrade"); + if ("failed" in result) return result.failed; + upstreamResponse = result; + continue recovery; + } + } break; } if (!upstreamResponse.ok) { diff --git a/src/usage/log.ts b/src/usage/log.ts index 2944c22f9a..2281f05461 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -71,7 +71,8 @@ export type AttemptRecoveryKind = | "oauth-account-429" | "image-413" | "opaque-blob-rejection" - | "empty-completion"; + | "empty-completion" + | "reasoning-effort-downgrade"; /** Request-time upstream credential class, never a credential or account identifier. */ export type UsageCredentialSource = "grok-oauth" | "xai-api-key"; @@ -311,6 +312,7 @@ const ATTEMPT_RECOVERY_KINDS = new Set([ "image-413", "opaque-blob-rejection", "empty-completion", + "reasoning-effort-downgrade", ]); const USAGE_STATUSES = new Set([ "reported", diff --git a/tests/responses/responses-reasoning-effort-downgrade.test.ts b/tests/responses/responses-reasoning-effort-downgrade.test.ts new file mode 100644 index 0000000000..616d25413c --- /dev/null +++ b/tests/responses/responses-reasoning-effort-downgrade.test.ts @@ -0,0 +1,158 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleResponses } from "../../src/server/responses/core"; +import { resetReasoningMetadataCachesForTests } from "../../src/providers/reasoning-metadata"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; + +/** + * Rejected-rung learning on the request path: a rung the catalog advertises can still be refused + * upstream because the ladder describes the model, not this account's entitlement (max on + * muse-spark-1.3-contributor needs an active Muse Code subscription). The pipeline must learn the + * refusal, replay once at the next published rung, and never replay an unrelated 400. + */ + +const originalFetch = globalThis.fetch; +const originalOpenCodexHome = process.env.OPENCODEX_HOME; +const MODEL = "muse-spark-1.3-contributor"; +const REFUSAL = JSON.stringify({ + error: { + param: "reasoning.effort", + type: "invalid_request_error", + message: "Error from provider (Console Go): Upstream request failed: [invalid_request_error] reasoning_effort max requires an active Muse Code subscription for model muse-spark-1.3-contributor.", + }, +}); +const UNRELATED = JSON.stringify({ error: { type: "invalid_request_error", message: "Invalid upload request." } }); + +let testDir = ""; + +function writeSnapshot(values: string[]): void { + writeFileSync(join(testDir, "reasoning-metadata-cache.json"), JSON.stringify({ + version: 1, + fetchedAt: Date.now(), + source: "test", + providers: { "opencode-go": { [MODEL]: { reasoning: true, options: [{ type: "effort", values }] } } }, + })); +} + +function config(): OcxConfig { + return { + defaultProvider: "first", + providers: { + first: { + adapter: "openai-chat", + baseUrl: "https://opencode.ai/zen/go/v1", + authMode: "key", + apiKey: "test-key", + }, + }, + } as OcxConfig; +} + +function request(stream = false): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "first/" + MODEL, + stream, + store: false, + reasoning: { effort: "max" }, + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "go" }] }], + }), + }); +} + +function success(): Response { + return Response.json({ id: "resp-ok", object: "response", status: "completed", model: MODEL, output: [] }); +} + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-reasoning-downgrade-")); + process.env.OPENCODEX_HOME = testDir; + resetReasoningMetadataCachesForTests(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + resetReasoningMetadataCachesForTests(); + if (originalOpenCodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalOpenCodexHome; + rmSync(testDir, { recursive: true, force: true }); +}); + +describe("rejected reasoning rungs", () => { + test("clamps a rung the model does not publish before dispatch", async () => { + writeSnapshot(["minimal", "low", "medium", "high", "xhigh"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(response.status).toBe(200); + expect(outbound).toHaveLength(1); + expect(outbound[0]?.reasoning_effort).toBe("xhigh"); + }); + + test("learns the refusal and replays once at the next published rung", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length === 1 + ? new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }) + : success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(outbound[0]?.reasoning_effort).toBe("max"); + expect(outbound[1]?.reasoning_effort).toBe("high"); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["reasoning-effort-downgrade"]); + }); + + test("does not replay an unrelated 400", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return new Response(UNRELATED, { status: 400, headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(1); + expect(response.ok).toBe(false); + expect(logCtx.activeAttempt?.recoveryKinds ?? []).toEqual([]); + }); + test("replays once on the streamed passthrough path too", async () => { + writeSnapshot(["low", "high", "max"]); + const outbound: Array> = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + outbound.push(JSON.parse(String(init?.body)) as Record); + return outbound.length === 1 + ? new Response(REFUSAL, { status: 400, headers: { "content-type": "application/json" } }) + : success(); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + + const response = await handleResponses(request(true), config(), logCtx); + await response.text(); + + expect(outbound).toHaveLength(2); + expect(outbound[1]?.reasoning_effort).toBe("high"); + expect(logCtx.activeAttempt?.recoveryKinds).toEqual(["reasoning-effort-downgrade"]); + }); +});