diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index 3afb7af916f..a2152185e52 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -586,6 +586,14 @@ Codex. Native custom calls and converted function calls use the same completion patch previews are held while their executable form is unresolved. JavaScript that merely contains patch text and unrelated native custom payloads stay unchanged. +A routed model can also mistakenly send a shell-argument object such as +`{"cmd":"git status --short"}` to code-mode `exec`. For a verified code-mode catalog, +opencodex converts an unambiguous shell object into `tools.exec_command(...)` JavaScript +and forwards its output through `text(...)`. Shell options are preserved, and Codex still +executes and authorizes the command. Valid JavaScript fallback fields, ambiguous objects, +and unrelated tool namespaces are not converted. This compatibility repair does not bypass +provider rate limits or change the configured retry policy. + Routed code-mode turns are also told the host's rules for the nested helpers before the first call: `tools.apply_patch` takes one string that opens and closes with the bare patch marker lines, the isolate has no `import`, and long-running commands are polled through `write_stdin`. When a diff --git a/docs-site/src/content/docs/zh-cn/guides/codex-integration.md b/docs-site/src/content/docs/zh-cn/guides/codex-integration.md index 982a734304b..f896c9928d6 100644 --- a/docs-site/src/content/docs/zh-cn/guides/codex-integration.md +++ b/docs-site/src/content/docs/zh-cn/guides/codex-integration.md @@ -202,6 +202,12 @@ Codex 显示的模型来自一个磁盘上的 catalog（默认是 `$CODEX_HOME/o 结果仍包含宿主的某条失败消息，opencodex 会追加一行提示，指出对应规则。此变更不会重写模型的 代码或补丁文本。 +如果路由模型误把 `{"cmd":"git status --short"}` 这样的 shell 参数对象传给 code-mode `exec`， +opencodex 会在确认工具目录为 code mode 且内容无歧义时，将它转换为调用 +`tools.exec_command(...)` 并通过 `text(...)` 返回结果的 JavaScript。shell 选项会保留， +命令执行与权限检查仍由 Codex 处理。合法的 JavaScript 后备字段、歧义对象和其他工具命名空间 +不会被转换；这项兼容修复不会绕过提供方限流，也不改变配置的重试策略。 + 所选 provider 必须支持 function/tool calling。不支持 tool call 的 text-only provider 无法使用 `exec`、 Browser 或 Computer Use。原生 OpenAI 条目会保持其上游 tool mode 不变。 diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index dcc7dafffde..45272a6c1db 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1583,6 +1583,7 @@ "devin-stated-reset-retry.test.ts": "providers", "ci-structure-gate.test.ts": "ci-workflows", "responses-code-mode-patch-compile.test.ts": "responses", + "responses-code-mode-shell-compile.test.ts": "responses", "gui-codex-usage-score-parity.test.ts": "gui", "web-search-sidecar-429.test.ts": "web-search", "management-google-tool-schema-policy.test.ts": "server", diff --git a/src/bridge/sse.ts b/src/bridge/sse.ts index 0466963eb57..7071e39af93 100644 --- a/src/bridge/sse.ts +++ b/src/bridge/sse.ts @@ -23,6 +23,7 @@ import { import { progressiveFreeformInput } from "../responses/progressive-freeform-input"; import { encodeCompactionSummary } from "../responses/compaction"; import { compileCodeModeHelperInput, resolveCodeModeHelperName } from "../responses/code-mode-helper-compat"; +import { mayBecomeCodeModeShellInput } from "../responses/code-mode-shell-input"; import { isTruncatedStopReason, truncationReasonFor } from "../responses/truncated-stop-reason"; import { encodeReasoningEnvelope, type ReasoningEnvelope } from "../responses/reasoning-envelope"; import { rememberReasoningForCall } from "../responses/reasoning-replay-cache"; @@ -1079,6 +1080,7 @@ export function bridgeToResponsesSSE( // replaced by the normalized ones. const mayNormalize = ownsFreeformGrammar && currentToolCall.name === "apply_patch"; if (!((mayCompile || mayNormalize) && mayBecomePatchEnvelope(full)) + && !(mayCompile && mayBecomeCodeModeShellInput(currentToolCall.args, full)) && full.startsWith(emitted) && full.length > emitted.length) { emit("response.custom_tool_call_input.delta", { item_id: currentToolCall.itemId, output_index: currentToolCall.outputIndex, diff --git a/src/responses/code-mode-helper-compat.ts b/src/responses/code-mode-helper-compat.ts index d390bb76cb4..dee812d4020 100644 --- a/src/responses/code-mode-helper-compat.ts +++ b/src/responses/code-mode-helper-compat.ts @@ -4,6 +4,7 @@ import { unwrapFreeformToolInput, } from "./apply-patch-envelope"; import { declaresCodeModeExec } from "../types/tools"; +import { parseCodeModeShellInput } from "./code-mode-shell-input"; function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); @@ -31,6 +32,10 @@ export function compileCodeModeHelperInput( const helperName = toolName.startsWith("default.") ? toolName.slice("default.".length) : toolName; + if (helperName === "exec_command" && wireToolName === "exec") { + const args = parseCodeModeShellInput(argumentsText); + if (args) return `const result = await tools.exec_command(${JSON.stringify(args)});\ntext(result);`; + } if (helperName === "apply_patch") { // `resolveCodeModeHelperName` decides this IS an apply-patch call by reading // `unwrapFreeformToolInput(argumentsText, wireToolName)`, which strips an outer Markdown @@ -96,8 +101,8 @@ export function compileCodeModeHelperInput( * wrong. * * This adds that second case: the name is already `exec` so nothing was rewritten, but - * the body is a complete patch envelope and therefore cannot be the JavaScript that - * `exec` runs. Same inference the name-based path makes, drawn from the payload. + * the body is a complete patch envelope or an unambiguous structured shell call. + * Same inference the name-based path makes, drawn from the payload. * * Returns undefined for everything else, including JavaScript that merely mentions a * patch envelope — that body is a real program and is forwarded byte-identical. @@ -116,5 +121,6 @@ export function resolveCodeModeHelperName( // `tools.apply_patch(...)` JavaScript would be the mis-route this repair exists to avoid. if (!declaresCodeModeExec(declaredNames)) return undefined; if (typeof argumentsText !== "string" || argumentsText === "") return undefined; - return isCompletePatchEnvelope(unwrapFreeformToolInput(argumentsText, "exec")) ? "apply_patch" : undefined; + if (isCompletePatchEnvelope(unwrapFreeformToolInput(argumentsText, "exec"))) return "apply_patch"; + return parseCodeModeShellInput(argumentsText) ? "exec_command" : undefined; } diff --git a/src/responses/code-mode-shell-input.ts b/src/responses/code-mode-shell-input.ts new file mode 100644 index 00000000000..79e83225ca4 --- /dev/null +++ b/src/responses/code-mode-shell-input.ts @@ -0,0 +1,54 @@ +import { unwrapFreeformToolInput } from "./apply-patch-envelope"; +import { scanFreeformWrapper } from "./freeform-wrapper-scan"; + +const SHELL_ARGUMENT_KEYS = new Set([ + "cmd", "command", "workdir", "shell", "login", "tty", "yield_time_ms", + "max_output_tokens", "sandbox_permissions", "justification", "prefix_rule", +]); +let javascriptParser: Bun.Transpiler | undefined; + +/** Recognize shell arguments, never guess a shell from an ordinary freeform program. */ +export function parseCodeModeShellInput(argumentsText: string): Record | undefined { + let parsed: unknown; + try { + // Only the canonical input wrapper is removed: the cmd/command object is the payload. + parsed = JSON.parse(unwrapFreeformToolInput(argumentsText)); + } catch { + return undefined; + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined; + const args = parsed as Record; + if (Object.keys(args).some(key => !SHELL_ARGUMENT_KEYS.has(key))) return undefined; + const keys = ["cmd", "command"].filter(key => Object.hasOwn(args, key)); + if (keys.length !== 1) return undefined; + const command = args[keys[0]!]; + if (typeof command !== "string" || command.trim() === "") return undefined; + // cmd/command also exist as historical JavaScript fallback fields. Preserve every valid + // program, including ambiguous identifiers such as `ls`. Parsing never executes the source. + try { + javascriptParser ??= new Bun.Transpiler({ loader: "js" }); + javascriptParser.scan(`async function __codeModeInput() {\n${command}\n}`); + return undefined; + } catch { + const { command: _alias, ...rest } = args; + return { ...rest, cmd: command }; + } +} + +/** Hold possible shell objects until completion can choose their executable representation. */ +export function mayBecomeCodeModeShellInput(argumentsText: string, input: string): boolean { + const head = input.trimStart(); + if (head === "" || head.startsWith("{")) return true; + // Canonical JavaScript streams progressively; avoid reparsing its growing wrapper on every + // delta. The shared prefix scanner is bounded independently of the command's size. + if (input === argumentsText || scanFreeformWrapper(argumentsText).kind === "input") return false; + try { + const args = JSON.parse(argumentsText); + // A fallback cmd value becomes visible only when the outer object closes. Do not emit + // that command before completion replaces it with tools.exec_command JavaScript. + return !!args && typeof args === "object" && !Object.hasOwn(args, "input") + && (Object.hasOwn(args, "cmd") || Object.hasOwn(args, "command")); + } catch { + return false; + } +} diff --git a/src/server/responses-custom-tool-repair.ts b/src/server/responses-custom-tool-repair.ts index 3b4130ae7f9..b0ba7f637ab 100644 --- a/src/server/responses-custom-tool-repair.ts +++ b/src/server/responses-custom-tool-repair.ts @@ -1,6 +1,7 @@ import type { TranslatorBudget } from "../lib/translator-budget"; import { mayBecomePatchEnvelope, normalizeApplyPatchDelimiters } from "../responses/apply-patch-envelope"; import { compileCodeModeHelperInput, resolveCodeModeHelperName } from "../responses/code-mode-helper-compat"; +import { mayBecomeCodeModeShellInput } from "../responses/code-mode-shell-input"; import { progressiveFreeformInput } from "../responses/progressive-freeform-input"; import { declaresCodeModeExec } from "../types/tools"; import { @@ -320,6 +321,7 @@ export function createRoutedCustomToolRestoreBlockRewrite( && itemName?.name === "exec"; const mayNormalize = ownsFreeformGrammar && itemName?.name === "apply_patch"; if ((mayCompile || mayNormalize) && mayBecomePatchEnvelope(fullInput)) return []; + if (mayCompile && mayBecomeCodeModeShellInput(open.argumentsText, fullInput)) return []; if (!fullInput.startsWith(open.emittedInput) || fullInput.length === open.emittedInput.length) return []; const inputDelta = fullInput.slice(open.emittedInput.length); open.emittedInput = fullInput; diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index cf601215ed8..6a262d6eb24 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -43,6 +43,9 @@ Chat models sometimes return a freeform call body under a common alternate field body in a Markdown fence. Restoration in `src/responses/apply-patch-envelope.ts` is deliberately narrow: only bare `exec` and `apply_patch` accept one recognized alternate field or one complete outer fence, while ambiguous wrappers and provider-owned freeform grammars remain byte-exact. +Structured shell arguments mistakenly sent to code-mode `exec` follow the shared +[Responses restoration contract](../transports/responses.md#responses-httpsse), including preview +holding and preservation of valid JavaScript fallback fields. Kiro groups only consecutive original-message tool results whose raw call ID exactly matches the originating call. Its wire-ID map retains the original ID privately so replacement or diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 09bf0d1ccb7..78436749647 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -125,6 +125,16 @@ Function-call wrappers around freeform bodies are restored by is recoverable because the wrapper is otherwise unusable; two alternate fields are ambiguous and therefore remain untouched. Foreign freeform grammars never receive that compatibility rewrite. +For a verified code-mode catalog, `src/responses/code-mode-shell-input.ts` recognizes a +structured `cmd` or `command` object submitted under `exec` and the canonical `input` wrapper. +Only known shell options and one command field are accepted, and any command that parses as +JavaScript remains unchanged, including ambiguous single identifiers. The existing helper +compiler serializes the recognized arguments into `tools.exec_command(...)` and emits its result +through `text(...)`; the proxy executes nothing. JSON, native Responses and adapter-event SSE +use the same completion rule. Possible shell-object previews stay held until completion so raw +JSON or shell text cannot precede the compiled JavaScript. Ordinary JavaScript stays progressive. +`tests/responses/responses-code-mode-shell-compile.test.ts` covers those paths and boundaries. + Progressive preview for those wrappers is decoded by `src/responses/progressive-freeform-input.ts` in both the adapter-event bridge and routed function-call restoration, over the classification in `src/responses/freeform-wrapper-scan.ts`. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 17dadddff96..115a9147f61 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1416,6 +1416,7 @@ "devin-stated-reset-retry.test.ts": "providers", "ci-structure-gate.test.ts": "ci-workflows", "responses-code-mode-patch-compile.test.ts": "responses", + "responses-code-mode-shell-compile.test.ts": "responses", "gui-codex-usage-score-parity.test.ts": "gui", "web-search-sidecar-429.test.ts": "web-search", "codex-shim-destroyed-probe.test.ts": "codex-integration", diff --git a/tests/responses/responses-code-mode-shell-compile.test.ts b/tests/responses/responses-code-mode-shell-compile.test.ts new file mode 100644 index 00000000000..5c5ced36510 --- /dev/null +++ b/tests/responses/responses-code-mode-shell-compile.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, test } from "bun:test"; +import { compileCodeModeHelperInput, resolveCodeModeHelperName } from "../../src/responses/code-mode-helper-compat"; +import { restoreRoutedCustomCallsInJson } from "../../src/responses/custom-tool-compat"; +import { bridgeToResponsesSSE, buildResponseJSON } from "../../src/bridge"; +import type { AdapterEvent } from "../../src/types"; +import { createRoutedCustomToolRestoreBlockRewrite } from "../../src/server/responses-custom-tool-repair"; +import { dataPayload, frame } from "../helpers/custom-tool-repair-fixtures"; + +const CODE_MODE = new Set(["exec"]); +const COMMAND = 'cd "/tmp/example repo" && git status --short'; + +describe("structured shell arguments submitted to code-mode exec", () => { + test("compiles the observed cmd object without losing shell options", async () => { + const args = { cmd: COMMAND, workdir: "/tmp", yield_time_ms: 1000, max_output_tokens: 2000 }; + const body = JSON.stringify(args); + expect(resolveCodeModeHelperName(undefined, "exec", body, undefined, CODE_MODE)).toBe("exec_command"); + const restored = JSON.parse(restoreRoutedCustomCallsInJson(JSON.stringify({ + output: [{ type: "function_call", id: "fc_shell", call_id: "call_shell", name: "exec", arguments: body }], + }), CODE_MODE, new Set(), CODE_MODE)); + const item = restored.output[0]; + expect(item).toMatchObject({ type: "custom_tool_call", name: "exec", call_id: "call_shell" }); + const calls: unknown[] = []; + const outputs: unknown[] = []; + const run = new Function("tools", "text", `return (async () => { ${item.input} })();`); + await run({ exec_command: async (value: unknown) => { calls.push(value); return { output: "ok" }; } }, + (value: unknown) => outputs.push(value)); + expect(calls).toEqual([args]); + expect(outputs).toEqual([{ output: "ok" }]); + }); + + test("the canonical input wrapper and command alias use the same shell payload", () => { + for (const args of [{ cmd: COMMAND }, { command: COMMAND }]) { + for (const body of [JSON.stringify(args), JSON.stringify({ input: JSON.stringify(args) })]) { + const helper = resolveCodeModeHelperName(undefined, "exec", body, undefined, CODE_MODE); + expect(helper).toBe("exec_command"); + expect(compileCodeModeHelperInput(body, helper!, "exec")) + .toBe(`const result = await tools.exec_command(${JSON.stringify({ cmd: COMMAND })});\ntext(result);`); + } + } + }); + + test("leaves JavaScript, ambiguous objects and unrelated catalogs alone", () => { + for (const body of [ + 'text("hello")', + JSON.stringify({ cmd: 'await tools.exec_command({ cmd: "pwd" });' }), + JSON.stringify({ command: 'text("hello")' }), + JSON.stringify({ cmd: "ls" }), // Also a valid JavaScript identifier: do not guess. + JSON.stringify({ input: "text(1)", cmd: COMMAND }), + JSON.stringify({ cmd: COMMAND, code: "text(1)" }), + JSON.stringify({ cmd: COMMAND, command: "echo other" }), + JSON.stringify({ cmd: COMMAND, unknownOption: true }), + JSON.stringify({ cmd: 42 }), + JSON.stringify([{ cmd: COMMAND }]), + '{"cmd":', + ]) expect(resolveCodeModeHelperName(undefined, "exec", body, undefined, CODE_MODE)).toBeUndefined(); + for (const declared of [undefined, new Set(["exec", "shell_command"]), new Set(["mcp__exec"])]) { + expect(resolveCodeModeHelperName(undefined, "exec", JSON.stringify({ cmd: COMMAND }), undefined, declared)).toBeUndefined(); + } + expect(resolveCodeModeHelperName(undefined, "exec", JSON.stringify({ cmd: COMMAND }), "mcp", CODE_MODE)).toBeUndefined(); + }); + + test("shell metacharacters remain data passed to the nested tool", async () => { + const args = { cmd: 'printf "%s" "`id` $(whoami)"\n# ${text("not source")}', tty: false }; + const body = JSON.stringify(args); + const helper = resolveCodeModeHelperName(undefined, "exec", body, undefined, CODE_MODE); + expect(helper).toBe("exec_command"); + const calls: unknown[] = []; + const run = new Function("tools", "text", `return (async () => { ${compileCodeModeHelperInput(body, helper!, "exec")} })();`); + await run({ exec_command: async (value: unknown) => { calls.push(value); return "ok"; } }, () => {}); + expect(calls).toEqual([args]); + }); + + test("Chat adapter JSON and fragmented SSE deliver the same executable call", async () => { + const args = { cmd: COMMAND, workdir: "/tmp" }; + const expected = `const result = await tools.exec_command(${JSON.stringify(args)});\ntext(result);`; + for (const body of [JSON.stringify(args), JSON.stringify({ input: JSON.stringify(args) })]) { + async function* events(): AsyncGenerator { + yield { type: "tool_call_start", id: "call-shell", name: "exec" }; + for (const arguments_ of body) yield { type: "tool_call_delta", id: "call-shell", arguments: arguments_ }; + yield { type: "tool_call_end", id: "call-shell" }; + yield { type: "done" }; + } + const options = { declaredToolNames: CODE_MODE }; + const collected: AdapterEvent[] = []; + for await (const event of events()) collected.push(event); + const json = buildResponseJSON(collected, "fixture", { ...options, freeformToolNames: CODE_MODE }); + expect(json.output).toMatchObject([{ type: "custom_tool_call", name: "exec", input: expected }]); + const stream = bridgeToResponsesSSE(events(), "fixture", undefined, CODE_MODE, undefined, undefined, 50_000, options); + const text = await new Response(stream).text(); + const payloads = text.split(/\r?\n\r?\n/).filter(block => block.includes("data: {")).map(dataPayload); + const preview = payloads.filter(p => p.type === "response.custom_tool_call_input.delta").map(p => p.delta).join(""); + expect(expected.startsWith(preview)).toBe(true); + expect(payloads.find(p => p.type === "response.custom_tool_call_input.done")?.input).toBe(expected); + expect(payloads.find(p => p.type === "response.output_item.done")?.item).toMatchObject({ input: expected }); + expect(payloads.find(p => p.type === "response.completed")?.response).toMatchObject({ output: [{ input: expected }] }); + } + }); + + test("native and lowered Responses streams agree at every split boundary", () => { + const args = { cmd: COMMAND }; + const expected = `const result = await tools.exec_command(${JSON.stringify(args)});\ntext(result);`; + for (const native of [false, true]) { + for (const body of [JSON.stringify(args), JSON.stringify({ input: JSON.stringify(args) })]) { + for (let split = 0; split <= body.length; split++) { + const rewrite = createRoutedCustomToolRestoreBlockRewrite(CODE_MODE, undefined, new Set(), CODE_MODE); + const type = native ? "custom_tool_call" : "function_call"; + const field = native ? "input" : "arguments"; + const event = native ? "response.custom_tool_call_input" : "response.function_call_arguments"; + const item = { type, id: "fc_shell", call_id: "call_shell", name: "exec", [field]: body }; + try { + rewrite(frame("response.output_item.added", { output_index: 0, item: { ...item, [field]: "" } })); + let preview = ""; + for (const delta of [body.slice(0, split), body.slice(split)]) { + preview += rewrite(frame(`${event}.delta`, { output_index: 0, item_id: "fc_shell", delta })) + .map(block => dataPayload(block).delta ?? "").join(""); + } + expect(preview).toBe(""); + const done = rewrite(frame(`${event}.done`, { output_index: 0, item_id: "fc_shell", [field]: body })); + expect(dataPayload(done[0]!).input).toBe(expected); + const itemDone = rewrite(frame("response.output_item.done", { output_index: 0, item })); + expect(dataPayload(itemDone[0]!).item).toMatchObject({ input: expected, call_id: "call_shell" }); + const terminal = rewrite(frame("response.completed", { response: { output: [item] } })); + expect(dataPayload(terminal[0]!).response).toMatchObject({ output: [{ input: expected }] }); + } finally { + rewrite.dispose?.(); + } + } + } + } + }); + + test("canonical JavaScript retains progressive output under a code-mode catalog", () => { + const rewrite = createRoutedCustomToolRestoreBlockRewrite(CODE_MODE, undefined, new Set(), CODE_MODE); + try { + rewrite(frame("response.output_item.added", { + output_index: 0, + item: { type: "function_call", id: "fc_js", call_id: "call_js", name: "exec", arguments: "" }, + })); + let preview = ""; + for (const delta of ['{"input":"text(', '1)', '"}']) { + preview += rewrite(frame("response.function_call_arguments.delta", { item_id: "fc_js", delta })) + .map(block => dataPayload(block).delta ?? "").join(""); + expect(preview.length).toBeGreaterThan(0); + } + expect(preview).toBe("text(1)"); + } finally { + rewrite.dispose?.(); + } + }); +});