diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 2b594646f..e3640c5f4 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -14,27 +14,17 @@ use thiserror::Error; use crate::DecisionTarget; -/// Capabilities that one route advertises on `GET /v1/models`. +/// Capabilities declared for one route. /// -/// An unset capability is undeclared: it serializes as `null` in the OpenAI -/// `data` entry, and the Codex entry falls back to a safe default for it. +/// `GET /v1/models` includes `context_window`, `tool_calling`, and `vision` in each +/// standard `data` entry, using `null` for unset values. `reasoning` remains route metadata. #[derive(Clone, Copy, Default)] pub struct ModelCapabilities { pub context_window: Option, pub tool_calling: Option, - /// Whether the routed model takes reasoning controls. A serving surface cannot - /// probe this, so a route opts in via config; undeclared routes advertise as - /// non-reasoning to Codex (fail closed). + /// Whether the routed model accepts reasoning controls, as declared in config. pub reasoning: Option, - /// Whether the routed model accepts image input. Declared per route for the same - /// reason as `reasoning`, and failing closed matters more here: a route may - /// resolve to a target with no vision at all. - /// - /// This is not cosmetic metadata. Codex reads `input_modalities` from the model - /// card and, when it reads text-only, replaces an attached image with the literal - /// text `image content omitted because you do not support image input` *before - /// sending*. An undeclared vision-capable route therefore loses the image in the - /// client, and the proxy never receives one to forward. + /// Whether the routed model accepts image input, as declared in config. pub vision: Option, } diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 885852a6c..c700c32ce 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -142,6 +142,18 @@ are required. All configured semantic names use exact ASCII case-insensitive mat handoff notes, per-tier system prompts, and a capability-judge fallback are documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). +## Codex model discovery + +`GET /v1/models` returns the standard `data` list and an empty Codex `models` list. +Codex keeps its own model catalog and instructions. Select a Switchyard route explicitly +with `codex --model route-id`; route aliases do not appear automatically in Codex's model +picker. Unknown aliases use Codex's generic defaults and do not receive Switchyard's +route-specific context limits or tool settings. + +To add instructions for a target, set `system_prompt` on its `[targets.]` entry. +Switchyard prepends that text when the selected target serves a completion and retains +the caller's instructions. Omit the setting to add no target instructions. + ## Endpoints | Method | Path | Purpose | diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index d239d66ad..6ecb9fdda 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1488,11 +1488,8 @@ fn model_list_payload<'a>( json!({ "object": "list", "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), - "models": entries - .iter() - .enumerate() - .map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority)) - .collect::>(), + // Codex requires this key; an empty list preserves its own catalog and instructions. + "models": [], "first_id": first_id, "last_id": last_id, "has_more": false, @@ -1523,83 +1520,6 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { }) } -// Builds the metadata Codex requires when it discovers models from a direct provider. -// -// This mirrors Codex's `ModelInfo` card. The benchmark harness builds the same card in -// `benchmark/codex_model_catalog_lib.py`; keep the two in sync when Codex changes -// the shape. Every field below is either derived from the route's declared capabilities or a -// required `ModelInfo` field the server has no better value for. -// -// Two kinds of fields live here. context_window, tool_calling, and reasoning are model -// facts a backend can publish; the route declares them in config today. The rest -// (shell_type, apply_patch_tool_type, base_instructions, the reasoning-level presets, -// truncation_policy) are Codex client conventions no backend returns, so they stay -// constant. -// -// TODO: source context_window, tool_calling, and reasoning from the backend, not route -// config. Switchyard is a proxy, so it should re-publish what the backend advertises -// when it can — OpenRouter's /api/v1/models exposes context_length and -// supported_parameters — and fall back to the route's declared value. Some backends -// publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info), -// so keep failing closed to config. -fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority: usize) -> Value { - // Codex is non-functional without shell and apply_patch, so an undeclared tool - // capability defaults to enabled here; the OpenAI `data` entry reports the raw - // Option separately for clients that want the undeclared state. - let tool_calling = capabilities.tool_calling.unwrap_or(true); - let reasoning = capabilities.reasoning.unwrap_or(false); - json!({ - "slug": model, - "display_name": model, - "description": "Switchyard-routed model.", - "default_reasoning_level": if reasoning { json!("xhigh") } else { Value::Null }, - "supported_reasoning_levels": if reasoning { reasoning_levels() } else { json!([]) }, - "shell_type": if tool_calling { "shell_command" } else { "disabled" }, - "visibility": "list", - "supported_in_api": true, - // Catalog list position (routes are listed in sorted id order), not a quality rank. - "priority": priority, - "additional_speed_tiers": [], - "availability_nux": null, - "upgrade": null, - // Required `ModelInfo` string. Unlike the launcher, the server cannot read - // Codex's bundled prompt, so it sends a minimal stub. - "base_instructions": "You are Codex, a coding agent.", - "supports_reasoning_summaries": reasoning, - "default_reasoning_summary": "none", - "support_verbosity": reasoning, - "default_verbosity": if reasoning { json!("low") } else { Value::Null }, - "apply_patch_tool_type": if tool_calling { Some("freeform") } else { None }, - "web_search_tool_type": "text", - "truncation_policy": {"mode": "tokens", "limit": 10_000}, - "supports_parallel_tool_calls": tool_calling, - "supports_image_detail_original": false, - "context_window": capabilities.context_window, - "max_context_window": capabilities.context_window, - "effective_context_window_percent": 95, - "experimental_supported_tools": [], - // Codex omits an attached image client-side when this says text-only, so a - // route whose target can see must declare `vision = true`. Fails closed. - "input_modalities": if capabilities.vision.unwrap_or(false) { - json!(["text", "image"]) - } else { - json!(["text"]) - }, - "supports_search_tool": false, - }) -} - -// The reasoning-effort presets Codex offers for a reasoning-capable route. Kept in -// step with the benchmark template in `codex_model_catalog_lib.py`. -fn reasoning_levels() -> Value { - json!([ - {"effort": "low", "description": "Fast responses with lighter reasoning"}, - {"effort": "medium", "description": "Balances speed and reasoning depth"}, - {"effort": "high", "description": "Greater reasoning depth"}, - {"effort": "xhigh", "description": "Extra high reasoning depth"}, - ]) -} - fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String { let scheme = if options.is_tls() { "https" } else { "http" }; let listen_url = url_for_addr(scheme, options.addr); diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 7097a7529..0734445d5 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2628,6 +2628,7 @@ type = "passthrough" target = "shared" context_window = 1000000 tool_calling = true +vision = true [routes.restricted] id = "restricted" @@ -2635,12 +2636,7 @@ type = "passthrough" target = "shared" context_window = 262000 tool_calling = false - -[routes.reasoning] -id = "reasoning" -type = "passthrough" -target = "shared" -reasoning = true +vision = false [routes.undeclared] id = "undeclared" @@ -2648,7 +2644,7 @@ type = "passthrough" target = "shared" "#; let app = build_switchyard_router(load_test_config(CONFIG)?); - let models = send(&app, "GET", "/v1/models", None).await?; + let models = send(&app, "GET", "/v1/models?client_version=0.152.0", None).await?; assert_eq!(models.status, StatusCode::OK); let body = models.json()?; let data = body["data"].as_array().cloned().unwrap_or_default(); @@ -2664,88 +2660,11 @@ target = "shared" assert_eq!(capabilities["undeclared"]["context_window"], json!(null)); assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null)); - let codex_models = body["models"].as_array().cloned().unwrap_or_default(); - let codex_metadata = codex_models - .iter() - .filter_map(|entry| entry["slug"].as_str().map(|slug| (slug, entry))) - .collect::>(); - // This checks the shape the server emits. That Codex 0.144.5 actually decodes it - // (context_window: null included) is verified by a live Codex run in SWITCH-1225. - assert_eq!(codex_metadata.len(), 4); - assert_eq!( - codex_metadata["declared"]["context_window"], - json!(1_000_000) - ); - assert_eq!(codex_metadata["declared"]["shell_type"], "shell_command"); - assert_eq!( - codex_metadata["declared"]["apply_patch_tool_type"], - "freeform" - ); - // Constant fields Codex requires: a typo here would fail its decode, so pin them. - assert_eq!(codex_metadata["declared"]["visibility"], "list"); - assert_eq!(codex_metadata["declared"]["supported_in_api"], json!(true)); - assert_eq!(codex_metadata["declared"]["web_search_tool_type"], "text"); - assert_eq!( - codex_metadata["declared"]["input_modalities"], - json!(["text"]) - ); - assert_eq!( - codex_metadata["declared"]["truncation_policy"], - json!({"mode": "tokens", "limit": 10_000}) - ); - assert_eq!( - codex_metadata["restricted"]["context_window"], - json!(262_000) - ); - assert_eq!(codex_metadata["restricted"]["shell_type"], "disabled"); - assert_eq!( - codex_metadata["restricted"]["apply_patch_tool_type"], - json!(null) - ); - // A reasoning route advertises the effort presets and reasoning controls. - assert_eq!( - codex_metadata["reasoning"]["default_reasoning_level"], - "xhigh" - ); - assert_eq!( - codex_metadata["reasoning"]["supported_reasoning_levels"] - .as_array() - .map(Vec::len), - Some(4) - ); - assert_eq!( - codex_metadata["reasoning"]["supports_reasoning_summaries"], - json!(true) - ); - assert_eq!( - codex_metadata["reasoning"]["support_verbosity"], - json!(true) - ); - assert_eq!(codex_metadata["reasoning"]["default_verbosity"], "low"); - // An undeclared route: null context window, non-reasoning, but tools default on so Codex - // remains usable when connected directly to the server. - assert_eq!(codex_metadata["undeclared"]["context_window"], json!(null)); - assert_eq!( - codex_metadata["undeclared"]["supported_reasoning_levels"], - json!([]) - ); - assert_eq!( - codex_metadata["undeclared"]["default_reasoning_level"], - json!(null) - ); - assert_eq!( - codex_metadata["undeclared"]["supports_reasoning_summaries"], - json!(false) - ); - assert_eq!(codex_metadata["undeclared"]["shell_type"], "shell_command"); - assert_eq!( - codex_metadata["undeclared"]["apply_patch_tool_type"], - "freeform" - ); - assert_eq!( - codex_metadata["undeclared"]["supports_parallel_tool_calls"], - json!(true) - ); + assert_eq!(capabilities["declared"]["vision"], json!(true)); + assert_eq!(capabilities["restricted"]["vision"], json!(false)); + assert_eq!(capabilities["undeclared"]["vision"], json!(null)); + assert_eq!(body["models"], json!([])); + Ok(()) } @@ -4366,75 +4285,3 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { ); Ok(()) } - -// Verifies a route declaring `vision = true` advertises image input, and that an -// undeclared route still fails closed to text-only. -// -// This is not cosmetic metadata. Codex reads `input_modalities` from the model card -// and, when it reads text-only, replaces an attached image with the literal text -// "image content omitted because you do not support image input" before sending — so -// a route whose target can see but which does not say so loses the image in the -// client, and Switchyard never receives one to forward. -#[tokio::test] -async fn models_endpoint_advertises_image_input_only_for_vision_routes() -> TestResult { - const CONFIG: &str = r#" -schema_version = 1 - -[llm_clients.shared] -format = "openai_responses" -base_url = "http://127.0.0.1:1/v1" - -[targets.shared] -id = "shared-model" -llm_client = "shared" - -[routes.sees] -id = "sees" -type = "passthrough" -target = "shared" -vision = true - -[routes.blind] -id = "blind" -type = "passthrough" -target = "shared" -"#; - let app = build_switchyard_router(load_test_config(CONFIG)?); - let models = send(&app, "GET", "/v1/models", None).await?; - assert_eq!(models.status, StatusCode::OK); - let body = models.json()?; - - let codex_metadata = body["models"] - .as_array() - .cloned() - .unwrap_or_default() - .iter() - .filter_map(|entry| { - entry["slug"] - .as_str() - .map(|slug| (slug.to_string(), entry.clone())) - }) - .collect::>(); - assert_eq!( - codex_metadata["sees"]["input_modalities"], - json!(["text", "image"]) - ); - assert_eq!(codex_metadata["blind"]["input_modalities"], json!(["text"])); - - // The OpenAI `data` entry reports the raw Option, so an undeclared route stays - // distinguishable from one that declared `false`. - let capabilities = body["data"] - .as_array() - .cloned() - .unwrap_or_default() - .iter() - .filter_map(|entry| { - entry["id"] - .as_str() - .map(|id| (id.to_string(), entry["capabilities"].clone())) - }) - .collect::>(); - assert_eq!(capabilities["sees"]["vision"], json!(true)); - assert_eq!(capabilities["blind"]["vision"], json!(null)); - Ok(()) -} diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 87a3eeb4e..9688a501d 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -62,10 +62,9 @@ inside the TOML file. Their `id` fields have different external meanings: The server lists route IDs on `GET /v1/models`. A request selects a route by putting that ID in its `model` field. The native Rust server does not discover -or register additional provider models automatically. The same response also -carries a Codex-compatible `models` array so Codex can use the server as a direct -provider; each entry reflects the route's declared context window, tool support, -and reasoning. +or register additional provider models automatically. The response includes an empty +Codex `models` array so Codex keeps its own catalog and instructions. Select route +aliases explicitly; they do not appear automatically in Codex's model picker. ## Routing Algorithms diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index b482ecb6e..c820d21d1 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -115,8 +115,8 @@ Every route takes the common keys below, plus the keys for its type. | `type` | Yes | — | Routing algorithm for this route. | | `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. | | `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. | -| `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. | -| `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. | +| `reasoning` | No | unset | Declared reasoning support, stored in route metadata. The server does not include it in `GET /v1/models`. | +| `vision` | No | unset | Image-input support advertised in `GET /v1/models` under `data[].capabilities.vision`. Unset values appear as `null`. Declare `true` only when every target the route can select accepts images. | ### `noop`