From 800b30169671e478f9e25c112566ba8853214c2a Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 14 Sep 2026 13:26:38 -0700 Subject: [PATCH 1/6] fix(server): serve a configured Codex prompt as base_instructions Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/README.md | 20 ++++++++ crates/switchyard-server/src/cli.rs | 15 ++++++ crates/switchyard-server/src/lib.rs | 44 +++++++++++++---- crates/switchyard-server/tests/server.rs | 60 ++++++++++++++++++++++++ docs/cli_reference.md | 1 + 5 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 885852a6c..07b5ea42d 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -142,6 +142,26 @@ are required. All configured semantic names use exact ASCII case-insensitive mat handoff notes, per-tier system prompts, and a capability-judge fallback are documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). +## Codex model discovery + +`GET /v1/models` also returns a `models` array in the shape Codex reads from a direct +provider. Codex requires a `base_instructions` string on every entry and adopts it as the +session's system prompt, replacing its own bundled instructions. Without configuration the +server sends the placeholder `You are Codex, a coding agent.`, so a routed session runs on a +one-line prompt. To keep parity with a direct session, save Codex's bundled prompt to a file +and pass `--codex-base-instructions-file PATH`; the file's contents are served verbatim to +every route. + +```bash +codex debug models --bundled \ + | python3 -c 'import json,sys; m=json.load(sys.stdin)["models"]; print(next(x for x in m if x["slug"]=="gpt-5.6-sol")["base_instructions"], end="")' \ + > codex-base-instructions.md +switchyard-server --config routes.toml --codex-base-instructions-file codex-base-instructions.md +``` + +A blank file is rejected at startup because Codex discards the whole catalog when the field +is empty. + ## Endpoints | Method | Path | Purpose | diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 231d4e2bc..ae6c8906d 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,6 +52,12 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, + /// File whose contents Codex adopts as its system prompt for every route in + /// `GET /v1/models`. Without it Codex gets a one-line placeholder instead of its + /// bundled instructions; export them with `codex debug models --bundled`. + #[arg(long, value_name = "PATH")] + codex_base_instructions_file: Option, + /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] tls_cert: Option, @@ -72,6 +78,15 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } + if let Some(path) = self.codex_base_instructions_file { + let text = std::fs::read_to_string(&path).map_err(|error| { + ServerError::new(format!( + "invalid --codex-base-instructions-file {}: {error}", + path.display() + )) + })?; + state = state.with_codex_base_instructions(text)?; + } let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { if !cert.exists() || !key.exists() { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index d239d66ad..8d5d4607b 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -87,6 +87,11 @@ fn should_forward_upstream_header(name: &HeaderName) -> bool { /// Non-standard status used only in logs and metrics for a request whose /// downstream client disconnected before any response was written. const CLIENT_CLOSED_REQUEST: u16 = 499; + +/// `base_instructions` served in the Codex model catalog when no +/// `--codex-base-instructions-file` is given. Codex adopts this string as its system +/// prompt, so operators wanting parity with a direct session must supply the file. +const DEFAULT_CODEX_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent."; const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt"); /// Error returned while configuring or running the server. @@ -162,6 +167,7 @@ pub struct ServerState { stats: StatsAccumulator, routing_log: Option, track_cache_eligibility: bool, + codex_base_instructions: Arc, } #[derive(Clone)] @@ -217,6 +223,7 @@ impl ServerState { stats, routing_log: None, track_cache_eligibility: tracking_enabled_from_env(), + codex_base_instructions: Arc::from(DEFAULT_CODEX_BASE_INSTRUCTIONS), }) } @@ -226,6 +233,20 @@ impl ServerState { Ok(self) } + /// Serves `text` verbatim as `base_instructions` for every Codex catalog entry. + /// + /// Rejects blank text: Codex discards the whole catalog when the field is empty. + pub fn with_codex_base_instructions(mut self, text: impl Into) -> ServerResult { + let text = text.into(); + if text.trim().is_empty() { + return Err(ServerError::new( + "codex base instructions must not be blank", + )); + } + self.codex_base_instructions = Arc::from(text); + Ok(self) + } + /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.runner.models().map(|model| model.id.as_str()) @@ -1397,6 +1418,7 @@ async fn models(State(state): State) -> Json { .runner .models() .map(|model| (model.id.as_str(), model.capabilities)), + &state.codex_base_instructions, )) } @@ -1479,6 +1501,7 @@ async fn not_found() -> Response { fn model_list_payload<'a>( entries: impl IntoIterator, + base_instructions: &str, ) -> Value { let mut entries = entries.into_iter().collect::>(); entries.sort_unstable_by_key(|(model_id, _)| *model_id); @@ -1491,7 +1514,9 @@ fn model_list_payload<'a>( "models": entries .iter() .enumerate() - .map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority)) + .map(|(priority, (model, caps))| { + codex_model_entry_json(model, *caps, priority, base_instructions) + }) .collect::>(), "first_id": first_id, "last_id": last_id, @@ -1532,9 +1557,9 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { // // Two kinds of fields live here. context_window, tool_calling, and reasoning are model // facts a backend can publish; the route declares them in config today. The rest -// (shell_type, apply_patch_tool_type, base_instructions, the reasoning-level presets, -// truncation_policy) are Codex client conventions no backend returns, so they stay -// constant. +// (shell_type, apply_patch_tool_type, the reasoning-level presets, truncation_policy) are +// Codex client conventions no backend returns, so they stay constant. `base_instructions` +// is whatever the operator configured; Codex adopts it as the session's system prompt. // // TODO: source context_window, tool_calling, and reasoning from the backend, not route // config. Switchyard is a proxy, so it should re-publish what the backend advertises @@ -1542,7 +1567,12 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { // supported_parameters — and fall back to the route's declared value. Some backends // publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info), // so keep failing closed to config. -fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority: usize) -> Value { +fn codex_model_entry_json( + model: &str, + capabilities: ModelCapabilities, + priority: usize, + base_instructions: &str, +) -> Value { // Codex is non-functional without shell and apply_patch, so an undeclared tool // capability defaults to enabled here; the OpenAI `data` entry reports the raw // Option separately for clients that want the undeclared state. @@ -1562,9 +1592,7 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority "additional_speed_tiers": [], "availability_nux": null, "upgrade": null, - // Required `ModelInfo` string. Unlike the launcher, the server cannot read - // Codex's bundled prompt, so it sends a minimal stub. - "base_instructions": "You are Codex, a coding agent.", + "base_instructions": base_instructions, "supports_reasoning_summaries": reasoning, "default_reasoning_summary": "none", "support_verbosity": reasoning, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 7097a7529..451b0edcb 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2975,6 +2975,66 @@ subagents = {{ type = "passthrough", target = "strong" }} Ok(()) } +#[tokio::test] +async fn codex_catalog_serves_configured_base_instructions_verbatim() -> TestResult { + const CONFIG: &str = r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[targets.shared] +id = "nvidia/deepseek-ai/deepseek-v4-pro" +llm_client = "primary" + +[routes.a] +id = "route/a" +type = "passthrough" +target = "shared" + +[routes.b] +id = "route/b" +type = "passthrough" +target = "shared" +"#; + let codex_instructions = |state: ServerState| async move { + let body = send(&build_switchyard_router(state), "GET", "/v1/models", None) + .await? + .json()?; + TestResult::Ok( + body["models"] + .as_array() + .cloned() + .unwrap_or_default() + .into_iter() + .map(|entry| entry["base_instructions"].clone()) + .collect::>(), + ) + }; + + // Without configuration every route carries the placeholder Codex requires to decode + // the catalog at all. + assert_eq!( + codex_instructions(load_test_config(CONFIG)?).await?, + vec![json!("You are Codex, a coding agent."); 2] + ); + + // Codex's own prompt is multi-line with trailing whitespace; it must reach the + // catalog byte for byte. + let prompt = "You are Codex, an agent based on GPT-5.\n\n # Tools\n\n- shell \n"; + let state = load_test_config(CONFIG)?.with_codex_base_instructions(prompt)?; + assert_eq!(codex_instructions(state).await?, vec![json!(prompt); 2]); + + // A blank prompt would make Codex discard the whole catalog, so it is rejected. + assert!( + load_test_config(CONFIG)? + .with_codex_base_instructions(" \n") + .is_err() + ); + Ok(()) +} + #[tokio::test] async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; diff --git a/docs/cli_reference.md b/docs/cli_reference.md index e5da32455..309cd34f9 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,6 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | +| `--codex-base-instructions-file PATH` | None | File served as `base_instructions` for every route in `GET /v1/models`. Codex adopts it as its system prompt; without it Codex gets a one-line placeholder. Export the bundled prompt with `codex debug models --bundled`. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | From e67de87b6e5e623408969d68819719f8d93fcbb8 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 14 Sep 2026 14:52:15 -0700 Subject: [PATCH 2/6] fix(server): let operators set the Codex system prompt Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/README.md | 22 ++++++----- crates/switchyard-server/src/cli.rs | 7 ++-- crates/switchyard-server/src/lib.rs | 13 ++++--- crates/switchyard-server/tests/cli.rs | 48 ++++++++++++++++++++++++ crates/switchyard-server/tests/server.rs | 12 +----- docs/cli_reference.md | 2 +- 6 files changed, 75 insertions(+), 29 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 07b5ea42d..af70af926 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -144,13 +144,16 @@ handoff notes, per-tier system prompts, and a capability-judge fallback are docu ## Codex model discovery -`GET /v1/models` also returns a `models` array in the shape Codex reads from a direct -provider. Codex requires a `base_instructions` string on every entry and adopts it as the -session's system prompt, replacing its own bundled instructions. Without configuration the -server sends the placeholder `You are Codex, a coding agent.`, so a routed session runs on a -one-line prompt. To keep parity with a direct session, save Codex's bundled prompt to a file -and pass `--codex-base-instructions-file PATH`; the file's contents are served verbatim to -every route. +`GET /v1/models` includes a `models` array for Codex. Codex uses each entry's +`base_instructions` as its system prompt, replacing its bundled instructions. By default, +Switchyard sends the one-line placeholder `You are Codex, a coding agent.`. + +Use `--codex-base-instructions-file PATH` to choose the system prompt for routed Codex +sessions. The server reads the UTF-8 file once at startup and preserves its whitespace. The +same text applies to every route; the operator chooses the prompt independently of the +target model. + +This example exports Codex's bundled prompt for `gpt-5.6-sol`: ```bash codex debug models --bundled \ @@ -159,8 +162,9 @@ codex debug models --bundled \ switchyard-server --config routes.toml --codex-base-instructions-file codex-base-instructions.md ``` -A blank file is rejected at startup because Codex discards the whole catalog when the field -is empty. +The server stops startup if the file is missing, unreadable, contains invalid UTF-8, or +contains only whitespace. After updating Codex or choosing a different prompt, export the +file again and restart the server. ## Endpoints diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index ae6c8906d..b7f5bc943 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,9 +52,10 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, - /// File whose contents Codex adopts as its system prompt for every route in - /// `GET /v1/models`. Without it Codex gets a one-line placeholder instead of its - /// bundled instructions; export them with `codex debug models --bundled`. + /// Read the Codex system prompt from this UTF-8 file once at startup. The same + /// text applies to every route in `GET /v1/models`. Without this option, Codex + /// gets a one-line placeholder. Export bundled prompts with + /// `codex debug models --bundled`, then choose one to save in the file. #[arg(long, value_name = "PATH")] codex_base_instructions_file: Option, diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 8d5d4607b..34a112db6 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -88,9 +88,9 @@ fn should_forward_upstream_header(name: &HeaderName) -> bool { /// downstream client disconnected before any response was written. const CLIENT_CLOSED_REQUEST: u16 = 499; -/// `base_instructions` served in the Codex model catalog when no -/// `--codex-base-instructions-file` is given. Codex adopts this string as its system -/// prompt, so operators wanting parity with a direct session must supply the file. +/// Default `base_instructions` in `GET /v1/models`. Codex uses this placeholder +/// as its system prompt. Operators can supply their chosen instructions with +/// `--codex-base-instructions-file`. const DEFAULT_CODEX_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent."; const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt"); @@ -233,9 +233,10 @@ impl ServerState { Ok(self) } - /// Serves `text` verbatim as `base_instructions` for every Codex catalog entry. + /// Uses `text` as `base_instructions` for every Codex entry in `GET /v1/models`. + /// Preserves whitespace. /// - /// Rejects blank text: Codex discards the whole catalog when the field is empty. + /// Returns an error for blank text so Codex does not receive an empty system prompt. pub fn with_codex_base_instructions(mut self, text: impl Into) -> ServerResult { let text = text.into(); if text.trim().is_empty() { @@ -1559,7 +1560,7 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { // facts a backend can publish; the route declares them in config today. The rest // (shell_type, apply_patch_tool_type, the reasoning-level presets, truncation_policy) are // Codex client conventions no backend returns, so they stay constant. `base_instructions` -// is whatever the operator configured; Codex adopts it as the session's system prompt. +// is the operator's chosen text or the default placeholder. Codex uses it as the system prompt. // // TODO: source context_window, tool_calling, and reasoning from the backend, not route // config. Switchyard is a proxy, so it should re-publish what the backend advertises diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index f72c45858..07b994df0 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -43,3 +43,51 @@ target = "invalid" ); Ok(()) } + +#[test] +fn dry_run_validates_codex_instruction_files() -> TestResult { + let directory = tempfile::tempdir()?; + let config = directory.path().join("routes.toml"); + fs::write( + &config, + r#" +schema_version = 1 +[llm_clients.local] +format = "openai_chat" +base_url = "http://127.0.0.1:1/v1" +[targets.local] +id = "upstream-model" +llm_client = "local" +[routes.local] +id = "test-route" +type = "passthrough" +target = "local" +"#, + )?; + let prompt = directory.path().join("instructions.txt"); + for (contents, expected_success) in [ + (None, false), + (Some(b" \n\t".as_slice()), false), + (Some(b"\xff".as_slice()), false), + ( + Some(b"Configured instructions.\n Keep whitespace. \n".as_slice()), + true, + ), + ] { + if let Some(contents) = contents { + fs::write(&prompt, contents)?; + } + let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) + .arg("--config") + .arg(&config) + .arg("--dry-run") + .arg("--codex-base-instructions-file") + .arg(&prompt) + .output()?; + assert_eq!(output.status.success(), expected_success); + if !expected_success { + assert!(String::from_utf8(output.stderr)?.contains("codex")); + } + } + Ok(()) +} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 451b0edcb..67f822c24 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -3013,25 +3013,17 @@ target = "shared" ) }; - // Without configuration every route carries the placeholder Codex requires to decode - // the catalog at all. + // Without a configured prompt, every route uses the default placeholder. assert_eq!( codex_instructions(load_test_config(CONFIG)?).await?, vec![json!("You are Codex, a coding agent."); 2] ); - // Codex's own prompt is multi-line with trailing whitespace; it must reach the - // catalog byte for byte. + // Preserve line breaks and trailing whitespace in the configured prompt. let prompt = "You are Codex, an agent based on GPT-5.\n\n # Tools\n\n- shell \n"; let state = load_test_config(CONFIG)?.with_codex_base_instructions(prompt)?; assert_eq!(codex_instructions(state).await?, vec![json!(prompt); 2]); - // A blank prompt would make Codex discard the whole catalog, so it is rejected. - assert!( - load_test_config(CONFIG)? - .with_codex_base_instructions(" \n") - .is_err() - ); Ok(()) } diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 309cd34f9..3d05d8ba8 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,7 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | -| `--codex-base-instructions-file PATH` | None | File served as `base_instructions` for every route in `GET /v1/models`. Codex adopts it as its system prompt; without it Codex gets a one-line placeholder. Export the bundled prompt with `codex debug models --bundled`. | +| `--codex-base-instructions-file PATH` | None | Read a UTF-8 file once at startup and use its text as the Codex system prompt for every route in `GET /v1/models`. Without this option, Codex gets a one-line placeholder. See [Codex model discovery](../crates/switchyard-server/README.md#codex-model-discovery) for prompt export and restart instructions. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | From 9e1f5612edf1ff8e8a8040ed58054865329e6c3c Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 14 Sep 2026 16:14:09 -0700 Subject: [PATCH 3/6] chore(server): explain instruction file validation test Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/tests/cli.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index 07b994df0..5e96c80ab 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -44,6 +44,8 @@ target = "invalid" Ok(()) } +/// Checks that `--dry-run` accepts valid UTF-8 instructions and rejects missing, +/// blank, or invalid UTF-8 files. #[test] fn dry_run_validates_codex_instruction_files() -> TestResult { let directory = tempfile::tempdir()?; From ab796cdccee8a290666ced8afcd025cb9e083f7a Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 15 Sep 2026 11:34:01 -0700 Subject: [PATCH 4/6] fix(server): preserve Codex instructions by default Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 17 ++++ crates/switchyard-server/Cargo.toml | 1 + crates/switchyard-server/README.md | 68 ++++++++++++---- crates/switchyard-server/src/cli.rs | 15 ++-- crates/switchyard-server/src/lib.rs | 98 ++++++++++++++++++------ crates/switchyard-server/tests/cli.rs | 89 +++++++++++++++++---- crates/switchyard-server/tests/server.rs | 94 ++++++++++------------- docs/cli_reference.md | 2 +- 8 files changed, 265 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b353d0657..bfb535534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,6 +1216,12 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memo-map" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5449c8c750f1a07ea702bbd212bd999fceece9b3d1508b17023b3e174583124b" + [[package]] name = "micromap" version = "0.3.0" @@ -1228,6 +1234,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "2.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86886cf6dbf4e614b19c9a1eec9775f021869d7eadde0fc73921a81b90c9b4c9" +dependencies = [ + "memo-map", + "serde", +] + [[package]] name = "mio" version = "1.2.2" @@ -2451,6 +2467,7 @@ dependencies = [ "http", "http-body-util", "humantime", + "minijinja", "opentelemetry", "opentelemetry-otlp", "opentelemetry-prometheus", diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 5c25b5805..a0a5df9e7 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -29,6 +29,7 @@ clap = { version = "4", features = ["derive", "env"] } futures-util.workspace = true http.workspace = true libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" } +minijinja = { version = "2.24", default-features = false, features = ["builtins", "serde"] } opentelemetry = { version = "0.32", default-features = false, features = ["metrics", "trace"] } opentelemetry-otlp = { version = "0.32", default-features = false, features = ["http-proto", "metrics", "reqwest-blocking-client", "reqwest-rustls", "trace"] } opentelemetry-prometheus = "0.32" diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index af70af926..6fedcb6c5 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -144,27 +144,61 @@ handoff notes, per-tier system prompts, and a capability-judge fallback are docu ## Codex model discovery -`GET /v1/models` includes a `models` array for Codex. Codex uses each entry's -`base_instructions` as its system prompt, replacing its bundled instructions. By default, -Switchyard sends the one-line placeholder `You are Codex, a coding agent.`. - -Use `--codex-base-instructions-file PATH` to choose the system prompt for routed Codex -sessions. The server reads the UTF-8 file once at startup and preserves its whitespace. The -same text applies to every route; the operator chooses the prompt independently of the -target model. - -This example exports Codex's bundled prompt for `gpt-5.6-sol`: +By default, `GET /v1/models` returns the standard `data` list and an empty Codex +`models` list. Codex keeps its bundled model catalog and instructions. Switchyard route +aliases remain usable through explicit model selection, such as `codex --model route-id`, +but do not appear automatically in Codex's model picker. For names it does not recognize, +Codex uses its own generic defaults; it does not receive Switchyard's custom context limits +or tool settings. + +**Custom Codex model records require an explicit system template.** Set +`--codex-system-template PATH` to publish a record for each route. Switchyard renders the +UTF-8 template once per route at startup and sends the result as `base_instructions`. +This is the complete replacement prompt for those records, including their tool-use +instructions. It does not inherit or append Codex's bundled prompt. + +Save this minimal example as `codex-system.jinja` and adapt it for your models: + +```jinja +You are a coding assistant working through route {{ model_id }}. +Read the relevant code before editing. Verify changes with focused checks. +{% if tool_calling is true %} +Use the available tools to inspect files, make changes, and run checks. +{% endif %} +{% if context_window is not none %} +The route advertises a context window of {{ context_window }} tokens. +{% endif %} +``` ```bash -codex debug models --bundled \ - | python3 -c 'import json,sys; m=json.load(sys.stdin)["models"]; print(next(x for x in m if x["slug"]=="gpt-5.6-sol")["base_instructions"], end="")' \ - > codex-base-instructions.md -switchyard-server --config routes.toml --codex-base-instructions-file codex-base-instructions.md +switchyard-server --config routes.toml --codex-system-template codex-system.jinja ``` -The server stops startup if the file is missing, unreadable, contains invalid UTF-8, or -contains only whitespace. After updating Codex or choosing a different prompt, export the -file again and restart the server. +Switchyard renders [MiniJinja syntax](https://docs.rs/minijinja/2.24.0/minijinja/syntax/index.html), +including expressions, conditionals, loops, and built-in filters. These variables describe +the configured route: + +| Variable | Value | +|---|---| +| `model_id` | Public route ID from `/v1/models`, not the backend model selected later. | +| `context_window` | Declared context limit in tokens, or `none`. | +| `tool_calling` | Declared tool support: `true`, `false`, or `none`. | +| `reasoning` | Declared reasoning support: `true`, `false`, or `none`. | +| `vision` | Declared image support: `true`, `false`, or `none`. | + +Use `is true`, `is false`, or `is none` to distinguish supported, unsupported, and undeclared +capabilities. The template has no access to Codex personality settings, session data, +conversation messages, or environment variables. File includes and template imports are +not supported. + +Startup rejects unreadable or invalid UTF-8 files, invalid syntax, rendering errors, and +instructions that render as only whitespace for any route. It also checks undeclared names +reported by MiniJinja. Using a template requires at least one configured route; without a +template, an empty route configuration is valid. + +Rendered text preserves whitespace and does not use HTML escaping. Restart the server after +editing the template. Existing Codex sessions may retain earlier instructions; use a fresh +session when checking a changed template. ## Endpoints diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index b7f5bc943..cff0343e6 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,12 +52,11 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, - /// Read the Codex system prompt from this UTF-8 file once at startup. The same - /// text applies to every route in `GET /v1/models`. Without this option, Codex - /// gets a one-line placeholder. Export bundled prompts with - /// `codex debug models --bundled`, then choose one to save in the file. + /// Publish custom Codex model records using this UTF-8 Jinja system template. + /// Render once per route at startup. Without this option, Codex keeps its + /// bundled model catalog and instructions. #[arg(long, value_name = "PATH")] - codex_base_instructions_file: Option, + codex_system_template: Option, /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] @@ -79,14 +78,14 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } - if let Some(path) = self.codex_base_instructions_file { + if let Some(path) = self.codex_system_template { let text = std::fs::read_to_string(&path).map_err(|error| { ServerError::new(format!( - "invalid --codex-base-instructions-file {}: {error}", + "invalid --codex-system-template {}: {error}", path.display() )) })?; - state = state.with_codex_base_instructions(text)?; + state = state.with_codex_system_template(&text)?; } let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 34a112db6..d9273ad21 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -34,6 +34,7 @@ use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; use libsy::{LibsyError, RoutingOutcome}; +use minijinja::{AutoEscape, Environment, UndefinedBehavior}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -88,10 +89,6 @@ fn should_forward_upstream_header(name: &HeaderName) -> bool { /// downstream client disconnected before any response was written. const CLIENT_CLOSED_REQUEST: u16 = 499; -/// Default `base_instructions` in `GET /v1/models`. Codex uses this placeholder -/// as its system prompt. Operators can supply their chosen instructions with -/// `--codex-base-instructions-file`. -const DEFAULT_CODEX_BASE_INSTRUCTIONS: &str = "You are Codex, a coding agent."; const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt"); /// Error returned while configuring or running the server. @@ -167,7 +164,7 @@ pub struct ServerState { stats: StatsAccumulator, routing_log: Option, track_cache_eligibility: bool, - codex_base_instructions: Arc, + codex_models: Arc<[Value]>, } #[derive(Clone)] @@ -223,7 +220,7 @@ impl ServerState { stats, routing_log: None, track_cache_eligibility: tracking_enabled_from_env(), - codex_base_instructions: Arc::from(DEFAULT_CODEX_BASE_INSTRUCTIONS), + codex_models: Arc::from([]), }) } @@ -233,18 +230,76 @@ impl ServerState { Ok(self) } - /// Uses `text` as `base_instructions` for every Codex entry in `GET /v1/models`. - /// Preserves whitespace. + /// Publishes custom Codex records by rendering a Jinja system template per route. /// - /// Returns an error for blank text so Codex does not receive an empty system prompt. - pub fn with_codex_base_instructions(mut self, text: impl Into) -> ServerResult { - let text = text.into(); - if text.trim().is_empty() { + /// The context contains `model_id`, `context_window`, `tool_calling`, `reasoning`, + /// and `vision`. Undeclared capabilities are `none`. Returns an error if no routes + /// are configured, or for invalid syntax, rendering failures, or blank instructions + /// for any route. Also checks undeclared names reported by MiniJinja. + pub fn with_codex_system_template(mut self, source: &str) -> ServerResult { + let mut models = self.runner.models().collect::>(); + if models.is_empty() { return Err(ServerError::new( - "codex base instructions must not be blank", + "cannot use a Codex system template without configured routes", )); } - self.codex_base_instructions = Arc::from(text); + let mut environment = Environment::new(); + environment.set_undefined_behavior(UndefinedBehavior::Strict); + environment.set_auto_escape_callback(|_| AutoEscape::None); + environment.set_keep_trailing_newline(true); + let template = environment + .template_from_named_str("codex-system-template", source) + .map_err(|error| ServerError::new(format!("invalid Codex system template: {error}")))?; + // Check names reported by MiniJinja before rendering each route. + let mut unknown_variables = template + .undeclared_variables(false) + .into_iter() + .filter(|name| { + !matches!( + name.as_str(), + "model_id" | "context_window" | "tool_calling" | "reasoning" | "vision" + ) && !environment.globals().any(|(global, _)| global == name) + }) + .collect::>(); + if !unknown_variables.is_empty() { + unknown_variables.sort(); + return Err(ServerError::new(format!( + "unknown Codex system template variables: {}", + unknown_variables.join(", ") + ))); + } + models.sort_unstable_by_key(|model| model.id.as_str()); + let mut records = Vec::with_capacity(models.len()); + for (priority, model) in models.into_iter().enumerate() { + let capabilities = model.capabilities; + let instructions = template + .render(json!({ + "model_id": model.id.as_str(), + "context_window": capabilities.context_window, + "tool_calling": capabilities.tool_calling, + "reasoning": capabilities.reasoning, + "vision": capabilities.vision, + })) + .map_err(|error| { + ServerError::new(format!( + "cannot render Codex system template for model '{}': {error}", + model.id + )) + })?; + if instructions.trim().is_empty() { + return Err(ServerError::new(format!( + "Codex system template renders blank instructions for model '{}'", + model.id + ))); + } + records.push(codex_model_entry_json( + model.id.as_str(), + capabilities, + priority, + &instructions, + )); + } + self.codex_models = records.into(); Ok(self) } @@ -1419,7 +1474,7 @@ async fn models(State(state): State) -> Json { .runner .models() .map(|model| (model.id.as_str(), model.capabilities)), - &state.codex_base_instructions, + &state.codex_models, )) } @@ -1502,7 +1557,7 @@ async fn not_found() -> Response { fn model_list_payload<'a>( entries: impl IntoIterator, - base_instructions: &str, + codex_models: &[Value], ) -> Value { let mut entries = entries.into_iter().collect::>(); entries.sort_unstable_by_key(|(model_id, _)| *model_id); @@ -1512,13 +1567,8 @@ fn model_list_payload<'a>( json!({ "object": "list", "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), - "models": entries - .iter() - .enumerate() - .map(|(priority, (model, caps))| { - codex_model_entry_json(model, *caps, priority, base_instructions) - }) - .collect::>(), + // Keep this key even when empty: Codex rejects a response without it. + "models": codex_models, "first_id": first_id, "last_id": last_id, "has_more": false, @@ -1560,7 +1610,7 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { // facts a backend can publish; the route declares them in config today. The rest // (shell_type, apply_patch_tool_type, the reasoning-level presets, truncation_policy) are // Codex client conventions no backend returns, so they stay constant. `base_instructions` -// is the operator's chosen text or the default placeholder. Codex uses it as the system prompt. +// is rendered from the operator's explicit template. Codex uses it as the system prompt. // // TODO: source context_window, tool_calling, and reasoning from the backend, not route // config. Switchyard is a proxy, so it should re-publish what the backend advertises diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index 5e96c80ab..eff815a80 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -44,10 +44,9 @@ target = "invalid" Ok(()) } -/// Checks that `--dry-run` accepts valid UTF-8 instructions and rejects missing, -/// blank, or invalid UTF-8 files. +/// Checks template loading and rendering before the server binds a socket. #[test] -fn dry_run_validates_codex_instruction_files() -> TestResult { +fn dry_run_validates_codex_system_templates() -> TestResult { let directory = tempfile::tempdir()?; let config = directory.path().join("routes.toml"); fs::write( @@ -64,16 +63,46 @@ llm_client = "local" id = "test-route" type = "passthrough" target = "local" +[routes.second] +id = "second-route" +type = "passthrough" +target = "local" "#, )?; - let prompt = directory.path().join("instructions.txt"); - for (contents, expected_success) in [ - (None, false), - (Some(b" \n\t".as_slice()), false), - (Some(b"\xff".as_slice()), false), + let prompt = directory.path().join("system.jinja"); + for (contents, expected_error) in [ + (None, Some("invalid --codex-system-template")), + ( + Some(b" \n\t".as_slice()), + Some("renders blank instructions"), + ), + ( + Some(b"\xff".as_slice()), + Some("invalid --codex-system-template"), + ), + ( + Some(b"{{ model_id".as_slice()), + Some("invalid Codex system template"), + ), + ( + Some(b"{{ model_id | nonexistent }}".as_slice()), + Some("cannot render Codex system template"), + ), + ( + Some(b"{% if typo is true %}tools{% endif %}Custom".as_slice()), + Some("unknown Codex system template variables: typo"), + ), + ( + Some(b"{% if false %}{{ personality }}{% endif %}Custom".as_slice()), + Some("unknown Codex system template variables: personality"), + ), ( - Some(b"Configured instructions.\n Keep whitespace. \n".as_slice()), - true, + Some(b"{% if model_id == 'second-route' %}Custom{% endif %}".as_slice()), + Some("renders blank instructions for model 'test-route'"), + ), + ( + Some(b"Custom instructions for {{ model_id }}.\n Keep whitespace. \n".as_slice()), + None, ), ] { if let Some(contents) = contents { @@ -83,13 +112,45 @@ target = "local" .arg("--config") .arg(&config) .arg("--dry-run") - .arg("--codex-base-instructions-file") + .arg("--codex-system-template") .arg(&prompt) .output()?; - assert_eq!(output.status.success(), expected_success); - if !expected_success { - assert!(String::from_utf8(output.stderr)?.contains("codex")); + let stderr = String::from_utf8(output.stderr)?; + assert_eq!( + output.status.success(), + expected_error.is_none(), + "{stderr}" + ); + if let Some(expected_error) = expected_error { + assert!(stderr.contains(expected_error), "{stderr}"); } } Ok(()) } + +#[test] +fn dry_run_requires_routes_for_codex_system_template() -> TestResult { + let directory = tempfile::tempdir()?; + let config = directory.path().join("routes.toml"); + fs::write(&config, "schema_version = 1\ntargets = {}\nroutes = {}\n")?; + let prompt = directory.path().join("system.jinja"); + fs::write(&prompt, "Custom instructions for {{ model_id }}.\n")?; + let mut command = Command::new(env!("CARGO_BIN_EXE_switchyard-server")); + command.arg("--config").arg(&config).arg("--dry-run"); + let output = command.output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let output = command + .arg("--codex-system-template") + .arg(&prompt) + .output()?; + assert!(!output.status.success()); + assert!( + String::from_utf8(output.stderr)? + .contains("cannot use a Codex system template without configured routes") + ); + Ok(()) +} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 67f822c24..183c9db4f 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2664,14 +2664,47 @@ target = "shared" assert_eq!(capabilities["undeclared"]["context_window"], json!(null)); assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null)); + assert_eq!(body["models"], json!([])); + + let template = "Route {{ model_id }} <&>\n\ + {% if context_window is not none %}Window {{ context_window }}. {% endif %}\ + {% if tool_calling is true %}Tools. {% elif tool_calling is false %}No tools. {% endif %}\ + {% if reasoning is true %}Reasoning. {% endif %}\ + {% if vision is none %}Vision undeclared.{% endif %} \n"; + let state = load_test_config(CONFIG)?.with_codex_system_template(template)?; + let body = send( + &build_switchyard_router(state), + "GET", + "/v1/models?client_version=0.152.0", + None, + ) + .await? + .json()?; + assert_eq!(body["data"], json!(data)); + let codex_models = body["models"].as_array().cloned().unwrap_or_default(); let codex_metadata = codex_models .iter() .filter_map(|entry| entry["slug"].as_str().map(|slug| (slug, entry))) .collect::>(); - // This checks the shape the server emits. That Codex 0.144.5 actually decodes it - // (context_window: null included) is verified by a live Codex run in SWITCH-1225. assert_eq!(codex_metadata.len(), 4); + for (id, expected) in [ + ( + "declared", + "Route declared <&>\nWindow 1000000. Tools. Vision undeclared. \n", + ), + ( + "reasoning", + "Route reasoning <&>\nReasoning. Vision undeclared. \n", + ), + ( + "restricted", + "Route restricted <&>\nWindow 262000. No tools. Vision undeclared. \n", + ), + ("undeclared", "Route undeclared <&>\nVision undeclared. \n"), + ] { + assert_eq!(codex_metadata[id]["base_instructions"], expected); + } assert_eq!( codex_metadata["declared"]["context_window"], json!(1_000_000) @@ -2975,58 +3008,6 @@ subagents = {{ type = "passthrough", target = "strong" }} Ok(()) } -#[tokio::test] -async fn codex_catalog_serves_configured_base_instructions_verbatim() -> TestResult { - const CONFIG: &str = r#" -schema_version = 1 - -[llm_clients.primary] -format = "openai_chat" -base_url = "https://example.test/v1" - -[targets.shared] -id = "nvidia/deepseek-ai/deepseek-v4-pro" -llm_client = "primary" - -[routes.a] -id = "route/a" -type = "passthrough" -target = "shared" - -[routes.b] -id = "route/b" -type = "passthrough" -target = "shared" -"#; - let codex_instructions = |state: ServerState| async move { - let body = send(&build_switchyard_router(state), "GET", "/v1/models", None) - .await? - .json()?; - TestResult::Ok( - body["models"] - .as_array() - .cloned() - .unwrap_or_default() - .into_iter() - .map(|entry| entry["base_instructions"].clone()) - .collect::>(), - ) - }; - - // Without a configured prompt, every route uses the default placeholder. - assert_eq!( - codex_instructions(load_test_config(CONFIG)?).await?, - vec![json!("You are Codex, a coding agent."); 2] - ); - - // Preserve line breaks and trailing whitespace in the configured prompt. - let prompt = "You are Codex, an agent based on GPT-5.\n\n # Tools\n\n- shell \n"; - let state = load_test_config(CONFIG)?.with_codex_base_instructions(prompt)?; - assert_eq!(codex_instructions(state).await?, vec![json!(prompt); 2]); - - Ok(()) -} - #[tokio::test] async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; @@ -4451,7 +4432,10 @@ id = "blind" type = "passthrough" target = "shared" "#; - let app = build_switchyard_router(load_test_config(CONFIG)?); + let app = build_switchyard_router( + load_test_config(CONFIG)? + .with_codex_system_template("Custom instructions for {{ model_id }}.")?, + ); let models = send(&app, "GET", "/v1/models", None).await?; assert_eq!(models.status, StatusCode::OK); let body = models.json()?; diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 3d05d8ba8..ee8ebcc92 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,7 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | -| `--codex-base-instructions-file PATH` | None | Read a UTF-8 file once at startup and use its text as the Codex system prompt for every route in `GET /v1/models`. Without this option, Codex gets a one-line placeholder. See [Codex model discovery](../crates/switchyard-server/README.md#codex-model-discovery) for prompt export and restart instructions. | +| `--codex-system-template PATH` | None | Publish custom Codex model records from a UTF-8 Jinja system template, rendered once per route at startup. Without this option, Codex keeps its bundled catalog and instructions. See [Codex model discovery](../crates/switchyard-server/README.md#codex-model-discovery) for an example and the available variables. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | From 5f2c0d79f8c534216c46b94cf3c8e24713985b2f Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 15 Sep 2026 11:51:03 -0700 Subject: [PATCH 5/6] chore(server): shorten Codex template tests Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/tests/cli.rs | 90 +++++++----------------- crates/switchyard-server/tests/server.rs | 23 ++---- 2 files changed, 32 insertions(+), 81 deletions(-) diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index eff815a80..b23bc1151 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -44,7 +44,6 @@ target = "invalid" Ok(()) } -/// Checks template loading and rendering before the server binds a socket. #[test] fn dry_run_validates_codex_system_templates() -> TestResult { let directory = tempfile::tempdir()?; @@ -53,77 +52,42 @@ fn dry_run_validates_codex_system_templates() -> TestResult { &config, r#" schema_version = 1 -[llm_clients.local] -format = "openai_chat" -base_url = "http://127.0.0.1:1/v1" -[targets.local] -id = "upstream-model" -llm_client = "local" -[routes.local] -id = "test-route" -type = "passthrough" -target = "local" -[routes.second] -id = "second-route" -type = "passthrough" -target = "local" +[llm_clients] +local = { format = "openai_chat", base_url = "http://127.0.0.1:1/v1" } +[targets] +local = { id = "upstream-model", llm_client = "local" } +[routes] +a = { id = "a", type = "passthrough", target = "local" } +b = { id = "b", type = "passthrough", target = "local" } "#, )?; let prompt = directory.path().join("system.jinja"); - for (contents, expected_error) in [ - (None, Some("invalid --codex-system-template")), - ( - Some(b" \n\t".as_slice()), - Some("renders blank instructions"), - ), - ( - Some(b"\xff".as_slice()), - Some("invalid --codex-system-template"), - ), - ( - Some(b"{{ model_id".as_slice()), - Some("invalid Codex system template"), - ), - ( - Some(b"{{ model_id | nonexistent }}".as_slice()), - Some("cannot render Codex system template"), - ), - ( - Some(b"{% if typo is true %}tools{% endif %}Custom".as_slice()), - Some("unknown Codex system template variables: typo"), - ), - ( - Some(b"{% if false %}{{ personality }}{% endif %}Custom".as_slice()), - Some("unknown Codex system template variables: personality"), - ), - ( - Some(b"{% if model_id == 'second-route' %}Custom{% endif %}".as_slice()), - Some("renders blank instructions for model 'test-route'"), - ), - ( - Some(b"Custom instructions for {{ model_id }}.\n Keep whitespace. \n".as_slice()), - None, - ), - ] { + let mut command = Command::new(env!("CARGO_BIN_EXE_switchyard-server")); + command + .arg("--config") + .arg(&config) + .args(["--dry-run", "--codex-system-template"]) + .arg(&prompt); + let cases: &[(Option<&[u8]>, bool)] = &[ + (None, false), + (Some(b"\xff"), false), + (Some(b"{{ model_id"), false), + (Some(b"{{ model_id | nonexistent }}"), false), + (Some(b"{% if typo is true %}x{% endif %}Custom"), false), + (Some(b" {% if model_id == 'a' %}Custom{% endif %}\n"), false), + (Some(b"Custom {{ model_id }}"), true), + ]; + for &(contents, expected_success) in cases { if let Some(contents) = contents { fs::write(&prompt, contents)?; } - let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) - .arg("--config") - .arg(&config) - .arg("--dry-run") - .arg("--codex-system-template") - .arg(&prompt) - .output()?; - let stderr = String::from_utf8(output.stderr)?; + let output = command.output()?; assert_eq!( output.status.success(), - expected_error.is_none(), - "{stderr}" + expected_success, + "{contents:?}: {}", + String::from_utf8_lossy(&output.stderr) ); - if let Some(expected_error) = expected_error { - assert!(stderr.contains(expected_error), "{stderr}"); - } } Ok(()) } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 183c9db4f..6e6eed14a 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2666,11 +2666,7 @@ target = "shared" assert_eq!(body["models"], json!([])); - let template = "Route {{ model_id }} <&>\n\ - {% if context_window is not none %}Window {{ context_window }}. {% endif %}\ - {% if tool_calling is true %}Tools. {% elif tool_calling is false %}No tools. {% endif %}\ - {% if reasoning is true %}Reasoning. {% endif %}\ - {% if vision is none %}Vision undeclared.{% endif %} \n"; + let template = "{{ model_id }} <&> {{ context_window }} {{ tool_calling }} {{ reasoning }} {{ vision }} \n"; let state = load_test_config(CONFIG)?.with_codex_system_template(template)?; let body = send( &build_switchyard_router(state), @@ -2689,19 +2685,10 @@ target = "shared" .collect::>(); assert_eq!(codex_metadata.len(), 4); for (id, expected) in [ - ( - "declared", - "Route declared <&>\nWindow 1000000. Tools. Vision undeclared. \n", - ), - ( - "reasoning", - "Route reasoning <&>\nReasoning. Vision undeclared. \n", - ), - ( - "restricted", - "Route restricted <&>\nWindow 262000. No tools. Vision undeclared. \n", - ), - ("undeclared", "Route undeclared <&>\nVision undeclared. \n"), + ("declared", "declared <&> 1000000 True None None \n"), + ("reasoning", "reasoning <&> None None True None \n"), + ("restricted", "restricted <&> 262000 False None None \n"), + ("undeclared", "undeclared <&> None None None None \n"), ] { assert_eq!(codex_metadata[id]["base_instructions"], expected); } From 8c2fcc478511797f1fdc48f16ec6194d95e0df90 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 15 Sep 2026 12:27:14 -0700 Subject: [PATCH 6/6] fix(server): preserve Codex instructions during discovery Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 17 -- crates/switchyard-runner/src/route.rs | 20 +-- crates/switchyard-server/Cargo.toml | 1 - crates/switchyard-server/README.md | 64 ++------ crates/switchyard-server/src/cli.rs | 15 -- crates/switchyard-server/src/lib.rs | 163 +------------------- crates/switchyard-server/tests/cli.rs | 75 --------- crates/switchyard-server/tests/server.rs | 188 +---------------------- docs/cli_reference.md | 1 - docs/core_concepts.md | 7 +- docs/reference/toml_schema.md | 4 +- 11 files changed, 27 insertions(+), 528 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfb535534..b353d0657 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,12 +1216,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "memo-map" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5449c8c750f1a07ea702bbd212bd999fceece9b3d1508b17023b3e174583124b" - [[package]] name = "micromap" version = "0.3.0" @@ -1234,16 +1228,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minijinja" -version = "2.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86886cf6dbf4e614b19c9a1eec9775f021869d7eadde0fc73921a81b90c9b4c9" -dependencies = [ - "memo-map", - "serde", -] - [[package]] name = "mio" version = "1.2.2" @@ -2467,7 +2451,6 @@ dependencies = [ "http", "http-body-util", "humantime", - "minijinja", "opentelemetry", "opentelemetry-otlp", "opentelemetry-prometheus", diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 2b594646f..e3640c5f4 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -14,27 +14,17 @@ use thiserror::Error; use crate::DecisionTarget; -/// Capabilities that one route advertises on `GET /v1/models`. +/// Capabilities declared for one route. /// -/// An unset capability is undeclared: it serializes as `null` in the OpenAI -/// `data` entry, and the Codex entry falls back to a safe default for it. +/// `GET /v1/models` includes `context_window`, `tool_calling`, and `vision` in each +/// standard `data` entry, using `null` for unset values. `reasoning` remains route metadata. #[derive(Clone, Copy, Default)] pub struct ModelCapabilities { pub context_window: Option, pub tool_calling: Option, - /// Whether the routed model takes reasoning controls. A serving surface cannot - /// probe this, so a route opts in via config; undeclared routes advertise as - /// non-reasoning to Codex (fail closed). + /// Whether the routed model accepts reasoning controls, as declared in config. pub reasoning: Option, - /// Whether the routed model accepts image input. Declared per route for the same - /// reason as `reasoning`, and failing closed matters more here: a route may - /// resolve to a target with no vision at all. - /// - /// This is not cosmetic metadata. Codex reads `input_modalities` from the model - /// card and, when it reads text-only, replaces an attached image with the literal - /// text `image content omitted because you do not support image input` *before - /// sending*. An undeclared vision-capable route therefore loses the image in the - /// client, and the proxy never receives one to forward. + /// Whether the routed model accepts image input, as declared in config. pub vision: Option, } diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index a0a5df9e7..5c25b5805 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -29,7 +29,6 @@ clap = { version = "4", features = ["derive", "env"] } futures-util.workspace = true http.workspace = true libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" } -minijinja = { version = "2.24", default-features = false, features = ["builtins", "serde"] } opentelemetry = { version = "0.32", default-features = false, features = ["metrics", "trace"] } opentelemetry-otlp = { version = "0.32", default-features = false, features = ["http-proto", "metrics", "reqwest-blocking-client", "reqwest-rustls", "trace"] } opentelemetry-prometheus = "0.32" diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 6fedcb6c5..c700c32ce 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -144,61 +144,15 @@ handoff notes, per-tier system prompts, and a capability-judge fallback are docu ## Codex model discovery -By default, `GET /v1/models` returns the standard `data` list and an empty Codex -`models` list. Codex keeps its bundled model catalog and instructions. Switchyard route -aliases remain usable through explicit model selection, such as `codex --model route-id`, -but do not appear automatically in Codex's model picker. For names it does not recognize, -Codex uses its own generic defaults; it does not receive Switchyard's custom context limits -or tool settings. - -**Custom Codex model records require an explicit system template.** Set -`--codex-system-template PATH` to publish a record for each route. Switchyard renders the -UTF-8 template once per route at startup and sends the result as `base_instructions`. -This is the complete replacement prompt for those records, including their tool-use -instructions. It does not inherit or append Codex's bundled prompt. - -Save this minimal example as `codex-system.jinja` and adapt it for your models: - -```jinja -You are a coding assistant working through route {{ model_id }}. -Read the relevant code before editing. Verify changes with focused checks. -{% if tool_calling is true %} -Use the available tools to inspect files, make changes, and run checks. -{% endif %} -{% if context_window is not none %} -The route advertises a context window of {{ context_window }} tokens. -{% endif %} -``` - -```bash -switchyard-server --config routes.toml --codex-system-template codex-system.jinja -``` - -Switchyard renders [MiniJinja syntax](https://docs.rs/minijinja/2.24.0/minijinja/syntax/index.html), -including expressions, conditionals, loops, and built-in filters. These variables describe -the configured route: - -| Variable | Value | -|---|---| -| `model_id` | Public route ID from `/v1/models`, not the backend model selected later. | -| `context_window` | Declared context limit in tokens, or `none`. | -| `tool_calling` | Declared tool support: `true`, `false`, or `none`. | -| `reasoning` | Declared reasoning support: `true`, `false`, or `none`. | -| `vision` | Declared image support: `true`, `false`, or `none`. | - -Use `is true`, `is false`, or `is none` to distinguish supported, unsupported, and undeclared -capabilities. The template has no access to Codex personality settings, session data, -conversation messages, or environment variables. File includes and template imports are -not supported. - -Startup rejects unreadable or invalid UTF-8 files, invalid syntax, rendering errors, and -instructions that render as only whitespace for any route. It also checks undeclared names -reported by MiniJinja. Using a template requires at least one configured route; without a -template, an empty route configuration is valid. - -Rendered text preserves whitespace and does not use HTML escaping. Restart the server after -editing the template. Existing Codex sessions may retain earlier instructions; use a fresh -session when checking a changed template. +`GET /v1/models` returns the standard `data` list and an empty Codex `models` list. +Codex keeps its own model catalog and instructions. Select a Switchyard route explicitly +with `codex --model route-id`; route aliases do not appear automatically in Codex's model +picker. Unknown aliases use Codex's generic defaults and do not receive Switchyard's +route-specific context limits or tool settings. + +To add instructions for a target, set `system_prompt` on its `[targets.]` entry. +Switchyard prepends that text when the selected target serves a completion and retains +the caller's instructions. Omit the setting to add no target instructions. ## Endpoints diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index cff0343e6..231d4e2bc 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,12 +52,6 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, - /// Publish custom Codex model records using this UTF-8 Jinja system template. - /// Render once per route at startup. Without this option, Codex keeps its - /// bundled model catalog and instructions. - #[arg(long, value_name = "PATH")] - codex_system_template: Option, - /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] tls_cert: Option, @@ -78,15 +72,6 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } - if let Some(path) = self.codex_system_template { - let text = std::fs::read_to_string(&path).map_err(|error| { - ServerError::new(format!( - "invalid --codex-system-template {}: {error}", - path.display() - )) - })?; - state = state.with_codex_system_template(&text)?; - } let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { if !cert.exists() || !key.exists() { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index d9273ad21..6ecb9fdda 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -34,7 +34,6 @@ use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; use libsy::{LibsyError, RoutingOutcome}; -use minijinja::{AutoEscape, Environment, UndefinedBehavior}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -88,7 +87,6 @@ fn should_forward_upstream_header(name: &HeaderName) -> bool { /// Non-standard status used only in logs and metrics for a request whose /// downstream client disconnected before any response was written. const CLIENT_CLOSED_REQUEST: u16 = 499; - const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt"); /// Error returned while configuring or running the server. @@ -164,7 +162,6 @@ pub struct ServerState { stats: StatsAccumulator, routing_log: Option, track_cache_eligibility: bool, - codex_models: Arc<[Value]>, } #[derive(Clone)] @@ -220,7 +217,6 @@ impl ServerState { stats, routing_log: None, track_cache_eligibility: tracking_enabled_from_env(), - codex_models: Arc::from([]), }) } @@ -230,79 +226,6 @@ impl ServerState { Ok(self) } - /// Publishes custom Codex records by rendering a Jinja system template per route. - /// - /// The context contains `model_id`, `context_window`, `tool_calling`, `reasoning`, - /// and `vision`. Undeclared capabilities are `none`. Returns an error if no routes - /// are configured, or for invalid syntax, rendering failures, or blank instructions - /// for any route. Also checks undeclared names reported by MiniJinja. - pub fn with_codex_system_template(mut self, source: &str) -> ServerResult { - let mut models = self.runner.models().collect::>(); - if models.is_empty() { - return Err(ServerError::new( - "cannot use a Codex system template without configured routes", - )); - } - let mut environment = Environment::new(); - environment.set_undefined_behavior(UndefinedBehavior::Strict); - environment.set_auto_escape_callback(|_| AutoEscape::None); - environment.set_keep_trailing_newline(true); - let template = environment - .template_from_named_str("codex-system-template", source) - .map_err(|error| ServerError::new(format!("invalid Codex system template: {error}")))?; - // Check names reported by MiniJinja before rendering each route. - let mut unknown_variables = template - .undeclared_variables(false) - .into_iter() - .filter(|name| { - !matches!( - name.as_str(), - "model_id" | "context_window" | "tool_calling" | "reasoning" | "vision" - ) && !environment.globals().any(|(global, _)| global == name) - }) - .collect::>(); - if !unknown_variables.is_empty() { - unknown_variables.sort(); - return Err(ServerError::new(format!( - "unknown Codex system template variables: {}", - unknown_variables.join(", ") - ))); - } - models.sort_unstable_by_key(|model| model.id.as_str()); - let mut records = Vec::with_capacity(models.len()); - for (priority, model) in models.into_iter().enumerate() { - let capabilities = model.capabilities; - let instructions = template - .render(json!({ - "model_id": model.id.as_str(), - "context_window": capabilities.context_window, - "tool_calling": capabilities.tool_calling, - "reasoning": capabilities.reasoning, - "vision": capabilities.vision, - })) - .map_err(|error| { - ServerError::new(format!( - "cannot render Codex system template for model '{}': {error}", - model.id - )) - })?; - if instructions.trim().is_empty() { - return Err(ServerError::new(format!( - "Codex system template renders blank instructions for model '{}'", - model.id - ))); - } - records.push(codex_model_entry_json( - model.id.as_str(), - capabilities, - priority, - &instructions, - )); - } - self.codex_models = records.into(); - Ok(self) - } - /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.runner.models().map(|model| model.id.as_str()) @@ -1474,7 +1397,6 @@ async fn models(State(state): State) -> Json { .runner .models() .map(|model| (model.id.as_str(), model.capabilities)), - &state.codex_models, )) } @@ -1557,7 +1479,6 @@ async fn not_found() -> Response { fn model_list_payload<'a>( entries: impl IntoIterator, - codex_models: &[Value], ) -> Value { let mut entries = entries.into_iter().collect::>(); entries.sort_unstable_by_key(|(model_id, _)| *model_id); @@ -1567,8 +1488,8 @@ fn model_list_payload<'a>( json!({ "object": "list", "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::>(), - // Keep this key even when empty: Codex rejects a response without it. - "models": codex_models, + // Codex requires this key; an empty list preserves its own catalog and instructions. + "models": [], "first_id": first_id, "last_id": last_id, "has_more": false, @@ -1599,86 +1520,6 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value { }) } -// Builds the metadata Codex requires when it discovers models from a direct provider. -// -// This mirrors Codex's `ModelInfo` card. The benchmark harness builds the same card in -// `benchmark/codex_model_catalog_lib.py`; keep the two in sync when Codex changes -// the shape. Every field below is either derived from the route's declared capabilities or a -// required `ModelInfo` field the server has no better value for. -// -// Two kinds of fields live here. context_window, tool_calling, and reasoning are model -// facts a backend can publish; the route declares them in config today. The rest -// (shell_type, apply_patch_tool_type, the reasoning-level presets, truncation_policy) are -// Codex client conventions no backend returns, so they stay constant. `base_instructions` -// is rendered from the operator's explicit template. Codex uses it as the system prompt. -// -// TODO: source context_window, tool_calling, and reasoning from the backend, not route -// config. Switchyard is a proxy, so it should re-publish what the backend advertises -// when it can — OpenRouter's /api/v1/models exposes context_length and -// supported_parameters — and fall back to the route's declared value. Some backends -// publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info), -// so keep failing closed to config. -fn codex_model_entry_json( - model: &str, - capabilities: ModelCapabilities, - priority: usize, - base_instructions: &str, -) -> Value { - // Codex is non-functional without shell and apply_patch, so an undeclared tool - // capability defaults to enabled here; the OpenAI `data` entry reports the raw - // Option separately for clients that want the undeclared state. - let tool_calling = capabilities.tool_calling.unwrap_or(true); - let reasoning = capabilities.reasoning.unwrap_or(false); - json!({ - "slug": model, - "display_name": model, - "description": "Switchyard-routed model.", - "default_reasoning_level": if reasoning { json!("xhigh") } else { Value::Null }, - "supported_reasoning_levels": if reasoning { reasoning_levels() } else { json!([]) }, - "shell_type": if tool_calling { "shell_command" } else { "disabled" }, - "visibility": "list", - "supported_in_api": true, - // Catalog list position (routes are listed in sorted id order), not a quality rank. - "priority": priority, - "additional_speed_tiers": [], - "availability_nux": null, - "upgrade": null, - "base_instructions": base_instructions, - "supports_reasoning_summaries": reasoning, - "default_reasoning_summary": "none", - "support_verbosity": reasoning, - "default_verbosity": if reasoning { json!("low") } else { Value::Null }, - "apply_patch_tool_type": if tool_calling { Some("freeform") } else { None }, - "web_search_tool_type": "text", - "truncation_policy": {"mode": "tokens", "limit": 10_000}, - "supports_parallel_tool_calls": tool_calling, - "supports_image_detail_original": false, - "context_window": capabilities.context_window, - "max_context_window": capabilities.context_window, - "effective_context_window_percent": 95, - "experimental_supported_tools": [], - // Codex omits an attached image client-side when this says text-only, so a - // route whose target can see must declare `vision = true`. Fails closed. - "input_modalities": if capabilities.vision.unwrap_or(false) { - json!(["text", "image"]) - } else { - json!(["text"]) - }, - "supports_search_tool": false, - }) -} - -// The reasoning-effort presets Codex offers for a reasoning-capable route. Kept in -// step with the benchmark template in `codex_model_catalog_lib.py`. -fn reasoning_levels() -> Value { - json!([ - {"effort": "low", "description": "Fast responses with lighter reasoning"}, - {"effort": "medium", "description": "Balances speed and reasoning depth"}, - {"effort": "high", "description": "Greater reasoning depth"}, - {"effort": "xhigh", "description": "Extra high reasoning depth"}, - ]) -} - fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String { let scheme = if options.is_tls() { "https" } else { "http" }; let listen_url = url_for_addr(scheme, options.addr); diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index b23bc1151..f72c45858 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -43,78 +43,3 @@ target = "invalid" ); Ok(()) } - -#[test] -fn dry_run_validates_codex_system_templates() -> TestResult { - let directory = tempfile::tempdir()?; - let config = directory.path().join("routes.toml"); - fs::write( - &config, - r#" -schema_version = 1 -[llm_clients] -local = { format = "openai_chat", base_url = "http://127.0.0.1:1/v1" } -[targets] -local = { id = "upstream-model", llm_client = "local" } -[routes] -a = { id = "a", type = "passthrough", target = "local" } -b = { id = "b", type = "passthrough", target = "local" } -"#, - )?; - let prompt = directory.path().join("system.jinja"); - let mut command = Command::new(env!("CARGO_BIN_EXE_switchyard-server")); - command - .arg("--config") - .arg(&config) - .args(["--dry-run", "--codex-system-template"]) - .arg(&prompt); - let cases: &[(Option<&[u8]>, bool)] = &[ - (None, false), - (Some(b"\xff"), false), - (Some(b"{{ model_id"), false), - (Some(b"{{ model_id | nonexistent }}"), false), - (Some(b"{% if typo is true %}x{% endif %}Custom"), false), - (Some(b" {% if model_id == 'a' %}Custom{% endif %}\n"), false), - (Some(b"Custom {{ model_id }}"), true), - ]; - for &(contents, expected_success) in cases { - if let Some(contents) = contents { - fs::write(&prompt, contents)?; - } - let output = command.output()?; - assert_eq!( - output.status.success(), - expected_success, - "{contents:?}: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(()) -} - -#[test] -fn dry_run_requires_routes_for_codex_system_template() -> TestResult { - let directory = tempfile::tempdir()?; - let config = directory.path().join("routes.toml"); - fs::write(&config, "schema_version = 1\ntargets = {}\nroutes = {}\n")?; - let prompt = directory.path().join("system.jinja"); - fs::write(&prompt, "Custom instructions for {{ model_id }}.\n")?; - let mut command = Command::new(env!("CARGO_BIN_EXE_switchyard-server")); - command.arg("--config").arg(&config).arg("--dry-run"); - let output = command.output()?; - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - let output = command - .arg("--codex-system-template") - .arg(&prompt) - .output()?; - assert!(!output.status.success()); - assert!( - String::from_utf8(output.stderr)? - .contains("cannot use a Codex system template without configured routes") - ); - Ok(()) -} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 6e6eed14a..0734445d5 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -2628,6 +2628,7 @@ type = "passthrough" target = "shared" context_window = 1000000 tool_calling = true +vision = true [routes.restricted] id = "restricted" @@ -2635,12 +2636,7 @@ type = "passthrough" target = "shared" context_window = 262000 tool_calling = false - -[routes.reasoning] -id = "reasoning" -type = "passthrough" -target = "shared" -reasoning = true +vision = false [routes.undeclared] id = "undeclared" @@ -2648,7 +2644,7 @@ type = "passthrough" target = "shared" "#; let app = build_switchyard_router(load_test_config(CONFIG)?); - let models = send(&app, "GET", "/v1/models", None).await?; + let models = send(&app, "GET", "/v1/models?client_version=0.152.0", None).await?; assert_eq!(models.status, StatusCode::OK); let body = models.json()?; let data = body["data"].as_array().cloned().unwrap_or_default(); @@ -2664,108 +2660,11 @@ target = "shared" assert_eq!(capabilities["undeclared"]["context_window"], json!(null)); assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null)); + assert_eq!(capabilities["declared"]["vision"], json!(true)); + assert_eq!(capabilities["restricted"]["vision"], json!(false)); + assert_eq!(capabilities["undeclared"]["vision"], json!(null)); assert_eq!(body["models"], json!([])); - let template = "{{ model_id }} <&> {{ context_window }} {{ tool_calling }} {{ reasoning }} {{ vision }} \n"; - let state = load_test_config(CONFIG)?.with_codex_system_template(template)?; - let body = send( - &build_switchyard_router(state), - "GET", - "/v1/models?client_version=0.152.0", - None, - ) - .await? - .json()?; - assert_eq!(body["data"], json!(data)); - - let codex_models = body["models"].as_array().cloned().unwrap_or_default(); - let codex_metadata = codex_models - .iter() - .filter_map(|entry| entry["slug"].as_str().map(|slug| (slug, entry))) - .collect::>(); - assert_eq!(codex_metadata.len(), 4); - for (id, expected) in [ - ("declared", "declared <&> 1000000 True None None \n"), - ("reasoning", "reasoning <&> None None True None \n"), - ("restricted", "restricted <&> 262000 False None None \n"), - ("undeclared", "undeclared <&> None None None None \n"), - ] { - assert_eq!(codex_metadata[id]["base_instructions"], expected); - } - assert_eq!( - codex_metadata["declared"]["context_window"], - json!(1_000_000) - ); - assert_eq!(codex_metadata["declared"]["shell_type"], "shell_command"); - assert_eq!( - codex_metadata["declared"]["apply_patch_tool_type"], - "freeform" - ); - // Constant fields Codex requires: a typo here would fail its decode, so pin them. - assert_eq!(codex_metadata["declared"]["visibility"], "list"); - assert_eq!(codex_metadata["declared"]["supported_in_api"], json!(true)); - assert_eq!(codex_metadata["declared"]["web_search_tool_type"], "text"); - assert_eq!( - codex_metadata["declared"]["input_modalities"], - json!(["text"]) - ); - assert_eq!( - codex_metadata["declared"]["truncation_policy"], - json!({"mode": "tokens", "limit": 10_000}) - ); - assert_eq!( - codex_metadata["restricted"]["context_window"], - json!(262_000) - ); - assert_eq!(codex_metadata["restricted"]["shell_type"], "disabled"); - assert_eq!( - codex_metadata["restricted"]["apply_patch_tool_type"], - json!(null) - ); - // A reasoning route advertises the effort presets and reasoning controls. - assert_eq!( - codex_metadata["reasoning"]["default_reasoning_level"], - "xhigh" - ); - assert_eq!( - codex_metadata["reasoning"]["supported_reasoning_levels"] - .as_array() - .map(Vec::len), - Some(4) - ); - assert_eq!( - codex_metadata["reasoning"]["supports_reasoning_summaries"], - json!(true) - ); - assert_eq!( - codex_metadata["reasoning"]["support_verbosity"], - json!(true) - ); - assert_eq!(codex_metadata["reasoning"]["default_verbosity"], "low"); - // An undeclared route: null context window, non-reasoning, but tools default on so Codex - // remains usable when connected directly to the server. - assert_eq!(codex_metadata["undeclared"]["context_window"], json!(null)); - assert_eq!( - codex_metadata["undeclared"]["supported_reasoning_levels"], - json!([]) - ); - assert_eq!( - codex_metadata["undeclared"]["default_reasoning_level"], - json!(null) - ); - assert_eq!( - codex_metadata["undeclared"]["supports_reasoning_summaries"], - json!(false) - ); - assert_eq!(codex_metadata["undeclared"]["shell_type"], "shell_command"); - assert_eq!( - codex_metadata["undeclared"]["apply_patch_tool_type"], - "freeform" - ); - assert_eq!( - codex_metadata["undeclared"]["supports_parallel_tool_calls"], - json!(true) - ); Ok(()) } @@ -4386,78 +4285,3 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { ); Ok(()) } - -// Verifies a route declaring `vision = true` advertises image input, and that an -// undeclared route still fails closed to text-only. -// -// This is not cosmetic metadata. Codex reads `input_modalities` from the model card -// and, when it reads text-only, replaces an attached image with the literal text -// "image content omitted because you do not support image input" before sending — so -// a route whose target can see but which does not say so loses the image in the -// client, and Switchyard never receives one to forward. -#[tokio::test] -async fn models_endpoint_advertises_image_input_only_for_vision_routes() -> TestResult { - const CONFIG: &str = r#" -schema_version = 1 - -[llm_clients.shared] -format = "openai_responses" -base_url = "http://127.0.0.1:1/v1" - -[targets.shared] -id = "shared-model" -llm_client = "shared" - -[routes.sees] -id = "sees" -type = "passthrough" -target = "shared" -vision = true - -[routes.blind] -id = "blind" -type = "passthrough" -target = "shared" -"#; - let app = build_switchyard_router( - load_test_config(CONFIG)? - .with_codex_system_template("Custom instructions for {{ model_id }}.")?, - ); - let models = send(&app, "GET", "/v1/models", None).await?; - assert_eq!(models.status, StatusCode::OK); - let body = models.json()?; - - let codex_metadata = body["models"] - .as_array() - .cloned() - .unwrap_or_default() - .iter() - .filter_map(|entry| { - entry["slug"] - .as_str() - .map(|slug| (slug.to_string(), entry.clone())) - }) - .collect::>(); - assert_eq!( - codex_metadata["sees"]["input_modalities"], - json!(["text", "image"]) - ); - assert_eq!(codex_metadata["blind"]["input_modalities"], json!(["text"])); - - // The OpenAI `data` entry reports the raw Option, so an undeclared route stays - // distinguishable from one that declared `false`. - let capabilities = body["data"] - .as_array() - .cloned() - .unwrap_or_default() - .iter() - .filter_map(|entry| { - entry["id"] - .as_str() - .map(|id| (id.to_string(), entry["capabilities"].clone())) - }) - .collect::>(); - assert_eq!(capabilities["sees"]["vision"], json!(true)); - assert_eq!(capabilities["blind"]["vision"], json!(null)); - Ok(()) -} diff --git a/docs/cli_reference.md b/docs/cli_reference.md index ee8ebcc92..e5da32455 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,7 +20,6 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | -| `--codex-system-template PATH` | None | Publish custom Codex model records from a UTF-8 Jinja system template, rendered once per route at startup. Without this option, Codex keeps its bundled catalog and instructions. See [Codex model discovery](../crates/switchyard-server/README.md#codex-model-discovery) for an example and the available variables. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 87a3eeb4e..9688a501d 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -62,10 +62,9 @@ inside the TOML file. Their `id` fields have different external meanings: The server lists route IDs on `GET /v1/models`. A request selects a route by putting that ID in its `model` field. The native Rust server does not discover -or register additional provider models automatically. The same response also -carries a Codex-compatible `models` array so Codex can use the server as a direct -provider; each entry reflects the route's declared context window, tool support, -and reasoning. +or register additional provider models automatically. The response includes an empty +Codex `models` array so Codex keeps its own catalog and instructions. Select route +aliases explicitly; they do not appear automatically in Codex's model picker. ## Routing Algorithms diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index b482ecb6e..c820d21d1 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -115,8 +115,8 @@ Every route takes the common keys below, plus the keys for its type. | `type` | Yes | — | Routing algorithm for this route. | | `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. | | `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. | -| `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. | -| `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. | +| `reasoning` | No | unset | Declared reasoning support, stored in route metadata. The server does not include it in `GET /v1/models`. | +| `vision` | No | unset | Image-input support advertised in `GET /v1/models` under `data[].capabilities.vision`. Unset values appear as `null`. Declare `true` only when every target the route can select accepts images. | ### `noop`