Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion crates/switchyard-runner/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ struct RouteConfig {
tool_calling: Option<bool>,
reasoning: Option<bool>,
vision: Option<bool>,
base_instructions: Option<String>,
algorithm: AlgorithmSpec,
}

Expand All @@ -88,6 +89,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
let tool_calling = take_optional(&mut table, "tool_calling")?;
let reasoning = take_optional(&mut table, "reasoning")?;
let vision = take_optional(&mut table, "vision")?;
let base_instructions = take_optional(&mut table, "base_instructions")?;
let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))
.map_err(serde::de::Error::custom)?;
Ok(Self {
Expand All @@ -96,6 +98,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
tool_calling,
reasoning,
vision,
base_instructions,
algorithm,
})
}
Expand Down Expand Up @@ -236,7 +239,7 @@ impl DeploymentConfig {
if let Some(subagent) = names.subagent {
models = models.with_subagent(resolve_category_models(subagent, &targets)?);
}
let route = Route::new(
let mut route = Route::new(
algorithm,
route_clients,
caller_auth,
Expand All @@ -246,6 +249,9 @@ impl DeploymentConfig {
decision_targets,
models,
);
if let Some(instructions) = &config.base_instructions {
route = route.with_base_instructions(instructions.clone())?;
}
routes.push((config.id.clone(), route));
}
let runner = Runner::new(routes).with_fallback_url(fallback_base_url);
Expand Down
19 changes: 19 additions & 0 deletions crates/switchyard-runner/src/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ pub struct Route {
clients: ClientRouter,
caller_auth: Option<CallerAuthKind>,
capabilities: ModelCapabilities,
base_instructions: Option<String>,
anthropic_auxiliary_target: Option<AuxiliaryTarget>,
responses_auxiliary_target: Option<AuxiliaryTarget>,
decision_targets: Vec<DecisionTarget>,
Expand Down Expand Up @@ -152,6 +153,7 @@ impl Route {
clients,
caller_auth,
capabilities,
base_instructions: None,
anthropic_auxiliary_target,
responses_auxiliary_target,
decision_targets,
Expand All @@ -169,6 +171,23 @@ impl Route {
self.capabilities
}

/// Sets the instructions advertised to Codex, preserving the text verbatim.
/// Returns a configuration error if the text is empty or whitespace-only.
pub fn with_base_instructions(mut self, instructions: String) -> Result<Self, RunnerError> {
if instructions.trim().is_empty() {
return Err(RunnerError::configuration(
"base_instructions must not be empty",
));
}
self.base_instructions = Some(instructions);
Ok(self)
}

/// Returns the route's Codex instructions, or `None` when undeclared.
pub fn base_instructions(&self) -> Option<&str> {
self.base_instructions.as_deref()
}

/// Returns the forwarded caller credential family.
pub fn caller_auth(&self) -> Option<CallerAuthKind> {
self.caller_auth
Expand Down
5 changes: 5 additions & 0 deletions crates/switchyard-runner/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ impl Runner {
.map(|(_, route)| route)
}

/// Iterates over route IDs and their configuration in caller-provided order.
pub fn routes(&self) -> impl Iterator<Item = (&ModelId, &Route)> {
self.routes.iter().map(|(id, route)| (id, route))
}

/// Iterates over configured routes in caller-provided order.
pub fn models(&self) -> impl Iterator<Item = ModelInfo<'_>> {
self.routes.iter().map(|(id, route)| ModelInfo {
Expand Down
53 changes: 36 additions & 17 deletions crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,20 @@ pub fn build_llm_router(state: ServerState) -> Router {

/// Builds the full Axum router used by the standalone Switchyard server.
pub fn build_switchyard_router(state: ServerState) -> Router {
// Report prompt substitution wherever model discovery is mounted, including embedded hosts.
let unconfigured = state
.runner
.routes()
.filter(|(_, route)| route.base_instructions().is_none())
.map(|(id, _)| id.as_str())
.collect::<Vec<_>>();
if !unconfigured.is_empty() {
tracing::warn!(
routes = ?unconfigured,
"Codex model discovery will replace the client's base instructions with a \
placeholder for these routes; set base_instructions to the intended Codex prompt"
);
}
let mut router = primary_llm_routes()
.route("/v1/decision", post(decision))
.route("/v1/messages/count_tokens", post(anthropic_count_tokens))
Expand Down Expand Up @@ -1362,12 +1376,9 @@ fn error_response(
}

async fn models(State(state): State<ServerState>) -> Json<Value> {
Json(model_list_payload(
state
.runner
.models()
.map(|model| (model.id.as_str(), model.capabilities)),
))
Json(model_list_payload(state.runner.routes().map(
|(id, route)| (id.as_str(), route.capabilities(), route.base_instructions()),
)))
}

async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
Expand Down Expand Up @@ -1448,20 +1459,23 @@ async fn not_found() -> Response {
}

fn model_list_payload<'a>(
entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
entries: impl IntoIterator<Item = (&'a str, ModelCapabilities, Option<&'a str>)>,
) -> Value {
let mut entries = entries.into_iter().collect::<Vec<_>>();
entries.sort_unstable_by_key(|(model_id, _)| *model_id);
let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
entries.sort_unstable_by_key(|(model_id, _, _)| *model_id);
let model_ids = entries
.iter()
.map(|(model, _, _)| *model)
.collect::<Vec<_>>();
let first_id = model_ids.first().copied();
let last_id = model_ids.last().copied();
json!({
"object": "list",
"data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
"data": entries.iter().map(|(model, caps, _)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
"models": entries
.iter()
.enumerate()
.map(|(priority, (model, caps))| codex_model_entry_json(model, *caps, priority))
.map(|(priority, (model, caps, instructions))| codex_model_entry_json(model, *caps, *instructions, priority))
.collect::<Vec<_>>(),
"first_id": first_id,
"last_id": last_id,
Expand Down Expand Up @@ -1502,17 +1516,22 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
//
// Two kinds of fields live here. context_window, tool_calling, and reasoning are model
// facts a backend can publish; the route declares them in config today. The rest
// (shell_type, apply_patch_tool_type, base_instructions, the reasoning-level presets,
// (shell_type, apply_patch_tool_type, the reasoning-level presets,
// truncation_policy) are Codex client conventions no backend returns, so they stay
// constant.
// constant. Base instructions are supplied by the route operator.
//
// TODO: source context_window, tool_calling, and reasoning from the backend, not route
// config. Switchyard is a proxy, so it should re-publish what the backend advertises
// when it can — OpenRouter's /api/v1/models exposes context_length and
// supported_parameters — and fall back to the route's declared value. Some backends
// publish nothing (the NVIDIA gateway returns id-only models and blocks /model/info),
// so keep failing closed to config.
fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority: usize) -> Value {
fn codex_model_entry_json(
model: &str,
capabilities: ModelCapabilities,
base_instructions: Option<&str>,
priority: usize,
) -> Value {
// Codex is non-functional without shell and apply_patch, so an undeclared tool
// capability defaults to enabled here; the OpenAI `data` entry reports the raw
// Option separately for clients that want the undeclared state.
Expand All @@ -1532,9 +1551,9 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority
"additional_speed_tiers": [],
"availability_nux": null,
"upgrade": null,
// Required `ModelInfo` string. Unlike the launcher, the server cannot read
// Codex's bundled prompt, so it sends a minimal stub.
"base_instructions": "You are Codex, a coding agent.",
// Codex adopts this text. Omitting both it and instructions_template rejects the
// entire catalog. Keep the default and warn when the discovery router is built.
"base_instructions": base_instructions.unwrap_or("You are Codex, a coding agent."),
"supports_reasoning_summaries": reasoning,
"default_reasoning_summary": "none",
"support_verbosity": reasoning,
Expand Down
106 changes: 106 additions & 0 deletions crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2533,6 +2533,112 @@ async fn json_extractor_statuses_keep_api_specific_error_envelopes() -> TestResu
Ok(())
}

// Configured instructions must survive TOML loading and catalog serialization verbatim.
#[tokio::test]
async fn models_endpoint_preserves_per_route_base_instructions() -> TestResult {
const CONFIG: &str = r#"
schema_version = 1
[targets]
[routes.configured]
id = "configured"
type = "noop"
vision = true
base_instructions = " First line.\nSecond line: 中文.\n"
[routes.other]
id = "other"
type = "noop"
base_instructions = "A different prompt."
[routes.default]
id = "default"
type = "noop"
"#;
let app = build_switchyard_router(load_test_config(CONFIG)?);
let response = send(&app, "GET", "/v1/models", None).await?;
assert_eq!(response.status, StatusCode::OK);
let body = response.json()?;
let models = body["models"].as_array().expect("Codex catalog");
assert_eq!(models.len(), 3);
let entries = models
.iter()
.map(|entry| (entry["slug"].as_str().expect("route id"), entry))
.collect::<BTreeMap<_, _>>();
assert_eq!(
entries["configured"]["base_instructions"],
" First line.\nSecond line: 中文.\n"
);
assert_eq!(entries["other"]["base_instructions"], "A different prompt.");
// Omitting instructions would invalidate the entire Codex catalog, including vision.
assert_eq!(
entries["default"]["base_instructions"],
"You are Codex, a coding agent."
);
assert_eq!(
entries["configured"]["input_modalities"],
json!(["text", "image"])
);
Ok(())
}

// Embedded discovery hosts must warn without logging any configured prompt text.
#[tokio::test]
async fn model_discovery_warns_only_for_unconfigured_routes() -> TestResult {
use std::io::{Read, Seek};

const CONFIG: &str = r#"
schema_version = 1
[targets]
[routes.configured]
id = "configured-id"
type = "noop"
base_instructions = "private prompt sentinel"
[routes.missing]
id = "missing-id"
type = "noop"
"#;
for (extra, needs_warning) in [("", true), ("base_instructions = 'another prompt'", false)] {
let mut log = tempfile::tempfile()?;
let subscriber = tracing_subscriber::fmt()
.without_time()
.with_ansi(false)
.with_writer(log.try_clone()?)
.finish();
let state = load_test_config(&format!("{CONFIG}{extra}"))?;
let _ = tracing::subscriber::with_default(subscriber, || build_switchyard_router(state));
log.rewind()?;
let mut output = String::new();
log.read_to_string(&mut output)?;
assert_eq!(
output.contains("Codex model discovery"),
needs_warning,
"{output}"
);
assert_eq!(output.contains("missing-id"), needs_warning, "{output}");
assert!(!output.contains("configured-id"), "{output}");
assert!(!output.contains("private prompt sentinel"), "{output}");
assert!(!output.contains("another prompt"), "{output}");
}
Ok(())
}

// A blank prompt must not silently disable the agent's base instructions.
#[test]
fn route_base_instructions_rejects_blank_values() {
for value in [r#""""#, r#"" \n\t ""#] {
let config = format!(
"schema_version = 1\n[targets]\n[routes.test]\nid = \"test\"\ntype = \"noop\"\nbase_instructions = {value}\n"
);
let error = load_test_config(&config)
.err()
.expect("blank prompt rejected");
assert!(
error
.to_string()
.contains("base_instructions must not be empty"),
"{error}"
);
}
}

#[tokio::test]
async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult
{
Expand Down
22 changes: 22 additions & 0 deletions docs/reference/toml_schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,30 @@ Every route takes the common keys below, plus the keys for its type.
| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. |
| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. |
| `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. |
| `base_instructions` | No | unset | Nonblank text served verbatim in the Codex model catalog. Codex uses it in place of its own base instructions. Unset routes keep the placeholder prompt and produce a startup warning. |
| `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. |

### Codex base instructions

For Codex direct-provider discovery, set `base_instructions` on each route to the
complete prompt you want Codex to use. TOML multiline strings can hold the prompt.
Switchyard preserves the text after TOML parsing, including leading and trailing
whitespace. TOML removes a newline immediately after the opening delimiter of a
multiline string before Switchyard receives the value. Empty or whitespace-only
values are rejected.

To compare a routed session with a direct session, use the same resolved base
instructions as the direct session for that Codex version, model, and configuration.
Check them again after upgrading Codex. This setting controls the advertised prompt;
it does not rewrite incoming request instructions or change other model metadata.

When the setting is omitted, Switchyard still serves
`You are Codex, a coding agent.` and logs the affected route IDs at startup. This
keeps existing catalogs usable, but does not preserve Codex's original prompt.
Codex versions that require instructions reject the entire catalog if both
`base_instructions` and `model_messages.instructions_template` are absent. Serving
the template instead also replaces Codex's prompt.

### `noop`

Returns a buffered assistant response containing `OK` without calling an
Expand Down