From 564416f6c6b81971be85799c63b21cb204232f15 Mon Sep 17 00:00:00 2001 From: Gabriel Amazonas Date: Sat, 19 Sep 2026 16:32:33 -0300 Subject: [PATCH 1/4] feat: add RLCD decision routing to libsy Signed-off-by: Gabriel Amazonas --- crates/libsy/src/algorithms.rs | 1 + crates/libsy/src/algorithms/llm_class.rs | 2 +- crates/libsy/src/algorithms/rlcd.rs | 727 ++++++++++++++++++ crates/libsy/src/algorithms/util/llm_judge.rs | 8 +- crates/libsy/src/lib.rs | 1 + crates/libsy/src/prompts/rlcd/prompt.md | 28 + crates/libsy/src/prompts/rlcd/schema.json | 44 ++ crates/switchyard-runner/src/algorithm.rs | 86 ++- crates/switchyard-runner/tests/route.rs | 201 +++++ crates/switchyard-server/tests/server.rs | 53 ++ docs/reference/toml_schema.md | 22 + docs/routing_algorithms/rlcd_routing.md | 102 +++ examples/rlcd/README.md | 77 ++ examples/rlcd/routes.toml | 30 + mkdocs.yml | 1 + 15 files changed, 1377 insertions(+), 6 deletions(-) create mode 100644 crates/libsy/src/algorithms/rlcd.rs create mode 100644 crates/libsy/src/prompts/rlcd/prompt.md create mode 100644 crates/libsy/src/prompts/rlcd/schema.json create mode 100644 docs/routing_algorithms/rlcd_routing.md create mode 100644 examples/rlcd/README.md create mode 100644 examples/rlcd/routes.toml diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index fa6c2af5d..7a9fa596c 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -14,6 +14,7 @@ pub mod llm_class; pub mod noop; pub mod passthrough; pub mod rand; +pub mod rlcd; pub mod stage; pub mod subagent; diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 12d9241d0..38e5891b7 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -149,7 +149,7 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { } /// Keeps the opening task and the latest user follow-up when they differ. -fn task_messages(messages: &[Message]) -> Vec { +pub(crate) fn task_messages(messages: &[Message]) -> Vec { // Decoders also use the user role for tool results. Select ordinary user content // first, so a tool result cannot replace the opening task or latest follow-up. let is_task_content = |block: &ContentBlock| { diff --git a/crates/libsy/src/algorithms/rlcd.rs b/crates/libsy/src/algorithms/rlcd.rs new file mode 100644 index 000000000..0c9234387 --- /dev/null +++ b/crates/libsy/src/algorithms/rlcd.rs @@ -0,0 +1,727 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RLCD-backed decision routing: a calibrated decision model picks among the +//! route's targets in one pass. +//! +//! RLCD ("Reinforcement Learning for Calibrated Decisions") models map a task +//! and a list of options to one calibrated probability per option without +//! writing an answer word by word. TypeSafe's Jev popularized the approach; +//! the most-liked open Jev counterpart on Hugging Face is +//! `AlexWortega/openjev`, a Qwen3.5 cross-encoder that scores a task against +//! every candidate option and returns one probability each. RLCD checkpoints +//! such as `harshatheg/Qwen-2.5-1B-RLCD` express the same contract. +//! +//! [`Rlcd`] builds a decision request that lists every runtime target as a +//! candidate option, routes to the option with the highest probability, and +//! falls back to the rest of the candidate list — then the configured default +//! target — when the decision verdict is unusable. A decision call that fails +//! in-band (the verdict never arrives) folds to the default target the same +//! way. + +use std::collections::HashSet; +use std::sync::Arc; + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::Value; +use switchyard_protocol::{ + Category, InstructionBlock, LlmRequest, Message, ModelId, OutputParams, Request, Response, Role, +}; + +use super::fall_through::FallThrough; +use super::llm_class::task_messages; +use super::util::classifier_contract::{ClassifierContract, ClassifierContractConfig}; +use super::util::llm_judge::{ + JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, VerdictDecoder, client_error_reason, + libsy_error_reason, report_fail_open, +}; +use super::util::robustness::{safe_client_error, safe_error_summary}; +use crate::core::algorithm::{Algorithm, Driver}; +use crate::core::classifier::{Classification, Classifier, Score}; +use crate::{LibsyError, Result}; + +const PROMPT_TEMPLATE: &str = include_str!("../prompts/rlcd/prompt.md"); +const SCHEMA_TEMPLATE: &str = include_str!("../prompts/rlcd/schema.json"); +/// Telemetry label for this algorithm's spans, metrics, and logs. +const ALGORITHM_NAME: &str = "rlcd"; +/// The decision model may lose precision rounding probabilities to JSON text. +const PROBABILITY_TOLERANCE: f64 = 0.02; + +/// One candidate option and the calibrated probability the decision model assigned it. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct RlcdOptionScore { + /// The candidate option this probability belongs to. + option: String, + /// Calibrated chance, in `[0, 1]`, that this option is the best target. + probability: f64, +} + +/// The typed decision response from the decision model. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct RlcdVerdict { + /// The option the decision model picked; must match the argmax probability. + target: String, + /// One probability per candidate option. + probabilities: Vec, +} + +impl RlcdVerdict { + /// The highest-probability option, or the first when probabilities tie. + fn best(&self) -> &RlcdOptionScore { + let mut best = 0; + for index in 1..self.probabilities.len() { + if self.probabilities[index].probability > self.probabilities[best].probability { + best = index; + } + } + &self.probabilities[best] + } + + /// The verdict is usable when it names every candidate exactly once with a + /// finite `[0, 1]` probability, the probabilities sum to about one, and + /// `target` is the highest-probability option. + fn is_valid(&self, candidates: &[ModelId]) -> bool { + if candidates.is_empty() || self.probabilities.len() != candidates.len() { + return false; + } + let mut sum = 0.0; + let mut seen: HashSet<&str> = HashSet::new(); + for score in &self.probabilities { + if !score.probability.is_finite() || !(0.0..=1.0).contains(&score.probability) { + return false; + } + if !candidates + .iter() + .any(|candidate| candidate.as_str() == score.option.as_str()) + { + return false; + } + if !seen.insert(score.option.as_str()) { + return false; + } + sum += score.probability; + } + (sum - 1.0).abs() <= PROBABILITY_TOLERANCE && self.target == self.best().option + } +} + +/// Settings that control RLCD decision routing. +#[derive(Clone, Debug)] +pub struct RlcdConfig { + /// Target chosen when the decision verdict is unusable or the decision + /// call fails in-band. + pub default_target: ModelId, + /// Prompt and verdict contract settings for the decision model. + pub contract: ClassifierContractConfig, + /// Maximum completion tokens available to the decision verdict. + pub max_output_tokens: u64, +} + +impl RlcdConfig { + fn validate(&self) -> Result<()> { + if self.max_output_tokens == 0 { + return Err(LibsyError::AlgorithmError { + message: "max_output_tokens must be at least 1".to_string(), + }); + } + Ok(()) + } +} + +/// Maps a validated decision verdict to routing scores. +struct RlcdPolicy; + +impl JudgePolicy for RlcdPolicy { + type Verdict = RlcdVerdict; + + fn to_classification( + &self, + verdict: Option<&Self::Verdict>, + driver: &Driver, + ) -> Result { + // Decision-model output is untrusted. An absent, invalid, or inconsistent verdict is + // ambiguous so the surrounding router applies its configured fallback. + let Some(verdict) = + verdict.filter(|verdict| verdict.is_valid(driver.models_for(&Category::Any))) + else { + return Ok(Classification::Ambiguous(vec![])); + }; + Ok(Classification::Scores( + verdict + .probabilities + .iter() + .map(|score| Score { + target: ModelId::from(score.option.clone()), + confidence: score.probability, + category: Some(Category::Any), + }) + .collect(), + )) + } +} + +/// Returns routing evidence for a usable decision. +fn decision_evidence(verdict: Option<&RlcdVerdict>) -> Option { + let verdict = verdict?; + Some(serde_json::json!({ + "source": "rlcd", + "verdict": verdict.best().option.clone(), + "score": verdict.best().probability, + "probabilities": verdict.probabilities.iter().map(|score| serde_json::json!({ + "option": score.option.clone(), + "probability": score.probability, + })).collect::>(), + })) +} + +/// Builds the decision request: the task messages plus a trailing user message +/// that enumerates the candidate options. +fn decision_messages(messages: &[Message], candidates: &[ModelId]) -> Vec { + let mut messages = task_messages(messages); + let options = candidates + .iter() + .enumerate() + .map(|(index, candidate)| format!("{index}: {candidate}")) + .collect::>() + .join("\n"); + messages.push(Message::text( + Role::User, + format!("Choose the best target for the task from these options:\n{options}"), + )); + messages +} + +/// Consults the runtime decision model and converts its verdict into routing scores. +struct RlcdClassifier { + contract: ClassifierContract, + policy: RlcdPolicy, + runtime: JudgeRuntimeConfig, +} + +impl RlcdClassifier { + fn new(contract: ClassifierContract, policy: RlcdPolicy, runtime: JudgeRuntimeConfig) -> Self { + Self { + contract, + policy, + runtime, + } + } + + fn build_decision_request(&self, request: &Request, driver: &Driver) -> Request { + Request { + llm_request: LlmRequest { + model: request.llm_request.model.clone(), + instructions: vec![InstructionBlock { + role: Role::System, + content: Message::text(Role::System, self.contract.system_prompt().to_string()) + .content, + }], + messages: decision_messages( + &request.llm_request.messages, + driver.models_for(&Category::Any), + ), + output: OutputParams { + max_output_tokens: Some(self.runtime.max_output_tokens()), + response_format: Some(self.contract.response_format().clone()), + }, + ..LlmRequest::default() + }, + raw_request: None, + metadata: request.metadata.clone(), + } + } + + /// Logs and counts a failed decision call. + fn record_fail_open(&self, driver: &Driver, error: String, reason: &'static str) { + let judge_target = driver + .first_model_for(&Category::Judge) + .map(|c| c.as_str()) + .unwrap_or("missing"); + report_fail_open(judge_target, error, reason); + driver.set_evidence_if_empty(serde_json::json!({ + "source": "fail_open", + "reason_code": reason, + })); + } + + /// Consults the decision model, yielding `None` when it is unavailable or + /// unintelligible so the surrounding router applies its fallback. + async fn decision( + &self, + request: &Request, + driver: &Driver, + judge_models: &[ModelId], + ) -> Option { + let judge_model = judge_models.first()?.as_str(); + + tracing::info!(target = judge_model, "consulting rlcd decision model"); + let response = driver + .call_model( + self.build_decision_request(request, driver), + judge_models.to_vec(), + ) + .await + .inspect_err(|error| { + self.record_fail_open(driver, safe_error_summary(error), libsy_error_reason(error)); + }) + .ok()?; + let aggregate = response + .llm_response + .into_agg() + .await + .inspect_err(|error| { + self.record_fail_open(driver, safe_client_error(error), client_error_reason(error)); + }) + .ok()?; + SerdeDecoder::::new() + .decode(&aggregate, &self.contract) + .inspect_err(|error| { + self.record_fail_open(driver, safe_error_summary(error), "parse_error"); + }) + .ok() + } +} + +#[async_trait] +impl Classifier<()> for RlcdClassifier { + async fn score( + &self, + _state: &mut (), + request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + let judge_models = driver.models_for(&Category::Judge); + if judge_models.is_empty() { + return Err(LibsyError::AlgorithmError { + message: "no models available for category Judge".to_string(), + }); + } + let verdict = self.decision(request, driver, judge_models).await; + let classification = self.policy.to_classification(verdict.as_ref(), driver)?; + if let Some(evidence) = decision_evidence(verdict.as_ref()) { + match &classification { + Classification::Scores(scores) if !scores.is_empty() => { + driver.set_evidence(evidence); + } + _ => driver.set_evidence_if_empty(evidence), + } + } + // The decision model is a side call, never the turn's answer. + Ok((classification, None)) + } +} + +/// Terminal classifier that routes the configured default target when the +/// decision model abstains. +pub struct RlcdFallback(pub ModelId); + +#[async_trait] +impl Classifier for RlcdFallback { + async fn score( + &self, + _state: &mut S, + _request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + driver.set_evidence_if_empty(serde_json::json!({"source": "fail_open"})); + Ok(( + Classification::Scores(vec![Score { + target: self.0.clone(), + confidence: 0.0, + category: Some(Category::Any), + }]), + None, + )) + } +} + +/// Routes each request by consulting an RLCD decision model. +pub struct Rlcd { + route: FallThrough<()>, +} + +impl Rlcd { + /// Builds an RLCD router. + /// + /// # Errors + /// + /// Returns an error when the decision contract or runtime settings are invalid. + pub fn new(config: RlcdConfig) -> Result { + config.validate()?; + let contract = + ClassifierContract::from_config(&config.contract, PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?; + let classifier = Arc::new(RlcdClassifier::new( + contract, + RlcdPolicy, + JudgeRuntimeConfig::new(config.max_output_tokens)?, + )); + Ok(Self { + route: FallThrough::new() + .with_name(ALGORITHM_NAME) + .with_classifier(classifier) + .with_classifier(Arc::new(RlcdFallback(config.default_target.clone()))), + }) + } +} + +#[async_trait] +impl Algorithm for Rlcd { + fn name(&self) -> &str { + ALGORITHM_NAME + } + + async fn route( + self: Arc, + driver: Driver, + request: Request, + ) -> Result { + self.route.execute(driver, request).await + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use futures::StreamExt; + use parking_lot::Mutex; + + use super::*; + use switchyard_protocol::{ + LlmClientError, LlmResponse, Metadata, completion_text, text_request, text_response, + }; + + use crate::core::testing::{Serve, test_drive_with_models}; + + fn test_config(default_target: &str) -> RlcdConfig { + RlcdConfig { + default_target: ModelId::from(default_target.to_string()), + contract: ClassifierContractConfig::default(), + max_output_tokens: 128, + } + } + + fn request() -> Request { + Request { + llm_request: text_request(Some("auto".to_string()), "classify this task"), + raw_request: None, + metadata: None, + } + } + + fn session_request() -> Request { + Request { + metadata: Some(Metadata { + session_id: Some("session-1".to_string()), + ..Metadata::default() + }), + ..request() + } + } + + fn runtime_models() -> HashMap> { + [ + (Category::Judge, vec![ModelId::from("decision")]), + ( + Category::Any, + vec![ModelId::from("efficient"), ModelId::from("capable")], + ), + ] + .into() + } + + fn router(default_target: &str) -> Result> { + Ok(Arc::new(Rlcd::new(test_config(default_target))?)) + } + + fn verdict(target: &str, scores: &[(&str, f64)]) -> String { + let probabilities = scores + .iter() + .map(|(option, probability)| { + format!(r#"{{"option":"{option}","probability":{probability}}}"#) + }) + .collect::>() + .join(","); + format!(r#"{{"target":"{target}","probabilities":[{probabilities}]}}"#) + } + + /// Answers the decision model with `completion`; every other target echoes its name. + fn serve_with(completion: String) -> impl Serve { + move |model: ModelId, _request: Request| { + let completion = completion.clone(); + async move { + let model = model.to_string(); + let text = if model == "decision" { + completion.to_string() + } else { + format!("answer from {model}") + }; + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, text)), + metadata: None, + upstream_headers: http::HeaderMap::new(), + }) + } + } + } + + /// A decision model that times out; every other target answers normally. + fn unreachable_decision() -> impl Serve { + |model: ModelId, _request: Request| async move { + let model = model.to_string(); + if model == "decision" { + return Err(LlmClientError::Timeout { + source: Box::new(std::io::Error::other("decision model unreachable")), + }); + } + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, format!("answer from {model}"))), + metadata: None, + upstream_headers: http::HeaderMap::new(), + }) + } + } + + /// Captures the request a target receives, then answers it with its name. + fn capturing(into: Arc>>) -> impl Serve { + move |model: ModelId, request: Request| { + let into = Arc::clone(&into); + async move { + if model.as_str() == "decision" { + *into.lock() = Some(request); + } + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, model.to_string())), + metadata: None, + upstream_headers: http::HeaderMap::new(), + }) + } + } + } + + fn capable_verdict() -> String { + verdict("capable", &[("efficient", 0.3), ("capable", 0.7)]) + } + + #[tokio::test] + async fn rlcd_routes_to_the_argmax_option() -> Result<()> { + let (selected, response) = test_drive_with_models( + router("efficient")?, + request(), + runtime_models(), + serve_with(capable_verdict()), + ) + .await?; + + assert_eq!(selected, "capable"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("answer from capable".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn a_high_probability_efficient_verdict_routes_efficient() -> Result<()> { + let (selected, _) = test_drive_with_models( + router("capable")?, + request(), + runtime_models(), + serve_with(verdict( + "efficient", + &[("efficient", 0.9), ("capable", 0.1)], + )), + ) + .await?; + assert_eq!(selected, "efficient"); + Ok(()) + } + + #[tokio::test] + async fn an_unreachable_decision_model_routes_the_default_target() -> Result<()> { + let (selected, response) = test_drive_with_models( + router("efficient")?, + request(), + runtime_models(), + unreachable_decision(), + ) + .await?; + + assert_eq!(selected, "efficient"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("answer from efficient".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn an_invalid_verdict_routes_the_default_target() -> Result<()> { + for completion in [ + "not json at all".to_string(), + verdict("efficient", &[("efficient", 1.5), ("capable", -0.5)]), + verdict("efficient", &[("efficient", 0.3)]), + verdict("efficient", &[("efficient", 0.3), ("capable", 0.3)]), + verdict("efficient", &[("efficient", 1.0), ("efficient", 0.0)]), + ] { + let (selected, _) = test_drive_with_models( + router("efficient")?, + request(), + runtime_models(), + serve_with(completion), + ) + .await?; + assert_eq!(selected, "efficient", "unusable verdict routed {selected}"); + } + Ok(()) + } + + #[tokio::test] + async fn a_target_that_mismatches_the_argmax_routes_the_default_target() -> Result<()> { + // The model answered "efficient" yet gave capable the higher probability. + let (selected, _) = test_drive_with_models( + router("efficient")?, + request(), + runtime_models(), + serve_with(verdict( + "efficient", + &[("efficient", 0.3), ("capable", 0.7)], + )), + ) + .await?; + assert_eq!(selected, "efficient"); + Ok(()) + } + + #[tokio::test] + async fn the_decision_request_enumerates_every_candidate_option() -> Result<()> { + let seen = Arc::new(Mutex::new(None)); + let serve = capturing(seen.clone()); + let models = runtime_models(); + test_drive_with_models(router("efficient")?, session_request(), models, serve).await?; + + let request = seen + .lock() + .take() + .ok_or_else(|| LibsyError::AlgorithmError { + message: "the decision model was never called".to_string(), + })?; + let text = request + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("\n")) + .collect::>() + .join("\n"); + assert!( + text.contains("efficient"), + "options missing from decision request: {text}" + ); + assert!( + text.contains("capable"), + "options missing from decision request: {text}" + ); + assert!(text.contains("Choose the best target")); + Ok(()) + } + + #[tokio::test] + async fn the_decision_records_the_probability_distribution_as_evidence() -> Result<()> { + use crate::core::algorithm::Step; + + let models = runtime_models(); + let stream = router("efficient")?.run_stream(request(), Arc::new(models.into())); + tokio::pin!(stream); + + let mut evidence = None; + while let Some(step) = stream.next().await { + match step? { + Step::CallModel(call) => { + let model_name = call + .models + .first() + .map(|model| model.to_string()) + .unwrap_or_default(); + let text = if model_name == "decision" { + capable_verdict() + } else { + "answer".to_string() + }; + call.respond(Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, text)), + metadata: None, + upstream_headers: http::HeaderMap::new(), + }))?; + } + Step::Done(outcome) => { + evidence = outcome + .metadata + .as_ref() + .expect("run_stream should attach outcome metadata") + .evidence + .clone(); + } + } + } + + let evidence = evidence.ok_or_else(|| LibsyError::AlgorithmError { + message: "no evidence recorded".to_string(), + })?; + assert_eq!( + evidence.pointer("/source").and_then(Value::as_str), + Some("rlcd") + ); + assert_eq!( + evidence.pointer("/verdict").and_then(Value::as_str), + Some("capable") + ); + let probabilities = evidence + .pointer("/probabilities") + .and_then(Value::as_array) + .ok_or_else(|| LibsyError::AlgorithmError { + message: "no probabilities in evidence".to_string(), + })?; + assert_eq!(probabilities.len(), 2); + Ok(()) + } + + #[test] + fn rlcd_config_rejects_zero_max_output_tokens() { + let error = Rlcd::new(RlcdConfig { + default_target: ModelId::from("capable"), + contract: ClassifierContractConfig::default(), + max_output_tokens: 0, + }) + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + assert!( + error.contains("max_output_tokens"), + "unexpected error: {error}" + ); + } + + #[test] + fn a_verdict_is_invalid_when_probabilities_do_not_sum_to_one() -> Result<()> { + let candidates = [ModelId::from("efficient"), ModelId::from("capable")]; + let verdict = RlcdVerdict { + target: "capable".to_string(), + probabilities: vec![ + RlcdOptionScore { + option: "efficient".to_string(), + probability: 0.3, + }, + RlcdOptionScore { + option: "capable".to_string(), + probability: 0.4, + }, + ], + }; + assert!(!verdict.is_valid(&candidates)); + Ok(()) + } +} diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 4798178e9..ab68a2ed3 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -117,6 +117,10 @@ impl JudgeRuntimeConfig { } Ok(Self { max_output_tokens }) } + + pub(crate) fn max_output_tokens(&self) -> u64 { + self.max_output_tokens + } } /// Reusable structured judge assembled from an input view, contract, and verdict decoder. @@ -302,7 +306,7 @@ where /// `error` must already be redacted: `LlmClientError::UpstreamHttp`'s `Display` interpolates the /// raw upstream body, which can quote the conversation back. Callers pass a /// `robustness::safe_*` summary rather than the error itself. -fn report_fail_open(judge_model: &str, error: String, reason: &'static str) { +pub(crate) fn report_fail_open(judge_model: &str, error: String, reason: &'static str) { tracing::warn!( target: "libsy", judge_model, @@ -322,7 +326,7 @@ pub(crate) fn libsy_error_reason(error: &LibsyError) -> &'static str { } /// Returns a bounded reason from the error kind and HTTP status only. -fn client_error_reason(error: &LlmClientError) -> &'static str { +pub(crate) fn client_error_reason(error: &LlmClientError) -> &'static str { match error { LlmClientError::Timeout { .. } => "timeout", LlmClientError::Transport { .. } => "transport", diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 6e84c456a..ae8ba11c7 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -26,6 +26,7 @@ pub use algorithms::llm_class::{ pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; +pub use algorithms::rlcd::{Rlcd, RlcdConfig}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::subagent::{SubagentRouter, SubagentRouterConfig}; pub use algorithms::util::affinity::{AffinityRouter, ClassifyTrigger}; diff --git a/crates/libsy/src/prompts/rlcd/prompt.md b/crates/libsy/src/prompts/rlcd/prompt.md new file mode 100644 index 000000000..5382007e5 --- /dev/null +++ b/crates/libsy/src/prompts/rlcd/prompt.md @@ -0,0 +1,28 @@ +You are a decision model for a model router. You receive the task's opening +instruction and, when present, its latest user follow-up, then a numbered list +of candidate options. Decide which option is the best target for this task. + +Return exactly one JSON object matching the response schema supplied with the +request. Do not include markdown or commentary. + +# Procedure + +1. Read the task and the candidate options once. +2. Privately reason about which option is most likely to complete the task + correctly on one fresh run under the actual harness, tools, and budget. +3. Assign one calibrated probability to every candidate option. The + probability is the chance that this option is the best target for the task. +4. Set `target` to the option with the highest probability. + +# Calibration rules + +Interpret probabilities as natural frequencies. If the option you rate 0.70 is +best for about 70 of 100 comparable fresh tasks, then about 70 should be best. +Use the full range when justified. Reserve 0.00 and 1.00 for outcomes that are +logically impossible or certain under the visible contract. + +- Assign exactly one probability to every candidate option. +- The probabilities must sum to 1.00. +- `target` must name the option with the highest probability. +- Do not invent options that were not listed. +- Do not output options, counts, comments, or any field outside the schema. \ No newline at end of file diff --git a/crates/libsy/src/prompts/rlcd/schema.json b/crates/libsy/src/prompts/rlcd/schema.json new file mode 100644 index 000000000..8d1adb6c9 --- /dev/null +++ b/crates/libsy/src/prompts/rlcd/schema.json @@ -0,0 +1,44 @@ +{ + "type": "json_schema", + "json_schema": { + "name": "RlcdDecision", + "strict": true, + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "target", + "probabilities" + ], + "properties": { + "target": { + "type": "string", + "minLength": 1 + }, + "probabilities": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "option", + "probability" + ], + "properties": { + "option": { + "type": "string", + "minLength": 1 + }, + "probability": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index c0630f199..6ff790b5c 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -14,8 +14,8 @@ use libsy::{ ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, - StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, - ToolSemantics, + Rlcd, RlcdConfig, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, + TaskClassifierConfig, ToolSemantics, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -428,6 +428,20 @@ pub enum AlgorithmSpec { /// Maximum prompts per encoder forward pass. batch_size: Option, }, + /// Asks an RLCD decision model for a calibrated probability per target and + /// routes to the argmax. JEV-style "System One" routing over open decision + /// models such as `harshatheg/Qwen-2.5-1B-RLCD`. + Rlcd { + /// Target through which the decision model is called. Never a routing destination itself. + classifier_target: String, + /// Candidate targets the decision model chooses among. + targets: Vec, + /// Target used when the decision model is unavailable or its reply is unusable. + default_target: String, + /// Most completion tokens the decision verdict may use. + #[serde(default = "default_classifier_max_output_tokens")] + max_output_tokens: u64, + }, } /// What fires an advisor route's review. @@ -586,6 +600,7 @@ impl AlgorithmSpec { executor_target, .. } => vec![executor_target], Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(), + Self::Rlcd { targets, .. } => targets.iter().map(String::as_str).collect(), } } @@ -621,6 +636,9 @@ impl AlgorithmSpec { names.push(&classifier.target); } Self::Advisor { advisor_target, .. } => names.push(advisor_target), + Self::Rlcd { + classifier_target, .. + } => names.push(classifier_target), _ => {} } // A sub-agent classifier calls its own judge, which is never a completion target. @@ -655,6 +673,14 @@ impl AlgorithmSpec { Self::Passthrough { target, .. } => { category_models([(Category::Any, vec![target.clone()])]) } + Self::Rlcd { + classifier_target, + targets, + .. + } => category_models([ + (Category::Judge, vec![classifier_target.clone()]), + (Category::Any, targets.clone()), + ]), Self::LlmClassifier { config } => { classifier_runtime_model_names(config.validated_classifier_mode(route_name)?) } @@ -745,7 +771,8 @@ impl AlgorithmSpec { | Self::StageRouter { .. } | Self::Auto { .. } | Self::Composite { .. } - | Self::PrefillRouter { .. } => None, + | Self::PrefillRouter { .. } + | Self::Rlcd { .. } => None, } } @@ -1358,6 +1385,59 @@ fn build_algorithm( })?; Ok(Arc::new(algorithm)) } + AlgorithmSpec::Rlcd { + classifier_target, + targets: names, + default_target, + max_output_tokens, + } => { + if names.len() < 2 { + return Err(AlgorithmConfigError::new(format!( + "rlcd route {route_name} requires at least two targets" + ))); + } + let mut seen = BTreeSet::new(); + if let Some(duplicate) = names.iter().find(|name| !seen.insert(*name)) { + return Err(AlgorithmConfigError::new(format!( + "rlcd route {route_name}: targets must be unique, {duplicate} is repeated" + ))); + } + if !names + .iter() + .any(|name| name.as_str() == default_target.as_str()) + { + return Err(AlgorithmConfigError::new(format!( + "rlcd route {route_name} default_target {default_target} must be one of targets" + ))); + } + if names + .iter() + .any(|name| name.as_str() == classifier_target.as_str()) + { + return Err(AlgorithmConfigError::new(format!( + "rlcd route {route_name} classifier_target must not be a routing target" + ))); + } + resolve_target_model_id(route_name, classifier_target, targets)?; + for name in names.iter() { + resolve_target_model_id(route_name, name, targets)?; + } + let config = RlcdConfig { + default_target: resolve_target_model_id(route_name, default_target, targets)?, + // Decision models are typically served by self-hosted OpenAI-compatible + // endpoints, which broadly support `json_object` but not strict JSON Schema. + contract: ClassifierContractConfig::default() + .with_response_format_type(ClassifierResponseFormat::JsonObject), + max_output_tokens: *max_output_tokens, + }; + let algorithm = Rlcd::new(config).map_err(|error| { + AlgorithmConfigError::with_source( + format!("rlcd route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } AlgorithmSpec::PrefillRouter { targets: names, checkpoint, diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs index 68bba5667..fa602d98e 100644 --- a/crates/switchyard-runner/tests/route.rs +++ b/crates/switchyard-runner/tests/route.rs @@ -181,3 +181,204 @@ batch_size = 8 assert_eq!(max_length, Some(4096)); assert_eq!(batch_size, Some(8)); } + +// ---RLCD decision routing ------------------------------------------------------- + +fn request() -> Request { + Request { + llm_request: text_request(Some("auto".to_string()), "hello"), + ..Request::default() + } +} + +/// Answers the decision model with a calibrated verdict that picks `strong`, +/// and echoes every other target's name. +struct RlcdStubClient; + +#[async_trait] +impl RoutedLlmClient for RlcdStubClient { + async fn call(&self, request: Request) -> Result { + let model = request.llm_request.model.clone().unwrap_or_default(); + let text = if model == "decision" { + r#"{"target":"strong","probabilities":[{"option":"fast","probability":0.2},{"option":"strong","probability":0.8}]}"#.to_string() + } else { + model + }; + Ok(Response { + llm_response: LlmResponse::Agg(text_response(request.llm_request.model.clone(), text)), + metadata: None, + upstream_headers: Default::default(), + }) + } +} + +fn rlcd_targets() -> BTreeMap { + BTreeMap::from([ + ("decision".to_string(), ModelId::from("decision")), + ("fast".to_string(), ModelId::from("fast")), + ("strong".to_string(), ModelId::from("strong")), + ]) +} + +fn rlcd_spec(default_target: &str) -> AlgorithmSpec { + AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string(), "strong".to_string()], + default_target: default_target.to_string(), + max_output_tokens: 128, + } +} + +fn rlcd_route(client: Arc) -> Route { + let spec = rlcd_spec("fast"); + let algorithm = spec + .build("rlcd_test", &rlcd_targets()) + .expect("rlcd spec should build"); + let clients = ClientRouter::new( + BTreeMap::from([ + (ModelId::from("decision"), client.clone()), + (ModelId::from("fast"), client.clone()), + (ModelId::from("strong"), client), + ]) + .into_iter() + .collect(), + ); + Route::new( + algorithm, + clients, + None, + ModelCapabilities::default(), + None, + None, + Vec::new(), + RuntimeModels::new( + [ + (Category::Judge, vec![ModelId::from("decision")]), + ( + Category::Any, + vec![ModelId::from("fast"), ModelId::from("strong")], + ), + ] + .into(), + ), + ) +} + +#[tokio::test] +async fn rlcd_route_routes_through_the_argmax_target() { + let route = rlcd_route(Arc::new(RlcdStubClient)); + let output = route + .execute(request(), None) + .await + .expect("rlcd route should execute"); + + assert_eq!(output.selected_model, "strong"); + assert_eq!( + output + .response + .llm_response + .as_agg() + .unwrap() + .model + .as_deref(), + Some("strong") + ); +} + +/// A decision model that is never available; other targets answer normally. +struct UnavailableDecisionClient; + +#[async_trait] +impl RoutedLlmClient for UnavailableDecisionClient { + async fn call(&self, request: Request) -> Result { + if request.llm_request.model.as_deref() == Some("decision") { + return Err(LlmClientError::Timeout { + source: Box::new(std::io::Error::other("decision model unreachable")), + }); + } + let model = request.llm_request.model.clone().unwrap_or_default(); + Ok(Response { + llm_response: LlmResponse::Agg(text_response(request.llm_request.model.clone(), model)), + metadata: None, + upstream_headers: Default::default(), + }) + } +} + +#[tokio::test] +async fn rlcd_route_fails_when_the_decision_model_is_unavailable() { + // A failed routing-time client call aborts the request — the same behavior + // as a stalled classifier judge (see the server's client-deadline tests). + let route = rlcd_route(Arc::new(UnavailableDecisionClient)); + let error = route + .execute(request(), None) + .await + .err() + .expect("route should fail with the decision model down"); + assert!(error.to_string().contains("decision"), "{error}"); +} + +#[test] +fn rlcd_spec_lists_decision_and_routing_targets() { + let spec = rlcd_spec("fast"); + assert_eq!(spec.routing_target_names(), ["fast", "strong"]); + assert_eq!(spec.callable_target_names(), ["fast", "strong", "decision"]); +} + +#[test] +fn rlcd_spec_rejects_an_invalid_configuration() { + let targets = rlcd_targets(); + let too_few = AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string()], + default_target: "fast".to_string(), + max_output_tokens: 128, + }; + let error = too_few + .build("test", &targets) + .err() + .expect("build should fail"); + assert!( + error.to_string().contains("at least two targets"), + "{error}" + ); + + let bad_default = AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string(), "strong".to_string()], + default_target: "missing".to_string(), + max_output_tokens: 128, + }; + let error = bad_default + .build("test", &targets) + .err() + .expect("build should fail"); + assert!( + error.to_string().contains("must be one of targets"), + "{error}" + ); + + let duplicate = AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string(), "fast".to_string()], + default_target: "fast".to_string(), + max_output_tokens: 128, + }; + let error = duplicate + .build("test", &targets) + .err() + .expect("build should fail"); + assert!(error.to_string().contains("must be unique"), "{error}"); + + let judge_is_target = AlgorithmSpec::Rlcd { + classifier_target: "strong".to_string(), + targets: vec!["fast".to_string(), "strong".to_string()], + default_target: "fast".to_string(), + max_output_tokens: 128, + }; + let error = judge_is_target + .build("test", &targets) + .err() + .expect("build should fail"); + assert!(error.to_string().contains("classifier_target"), "{error}"); +} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 29bb561e7..3c6487943 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -414,6 +414,10 @@ async fn upstream_chat( r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"#.to_string() } else if model == "model/classifier" { r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string() + } else if model == "model/rlcd-decision" { + // The verdict options are the resolved runtime model ids, matching the + // candidate enumeration in the decision request. + r#"{"target":"model/premium","probabilities":[{"option":"model/weak","probability":0.2},{"option":"model/premium","probability":0.8}]}"#.to_string() } else { "ok".to_string() }; @@ -2485,6 +2489,55 @@ selector = "/decision/target" Ok(()) } +#[tokio::test] +async fn rlcd_route_consults_a_decision_model_before_routing() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 +[llm_clients.mock] +format = "openai_chat" +base_url = "{}" +max_retries = 0 +[targets.classifier] +id = "model/rlcd-decision" +llm_client = "mock" +[targets.weak] +id = "model/weak" +llm_client = "mock" +[targets.premium] +id = "model/premium" +llm_client = "mock" +[routes.rlcd] +id = "switchyard/rlcd" +type = "rlcd" +classifier_target = "classifier" +targets = ["weak", "premium"] +default_target = "weak" +"#, + upstream.base_url + ))?; + let app = build_switchyard_router(state); + + upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/rlcd", + "messages": [{"role": "user", "content": "bounded task"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + let calls = upstream.calls.lock().await; + assert_eq!(calls.len(), 2); + assert_eq!(calls[0]["model"], "model/rlcd-decision"); + assert_eq!(calls[1]["model"], "model/premium"); + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 4be77ed86..d0bd67d59 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -206,6 +206,28 @@ targets = ["fast", "strong"] checkpoint = "/models/router.pt" ``` +### `rlcd` + +Asks an RLCD (calibrated decision) model for one probability per target and +routes to the argmax. See +[RLCD Decision Routing](../routing_algorithms/rlcd_routing.md). + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `classifier_target` | Yes | — | Target the decision model is called through. Not a routing destination. | +| `targets` | Yes | — | Candidate targets the decision model chooses among. Must have at least two entries with no duplicates. | +| `default_target` | Yes | — | Target used when the decision model's reply cannot be used. Must name one of `targets`. | +| `max_output_tokens` | No | `4096` | Maximum completion tokens for the decision verdict. Must be at least `1`. | + +```toml +[routes.decide] +id = "switchyard/rlcd" +type = "rlcd" +classifier_target = "decision" +targets = ["weak", "strong"] +default_target = "weak" +``` + ### `llm_classifier` Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. diff --git a/docs/routing_algorithms/rlcd_routing.md b/docs/routing_algorithms/rlcd_routing.md new file mode 100644 index 000000000..08362440c --- /dev/null +++ b/docs/routing_algorithms/rlcd_routing.md @@ -0,0 +1,102 @@ +# RLCD Decision Routing + +RLCD ("Reinforcement Learning for Calibrated Decisions") routes by asking a +decision model for one calibrated probability per candidate target and picking +the target with the highest probability. This is the same "System One" approach +pioneered by TypeSafe's Jev. The most-liked open implementation of it is +[`AlexWortega/openjev`](https://huggingface.co/AlexWortega/openjev), a Qwen3.5 +cross-encoder that scores a task against every candidate option and returns one +probability each. RLCD checkpoints such as +[`harshatheg/Qwen-2.5-1B-RLCD`](https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD) +express the same contract. + +A generative judge writes an answer token by token. A decision model instead +takes the task plus a list of options and returns every option's probability in +one pass. Routing with it is fast and returns a calibrated confidence for every +target, not just the winner. + +Use it when you have a decision model available and want a content-aware route +with per-target confidence. The decision model is a side call, never the +turn's answer. + +## Configure an RLCD route + +This example serves the decision model behind its own endpoint and lets it +choose between a strong and a weak target: + +```toml +schema_version = 1 + +[llm_clients.rlcd] +format = "openai_chat" +base_url = "http://localhost:8080/v1" +api_key_env = "RLCD_API_KEY" + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.decision] +id = "Qwen-2.5-1B-RLCD" +llm_client = "rlcd" + +[targets.strong] +id = "openai/gpt-5" +llm_client = "openrouter" + +[targets.weak] +id = "openai/gpt-4o-mini" +llm_client = "openrouter" + +[routes.best_fit] +id = "best-fit" +type = "rlcd" +classifier_target = "decision" +targets = ["weak", "strong"] +default_target = "weak" +``` + +The route has three moving parts: + +- `classifier_target` is the model that makes routing decisions. It is a side + call and never a completion destination. +- `targets` lists the candidates the decision model chooses among. There must + be at least two. +- `default_target` is the candidate used when the decision model's reply + cannot be used. It must name one of `targets`. + +## How a decision is made + +For each request Switchyard: + +1. Enumerates every `targets` candidate as a numbered option. +2. Sends the task and the option list to `classifier_target`. +3. Parses one JSON verdict with a probability for every option: + +```json +{ + "target": "model/strong", + "probabilities": [ + {"option": "model/weak", "probability": 0.2}, + {"option": "model/strong", "probability": 0.8} + ] +} +``` + +4. Routes to the option with the highest probability. +5. Keeps the remaining candidates in `targets` order as fallbacks for + eligible non-timeout failures. + +A verdict is used only when it names every candidate exactly once with a +finite probability in `[0, 1]`, the probabilities sum to about `1.00`, and +`target` is the option with the highest probability. Anything else — an +unparseable reply, a missing or duplicated option, a mismatched `target` — +is treated as an unusable verdict and routes to `default_target`. + +An HTTP client failure on the decision call stops the request, exactly like a +failed classifier judge. + +Decision requests use `response_format = {"type": "json_object"}` and the +verdict schema is checked locally, because self-hosted decision-model +endpoints broadly support JSON objects but not strict JSON Schema. \ No newline at end of file diff --git a/examples/rlcd/README.md b/examples/rlcd/README.md new file mode 100644 index 000000000..14af5d2d4 --- /dev/null +++ b/examples/rlcd/README.md @@ -0,0 +1,77 @@ +# RLCD decision routing + +Serve an RLCD (calibrated decision) model behind its own OpenAI-compatible +endpoint and let it choose the best target per request. + +## What you need + +- An endpoint that serves a decision model. The most-liked open Jev + counterpart is + [`AlexWortega/openjev`](https://huggingface.co/AlexWortega/openjev), a + Qwen3.5 cross-encoder that scores a task against every candidate option and + returns one probability per option. Any OpenAI-compatible server that + answers the decision prompt with one JSON object works. +- Two or more completion targets (the candidates the decision model chooses + between). + +## Run + +Start the decision endpoint, then point `routes.toml` at it and the completion +targets through `[llm_clients]`: + +```toml +schema_version = 1 + +[llm_clients.rlcd] +format = "openai_chat" +base_url = "http://localhost:8080/v1" +api_key_env = "RLCD_API_KEY" + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.decision] +id = "Qwen-2.5-1B-RLCD" +llm_client = "rlcd" + +[targets.strong] +id = "openai/gpt-5" +llm_client = "openrouter" + +[targets.weak] +id = "openai/gpt-4o-mini" +llm_client = "openrouter" + +[routes.decide] +id = "switchyard/rlcd" +type = "rlcd" +classifier_target = "decision" +targets = ["weak", "strong"] +default_target = "weak" +``` + +Send requests to the route: + +```bash +# from the repository root: run the Rust server against this example config +cargo run -p switchyard-server -- examples/rlcd/routes.toml + +curl http://localhost:8000/v1/chat/completions \ + -h 'content-type: application/json' \ + -d '{ + "model": "switchyard/rlcd", + "messages": [{"role": "user", "content": "Refactor this module and add tests."}] + }' +``` + +Switchyard sends the task plus a numbered option list to the decision model, +routes to the option with the highest probability, and falls back to +`default_target` when the verdict cannot be used. A failed decision call stops +the request, exactly like a failed classifier judge. + +See +[RLCD Decision Routing](../../docs/routing_algorithms/rlcd_routing.md) for the +full behavior, including the exact verdict schema the decision endpoint must +return. \ No newline at end of file diff --git a/examples/rlcd/routes.toml b/examples/rlcd/routes.toml new file mode 100644 index 000000000..9c18f1929 --- /dev/null +++ b/examples/rlcd/routes.toml @@ -0,0 +1,30 @@ +schema_version = 1 + +[llm_clients.rlcd] +format = "openai_chat" +base_url = "http://localhost:8080/v1" +api_key_env = "RLCD_API_KEY" + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.decision] +id = "AlexWortega/openjev" +llm_client = "rlcd" + +[targets.strong] +id = "openai/gpt-5" +llm_client = "openrouter" + +[targets.weak] +id = "openai/gpt-4o-mini" +llm_client = "openrouter" + +[routes.decide] +id = "switchyard/rlcd" +type = "rlcd" +classifier_target = "decision" +targets = ["weak", "strong"] +default_target = "weak" \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index bcd196402..c17bb2f9d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Execution (Stage Router): routing_algorithms/stage_router_routing.md - Sub-Agent-Aware Routing: routing_algorithms/subagent_routing.md - Random Routing: routing_algorithms/random_routing.md + - RLCD Decision Routing: routing_algorithms/rlcd_routing.md - Composite Routing: routing_algorithms/composite_routing.md - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md From a0cb1a52e2efb8ea4d0bf16e1d23b8ef14ef35c1 Mon Sep 17 00:00:00 2001 From: Gabriel Amazonas Date: Sat, 19 Sep 2026 16:49:16 -0300 Subject: [PATCH 2/4] refactor: harden RLCD verdict handling and trim redundant tests Signed-off-by: Gabriel Amazonas --- crates/libsy/src/algorithms/rlcd.rs | 57 ++++++----------------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/crates/libsy/src/algorithms/rlcd.rs b/crates/libsy/src/algorithms/rlcd.rs index 0c9234387..fde0f19ed 100644 --- a/crates/libsy/src/algorithms/rlcd.rs +++ b/crates/libsy/src/algorithms/rlcd.rs @@ -49,7 +49,7 @@ const ALGORITHM_NAME: &str = "rlcd"; const PROBABILITY_TOLERANCE: f64 = 0.02; /// One candidate option and the calibrated probability the decision model assigned it. -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RlcdOptionScore { /// The candidate option this probability belongs to. @@ -59,7 +59,7 @@ struct RlcdOptionScore { } /// The typed decision response from the decision model. -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RlcdVerdict { /// The option the decision model picked; must match the argmax probability. @@ -70,21 +70,21 @@ struct RlcdVerdict { impl RlcdVerdict { /// The highest-probability option, or the first when probabilities tie. - fn best(&self) -> &RlcdOptionScore { + fn best(&self) -> Option<&RlcdOptionScore> { let mut best = 0; for index in 1..self.probabilities.len() { if self.probabilities[index].probability > self.probabilities[best].probability { best = index; } } - &self.probabilities[best] + self.probabilities.get(best) } /// The verdict is usable when it names every candidate exactly once with a /// finite `[0, 1]` probability, the probabilities sum to about one, and /// `target` is the highest-probability option. fn is_valid(&self, candidates: &[ModelId]) -> bool { - if candidates.is_empty() || self.probabilities.len() != candidates.len() { + if self.probabilities.len() != candidates.len() { return false; } let mut sum = 0.0; @@ -104,7 +104,8 @@ impl RlcdVerdict { } sum += score.probability; } - (sum - 1.0).abs() <= PROBABILITY_TOLERANCE && self.target == self.best().option + (sum - 1.0).abs() <= PROBABILITY_TOLERANCE + && self.best().is_some_and(|best| self.target == best.option) } } @@ -166,10 +167,11 @@ impl JudgePolicy for RlcdPolicy { /// Returns routing evidence for a usable decision. fn decision_evidence(verdict: Option<&RlcdVerdict>) -> Option { let verdict = verdict?; + let best = verdict.best()?; Some(serde_json::json!({ "source": "rlcd", - "verdict": verdict.best().option.clone(), - "score": verdict.best().probability, + "verdict": best.option.clone(), + "score": best.probability, "probabilities": verdict.probabilities.iter().map(|score| serde_json::json!({ "option": score.option.clone(), "probability": score.probability, @@ -316,7 +318,7 @@ impl Classifier<()> for RlcdClassifier { /// Terminal classifier that routes the configured default target when the /// decision model abstains. -pub struct RlcdFallback(pub ModelId); +struct RlcdFallback(ModelId); #[async_trait] impl Classifier for RlcdFallback { @@ -525,22 +527,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn a_high_probability_efficient_verdict_routes_efficient() -> Result<()> { - let (selected, _) = test_drive_with_models( - router("capable")?, - request(), - runtime_models(), - serve_with(verdict( - "efficient", - &[("efficient", 0.9), ("capable", 0.1)], - )), - ) - .await?; - assert_eq!(selected, "efficient"); - Ok(()) - } - #[tokio::test] async fn an_unreachable_decision_model_routes_the_default_target() -> Result<()> { let (selected, response) = test_drive_with_models( @@ -563,6 +549,7 @@ mod tests { async fn an_invalid_verdict_routes_the_default_target() -> Result<()> { for completion in [ "not json at all".to_string(), + verdict("efficient", &[]), verdict("efficient", &[("efficient", 1.5), ("capable", -0.5)]), verdict("efficient", &[("efficient", 0.3)]), verdict("efficient", &[("efficient", 0.3), ("capable", 0.3)]), @@ -704,24 +691,4 @@ mod tests { "unexpected error: {error}" ); } - - #[test] - fn a_verdict_is_invalid_when_probabilities_do_not_sum_to_one() -> Result<()> { - let candidates = [ModelId::from("efficient"), ModelId::from("capable")]; - let verdict = RlcdVerdict { - target: "capable".to_string(), - probabilities: vec![ - RlcdOptionScore { - option: "efficient".to_string(), - probability: 0.3, - }, - RlcdOptionScore { - option: "capable".to_string(), - probability: 0.4, - }, - ], - }; - assert!(!verdict.is_valid(&candidates)); - Ok(()) - } } From 4257ab0ab2ba3ac558ab0fcd7616a356baed2dba Mon Sep 17 00:00:00 2001 From: Gabriel Amazonas Date: Sat, 19 Sep 2026 17:00:20 -0300 Subject: [PATCH 3/4] fix: address RLCD review findings and reference the RLCD paper Signed-off-by: Gabriel Amazonas --- crates/libsy/src/algorithms/rlcd.rs | 62 +++++++++++++++++------ crates/switchyard-runner/src/algorithm.rs | 26 +++++++--- crates/switchyard-runner/tests/route.rs | 44 ++++++++++++++++ docs/routing_algorithms/rlcd_routing.md | 40 +++++++++------ examples/rlcd/README.md | 11 ++-- examples/rlcd/routes.toml | 2 +- 6 files changed, 137 insertions(+), 48 deletions(-) diff --git a/crates/libsy/src/algorithms/rlcd.rs b/crates/libsy/src/algorithms/rlcd.rs index fde0f19ed..b5e9f6f17 100644 --- a/crates/libsy/src/algorithms/rlcd.rs +++ b/crates/libsy/src/algorithms/rlcd.rs @@ -4,13 +4,13 @@ //! RLCD-backed decision routing: a calibrated decision model picks among the //! route's targets in one pass. //! -//! RLCD ("Reinforcement Learning for Calibrated Decisions") models map a task -//! and a list of options to one calibrated probability per option without -//! writing an answer word by word. TypeSafe's Jev popularized the approach; -//! the most-liked open Jev counterpart on Hugging Face is -//! `AlexWortega/openjev`, a Qwen3.5 cross-encoder that scores a task against -//! every candidate option and returns one probability each. RLCD checkpoints -//! such as `harshatheg/Qwen-2.5-1B-RLCD` express the same contract. +//! RLCD models map a task and a list of options to one calibrated probability +//! per option without writing an answer word by word — the "System One" model +//! class TypeSafe's Jev announcement +//! ([docs.typesafe.ai/concepts/system-one](https://docs.typesafe.ai/concepts/system-one)) +//! introduced, trained with Reinforcement Learning for Calibrated Decisions. +//! The RLCD name originates in Reinforcement Learning from Contrastive +//! Distillation ([arXiv:2307.12950](https://arxiv.org/abs/2307.12950)). //! //! [`Rlcd`] builds a decision request that lists every runtime target as a //! candidate option, routes to the option with the highest probability, and @@ -303,13 +303,19 @@ impl Classifier<()> for RlcdClassifier { } let verdict = self.decision(request, driver, judge_models).await; let classification = self.policy.to_classification(verdict.as_ref(), driver)?; - if let Some(evidence) = decision_evidence(verdict.as_ref()) { - match &classification { - Classification::Scores(scores) if !scores.is_empty() => { + match &classification { + Classification::Scores(scores) if !scores.is_empty() => { + if let Some(evidence) = decision_evidence(verdict.as_ref()) { driver.set_evidence(evidence); } - _ => driver.set_evidence_if_empty(evidence), } + // A present but unusable verdict must not credit the rejected + // decision: the fallback target decides, and the evidence says why. + _ if verdict.is_some() => driver.set_evidence_if_empty(serde_json::json!({ + "source": "fail_open", + "reason_code": "invalid_verdict", + })), + _ => {} } // The decision model is a side call, never the turn's answer. Ok((classification, None)) @@ -616,8 +622,9 @@ mod tests { Ok(()) } - #[tokio::test] - async fn the_decision_records_the_probability_distribution_as_evidence() -> Result<()> { + /// Drives one request, answering the decision model with `completion`, and + /// returns the outcome evidence. + async fn evidence_for_decision(completion: String) -> Result { use crate::core::algorithm::Step; let models = runtime_models(); @@ -634,7 +641,7 @@ mod tests { .map(|model| model.to_string()) .unwrap_or_default(); let text = if model_name == "decision" { - capable_verdict() + completion.clone() } else { "answer".to_string() }; @@ -655,9 +662,14 @@ mod tests { } } - let evidence = evidence.ok_or_else(|| LibsyError::AlgorithmError { + evidence.ok_or_else(|| LibsyError::AlgorithmError { message: "no evidence recorded".to_string(), - })?; + }) + } + + #[tokio::test] + async fn the_decision_records_the_probability_distribution_as_evidence() -> Result<()> { + let evidence = evidence_for_decision(capable_verdict()).await?; assert_eq!( evidence.pointer("/source").and_then(Value::as_str), Some("rlcd") @@ -676,6 +688,24 @@ mod tests { Ok(()) } + #[tokio::test] + async fn an_invalid_verdict_records_fail_open_evidence() -> Result<()> { + // The probabilities do not sum to one, so the verdict is rejected and + // the evidence must not credit it. + let evidence = + evidence_for_decision(verdict("capable", &[("efficient", 0.3), ("capable", 0.3)])) + .await?; + assert_eq!( + evidence.pointer("/source").and_then(Value::as_str), + Some("fail_open") + ); + assert_eq!( + evidence.pointer("/reason_code").and_then(Value::as_str), + Some("invalid_verdict") + ); + Ok(()) + } + #[test] fn rlcd_config_rejects_zero_max_output_tokens() { let error = Rlcd::new(RlcdConfig { diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 6ff790b5c..4775ebed7 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -429,8 +429,7 @@ pub enum AlgorithmSpec { batch_size: Option, }, /// Asks an RLCD decision model for a calibrated probability per target and - /// routes to the argmax. JEV-style "System One" routing over open decision - /// models such as `harshatheg/Qwen-2.5-1B-RLCD`. + /// routes to the argmax — TypeSafe Jev-style "System One" routing. Rlcd { /// Target through which the decision model is called. Never a routing destination itself. classifier_target: String, @@ -1410,17 +1409,28 @@ fn build_algorithm( "rlcd route {route_name} default_target {default_target} must be one of targets" ))); } - if names + // Candidates are matched by their resolved model id, so validation + // runs on ids: two aliases of one model would make every verdict + // invalid, and a classifier alias could route to itself. + let classifier_id = resolve_target_model_id(route_name, classifier_target, targets)?; + let candidate_ids = names .iter() - .any(|name| name.as_str() == classifier_target.as_str()) + .map(|name| resolve_target_model_id(route_name, name, targets)) + .collect::>>()?; + let mut resolved = BTreeSet::new(); + if let Some(duplicate) = candidate_ids + .iter() + .map(|id| id.as_str()) + .find(|id| !resolved.insert(*id)) { return Err(AlgorithmConfigError::new(format!( - "rlcd route {route_name} classifier_target must not be a routing target" + "rlcd route {route_name} targets resolve to duplicate model {duplicate}" ))); } - resolve_target_model_id(route_name, classifier_target, targets)?; - for name in names.iter() { - resolve_target_model_id(route_name, name, targets)?; + if candidate_ids.contains(&classifier_id) { + return Err(AlgorithmConfigError::new(format!( + "rlcd route {route_name} classifier_target resolves to candidate model {classifier_id}" + ))); } let config = RlcdConfig { default_target: resolve_target_model_id(route_name, default_target, targets)?, diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs index fa602d98e..57daeae44 100644 --- a/crates/switchyard-runner/tests/route.rs +++ b/crates/switchyard-runner/tests/route.rs @@ -382,3 +382,47 @@ fn rlcd_spec_rejects_an_invalid_configuration() { .expect("build should fail"); assert!(error.to_string().contains("classifier_target"), "{error}"); } + +#[test] +fn rlcd_spec_rejects_aliases_that_collide_after_resolution() { + // Two target names resolving to one model would make every verdict + // invalid, so the route would permanently fall back. + let mut aliased = rlcd_targets(); + aliased.insert("fast-alias".to_string(), ModelId::from("fast")); + let spec = AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string(), "fast-alias".to_string()], + default_target: "fast".to_string(), + max_output_tokens: 128, + }; + let error = spec + .build("test", &aliased) + .err() + .expect("build should fail"); + assert!( + error + .to_string() + .contains("resolve to duplicate model fast"), + "{error}" + ); + + // The classifier itself must not resolve to a candidate model. + let mut shared = rlcd_targets(); + shared.insert("decision".to_string(), ModelId::from("strong")); + let spec = AlgorithmSpec::Rlcd { + classifier_target: "decision".to_string(), + targets: vec!["fast".to_string(), "strong".to_string()], + default_target: "fast".to_string(), + max_output_tokens: 128, + }; + let error = spec + .build("test", &shared) + .err() + .expect("build should fail"); + assert!( + error + .to_string() + .contains("classifier_target resolves to candidate model strong"), + "{error}" + ); +} diff --git a/docs/routing_algorithms/rlcd_routing.md b/docs/routing_algorithms/rlcd_routing.md index 08362440c..2f4f16598 100644 --- a/docs/routing_algorithms/rlcd_routing.md +++ b/docs/routing_algorithms/rlcd_routing.md @@ -3,12 +3,8 @@ RLCD ("Reinforcement Learning for Calibrated Decisions") routes by asking a decision model for one calibrated probability per candidate target and picking the target with the highest probability. This is the same "System One" approach -pioneered by TypeSafe's Jev. The most-liked open implementation of it is -[`AlexWortega/openjev`](https://huggingface.co/AlexWortega/openjev), a Qwen3.5 -cross-encoder that scores a task against every candidate option and returns one -probability each. RLCD checkpoints such as -[`harshatheg/Qwen-2.5-1B-RLCD`](https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD) -express the same contract. +TypeSafe's Jev announcement introduced. Serve any decision model that answers +the decision prompt with one JSON object behind an OpenAI-compatible endpoint. A generative judge writes an answer token by token. A decision model instead takes the task plus a list of options and returns every option's probability in @@ -38,7 +34,7 @@ base_url = "https://openrouter.ai/api/v1" api_key_env = "OPENROUTER_API_KEY" [targets.decision] -id = "Qwen-2.5-1B-RLCD" +id = "decision-model" llm_client = "rlcd" [targets.strong] @@ -72,22 +68,25 @@ For each request Switchyard: 1. Enumerates every `targets` candidate as a numbered option. 2. Sends the task and the option list to `classifier_target`. -3. Parses one JSON verdict with a probability for every option: +3. Parses one JSON verdict with a probability for every option. +4. Routes to the option with the highest probability. +5. Keeps the remaining candidates in `targets` order as fallbacks for + eligible non-timeout failures. + +The verdict's options are the targets' resolved `id` values — the same +identifiers the upstream provider sees, not the local target names. For the +configuration above: ```json { - "target": "model/strong", + "target": "openai/gpt-5", "probabilities": [ - {"option": "model/weak", "probability": 0.2}, - {"option": "model/strong", "probability": 0.8} + {"option": "openai/gpt-4o-mini", "probability": 0.2}, + {"option": "openai/gpt-5", "probability": 0.8} ] } ``` -4. Routes to the option with the highest probability. -5. Keeps the remaining candidates in `targets` order as fallbacks for - eligible non-timeout failures. - A verdict is used only when it names every candidate exactly once with a finite probability in `[0, 1]`, the probabilities sum to about `1.00`, and `target` is the option with the highest probability. Anything else — an @@ -99,4 +98,13 @@ failed classifier judge. Decision requests use `response_format = {"type": "json_object"}` and the verdict schema is checked locally, because self-hosted decision-model -endpoints broadly support JSON objects but not strict JSON Schema. \ No newline at end of file +endpoints broadly support JSON objects but not strict JSON Schema. + +## References + +- TypeSafe's Jev announcement — the "System One" model class trained with + Reinforcement Learning for Calibrated Decisions: + [docs.typesafe.ai/concepts/system-one](https://docs.typesafe.ai/concepts/system-one) +- RLCD, Reinforcement Learning from Contrastive Distillation — the paper that + introduced the RLCD name: + [arXiv:2307.12950](https://arxiv.org/abs/2307.12950) \ No newline at end of file diff --git a/examples/rlcd/README.md b/examples/rlcd/README.md index 14af5d2d4..5c6180c74 100644 --- a/examples/rlcd/README.md +++ b/examples/rlcd/README.md @@ -5,12 +5,9 @@ endpoint and let it choose the best target per request. ## What you need -- An endpoint that serves a decision model. The most-liked open Jev - counterpart is - [`AlexWortega/openjev`](https://huggingface.co/AlexWortega/openjev), a - Qwen3.5 cross-encoder that scores a task against every candidate option and - returns one probability per option. Any OpenAI-compatible server that - answers the decision prompt with one JSON object works. +- An endpoint that serves a decision model. Any OpenAI-compatible server that + answers the decision prompt with one JSON object — a calibrated probability + per candidate option — works. - Two or more completion targets (the candidates the decision model chooses between). @@ -33,7 +30,7 @@ base_url = "https://openrouter.ai/api/v1" api_key_env = "OPENROUTER_API_KEY" [targets.decision] -id = "Qwen-2.5-1B-RLCD" +id = "decision-model" llm_client = "rlcd" [targets.strong] diff --git a/examples/rlcd/routes.toml b/examples/rlcd/routes.toml index 9c18f1929..1a9be3f55 100644 --- a/examples/rlcd/routes.toml +++ b/examples/rlcd/routes.toml @@ -11,7 +11,7 @@ base_url = "https://openrouter.ai/api/v1" api_key_env = "OPENROUTER_API_KEY" [targets.decision] -id = "AlexWortega/openjev" +id = "decision-model" llm_client = "rlcd" [targets.strong] From b98e6aca8f506469cdba3efbdb0eaa72e65f301b Mon Sep 17 00:00:00 2001 From: Gabriel Amazonas Date: Sat, 19 Sep 2026 17:26:06 -0300 Subject: [PATCH 4/4] docs: cite only the Jev announcement for RLCD and note the acronym collision Signed-off-by: Gabriel Amazonas --- crates/libsy/src/algorithms/rlcd.rs | 7 +++---- docs/routing_algorithms/rlcd_routing.md | 12 ++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/libsy/src/algorithms/rlcd.rs b/crates/libsy/src/algorithms/rlcd.rs index b5e9f6f17..9c08eb0c5 100644 --- a/crates/libsy/src/algorithms/rlcd.rs +++ b/crates/libsy/src/algorithms/rlcd.rs @@ -7,10 +7,9 @@ //! RLCD models map a task and a list of options to one calibrated probability //! per option without writing an answer word by word — the "System One" model //! class TypeSafe's Jev announcement -//! ([docs.typesafe.ai/concepts/system-one](https://docs.typesafe.ai/concepts/system-one)) -//! introduced, trained with Reinforcement Learning for Calibrated Decisions. -//! The RLCD name originates in Reinforcement Learning from Contrastive -//! Distillation ([arXiv:2307.12950](https://arxiv.org/abs/2307.12950)). +//! ([typesafe.ai/blog/introducing-system-one-models-and-jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev)) +//! introduced, trained with its Reinforcement Learning for Calibrated +//! Decisions (RLCD) method. TypeSafe has published no RLCD paper. //! //! [`Rlcd`] builds a decision request that lists every runtime target as a //! candidate option, routes to the option with the highest probability, and diff --git a/docs/routing_algorithms/rlcd_routing.md b/docs/routing_algorithms/rlcd_routing.md index 2f4f16598..58a886d97 100644 --- a/docs/routing_algorithms/rlcd_routing.md +++ b/docs/routing_algorithms/rlcd_routing.md @@ -103,8 +103,12 @@ endpoints broadly support JSON objects but not strict JSON Schema. ## References - TypeSafe's Jev announcement — the "System One" model class trained with - Reinforcement Learning for Calibrated Decisions: + Reinforcement Learning for Calibrated Decisions (RLCD): + [typesafe.ai/blog/introducing-system-one-models-and-jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev), + with the concept overview at [docs.typesafe.ai/concepts/system-one](https://docs.typesafe.ai/concepts/system-one) -- RLCD, Reinforcement Learning from Contrastive Distillation — the paper that - introduced the RLCD name: - [arXiv:2307.12950](https://arxiv.org/abs/2307.12950) \ No newline at end of file + +Throughout Switchyard, RLCD always means TypeSafe's method. It has no public +paper, code, or weights; it shares its acronym with an unrelated 2023 +alignment method, Reinforcement Learning from Contrastive Distillation +([arXiv:2307.12950](https://arxiv.org/abs/2307.12950)). \ No newline at end of file