diff --git a/CHANGELOG.md b/CHANGELOG.md index c6adadd03..0b048f273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,21 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Escalation up-front capability gate** — `escalation.gate = { base_threshold }` + judges the first request of a session with the packaged capability forecaster + and latches to the strong tier when `p_solve` is below the threshold, before + the weak tier is called. Sets the turn-one split with a number instead of a + task-level rule in the judge prompt, which latched almost every multi-file + task in practice. The trajectory judge takes over afterwards. + `gate.classifier_target` lets the once-per-session forecast use a stronger + model than the per-turn trajectory judge. +- **Ordinal confidence scale for the capability forecaster** — `verdict_scale = + "ordinal"` on capability routes and `escalation.gate.verdict_scale` ask the + judge for one of eight named rungs instead of a probability; rungs map to + band midpoints so thresholds keep working. Models rank tasks more reliably + than they number them, and a stated numeric cutoff invites answers just + under it. `escalation.gate.min_confidence` writes the gate's threshold as a + rung, so an ordinal configuration carries no numbers at all. - **Per-target `reasoning_effort`** — a target can force the reasoning effort of every request it serves, replacing the caller's value (`reasoning.effort` on the Responses wire, `reasoning_effort` on Chat Completions), so a strong diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 825e32428..3fa38ca2e 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -6,10 +6,12 @@ use std::sync::Arc; use async_trait::async_trait; +use serde_json::Value; use switchyard_protocol::{ AggLlmResponse, Category, LlmClientError, LlmResponse, Message, Request, Response, Role, }; +use super::llm_class; use super::util::classifier_contract::ClassifierContractConfig; use super::util::decisive; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; @@ -21,6 +23,8 @@ use crate::{LibsyError, Result}; /// Session-state key holding the consecutive-escalate streak. const STREAK_KEY: &str = "escalation_streak"; +/// Session-state key recording that the up-front capability gate has run for this session. +const GATE_KEY: &str = "escalation_gate_done"; fn streak(state: &State) -> u32 { match state.extra.get(STREAK_KEY) { @@ -44,6 +48,8 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// not pay for a second model call. struct EscalationClassifier { judge: JudgeClassifier, + /// Optional capability forecast run once per session before the first efficient call. + gate: Option>>, /// Consecutive escalate verdicts required to latch. confirmations: u32, } @@ -55,7 +61,16 @@ pub(super) fn build_classifier( max_output_tokens: u64, ) -> Result>> { let confirmations = config.confirmations; + let gate = match &config.gate { + Some(gate) => Some(llm_class::build_capability_gate( + gate, + contract_config.response_format_type(), + max_output_tokens, + )?), + None => None, + }; let classifier: Arc> = Arc::new(EscalationClassifier { + gate, judge: escalation::build_judge(&contract_config, config, max_output_tokens)?, confirmations, }); @@ -82,6 +97,62 @@ impl Classifier for EscalationClassifier { return Ok((decisive(&capable), None)); } + // The gate forecasts once per session, from the task framing alone, before the efficient + // tier spends anything. A verdict below the threshold latches immediately; anything else + // (including an unusable verdict or a failed call) falls open to the efficient tier and + // leaves the trajectory judge in charge. + if let Some(gate) = &self.gate + && !matches!(state.extra.get(GATE_KEY), Some(StateValue::Count(_))) + { + state + .extra + .insert(GATE_KEY.to_string(), StateValue::Count(1)); + let mut gate_request = request.clone(); + match gate.score(state, &mut gate_request, driver).await { + Ok((classification, _)) => { + let to_capable = classification + .argmax(false)? + .is_some_and(|score| score.target == capable); + // The forecaster's evidence carries `score` (p_solve) and `threshold`; log it + // so an operator can read the forecast distribution and tune the threshold. + let forecast = driver.evidence().unwrap_or(Value::Null); + tracing::info!( + latched = to_capable, + forecast = %forecast, + "escalation gate verdict" + ); + if to_capable { + state.extra.insert( + STREAK_KEY.to_string(), + StateValue::Count(self.confirmations), + ); + tracing::info!( + target = %capable, + "escalation gate latched the session to the capable tier" + ); + let mut evidence = serde_json::json!({ + "source": "escalation", + "verdict": "gate", + }); + if let (Some(evidence), Some(forecast)) = + (evidence.as_object_mut(), forecast.as_object()) + { + for key in ["score", "threshold"] { + if let Some(value) = forecast.get(key) { + evidence.insert(key.to_string(), value.clone()); + } + } + } + driver.set_evidence(evidence); + return Ok((decisive(&capable), None)); + } + } + Err(error) => { + tracing::warn!(%error, "escalation gate failed; serving the efficient tier"); + } + } + } + // Call efficient model and buffer the response so the judge can read it. // // If the efficient model exceeds its context window, fall through to capable. This call @@ -182,8 +253,10 @@ mod tests { }; use super::*; + use crate::algorithms::llm_class::VerdictScale; use crate::algorithms::llm_class::{LlmClassifierConfig, LlmTaskClassifier}; use crate::algorithms::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; + use crate::algorithms::util::escalation::EscalationGateConfig; use crate::core::testing::{Serve, reply, test_drive_with_models}; /// A queue of replies, drained in order. @@ -381,6 +454,205 @@ mod tests { Ok(()) } + /// Builds a router with the up-front capability gate at the given threshold. + fn gated_router(base_threshold: f64) -> Result> { + Ok(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + gate: Some(EscalationGateConfig { + base_threshold: Some(base_threshold), + min_confidence: None, + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::default(), + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }, + )?)) + } + + const GATE_LOW: &str = r#"{"crux":"protocol framing","primary_rule":"LIM-2","capability_boundary":"unsupported","p_solve":0.2}"#; + const GATE_HIGH: &str = r#"{"crux":"local change","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#; + + #[tokio::test] + async fn gate_latches_before_the_efficient_tier_is_called() -> Result<()> { + // Only the gate verdict is queued for the judge and only one model reply exists: had the + // efficient tier been called first, capable would have received "unexpected call". + let judge = Queue::new([GATE_LOW]); + let model = Queue::new(["capable answer"]); + + let (selected_model, response) = test_drive_with_models( + gated_router(0.5)?, + classify_session_request(), + runtime_models(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "capable"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("capable answer".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn gate_on_the_ordinal_scale_latches_below_the_threshold_rung() -> Result<()> { + let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + gate: Some(EscalationGateConfig { + // Keep `uncertain` and above on the efficient tier; `unlikely` and below latch. + base_threshold: None, + min_confidence: Some("uncertain".to_string()), + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::Ordinal, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + })?); + let judge = Queue::new([ + r#"{"crux":"protocol framing","primary_rule":"LIM-2","capability_boundary":"unsupported","confidence":"unlikely"}"#, + ]); + let model = Queue::new(["capable answer"]); + + let (selected_model, _) = test_drive_with_models( + router, + classify_session_request(), + runtime_models(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "capable"); + Ok(()) + } + + #[tokio::test] + async fn gate_passes_then_trajectory_judge_decides() -> Result<()> { + let judge = Queue::new([GATE_HIGH, r#"{"escalate":false,"reason":"progressing"}"#]); + let model = Queue::new(["efficient answer"]); + + let (selected_model, response) = test_drive_with_models( + gated_router(0.5)?, + classify_session_request(), + runtime_models(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "efficient"); + assert_eq!( + response.llm_response.as_agg().map(completion_text), + Some("efficient answer".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn gate_runs_once_per_session() -> Result<()> { + // Second request: the judge queue holds only an escalate verdict. If the gate ran again + // it would consume that verdict (invalid for the gate, so fail-open) and the trajectory + // judge would then read "unexpected call" and keep efficient. + let judge = Queue::new([ + GATE_HIGH, + r#"{"escalate":false,"reason":"progressing"}"#, + r#"{"escalate":true,"reason":"stuck"}"#, + ]); + let model = Queue::new(["efficient t1", "efficient t2", "capable t2"]); + let router = gated_router(0.5)?; + let request = classify_session_request(); + + let (first, _) = test_drive_with_models( + router.clone(), + request.clone(), + runtime_models(), + queued(Arc::clone(&model), Arc::clone(&judge)), + ) + .await?; + let (second, _) = + test_drive_with_models(router, request, runtime_models(), queued(model, judge)).await?; + + assert_eq!(first, "efficient"); + assert_eq!(second, "capable"); + Ok(()) + } + + #[tokio::test] + async fn unusable_gate_verdict_falls_open_to_efficient() -> Result<()> { + let judge = Queue::new([ + "not a verdict", + r#"{"escalate":false,"reason":"progressing"}"#, + ]); + let model = Queue::new(["efficient answer"]); + + let (selected_model, _) = test_drive_with_models( + gated_router(0.5)?, + classify_session_request(), + runtime_models(), + queued(model, judge), + ) + .await?; + + assert_eq!(selected_model, "efficient"); + Ok(()) + } + + #[test] + fn gate_threshold_is_validated() { + let build = |base_threshold: f64| { + LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + gate: Some(EscalationGateConfig { + base_threshold: Some(base_threshold), + min_confidence: None, + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::default(), + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }) + }; + assert!(build(0.4).is_ok()); + assert!(build(1.5).is_err()); + + let rung = |min_confidence: Option<&str>, base_threshold: Option| { + LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + gate: Some(EscalationGateConfig { + base_threshold, + min_confidence: min_confidence.map(str::to_string), + threshold_step: 0.0, + prompt: None, + classifier_target: None, + verdict_scale: VerdictScale::Ordinal, + }), + ..EscalationJudgeConfig::default() + }, + max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + }) + }; + assert!(rung(Some("uncertain"), None).is_ok()); + assert!(rung(Some("maybe"), None).is_err()); + assert!(rung(Some("uncertain"), Some(0.5)).is_err()); + assert!(rung(None, None).is_err()); + } + #[tokio::test] async fn falls_back_to_capable_when_efficient_overflows() -> Result<()> { let serve = |target: ModelId, _request: Request| async move { diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 4bb5e7ff0..6b6a27c10 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -18,7 +18,9 @@ use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, }; -use super::util::escalation::EscalationJudgeConfig; +use super::util::escalation::{ + self as escalation_util, EscalationGateConfig, EscalationJudgeConfig, +}; use super::util::llm_judge::{ ClassifierInput, JsonSchemaDecoder, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, @@ -32,6 +34,69 @@ use switchyard_protocol::{Request, Response}; const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/schema.json"); +const ORDINAL_PROMPT_TEMPLATE: &str = + include_str!("../prompts/capability-classifier/prompt-ordinal.md"); +const ORDINAL_SCHEMA_TEMPLATE: &str = + include_str!("../prompts/capability-classifier/schema-ordinal.json"); + +/// How the capability forecaster reports its verdict. +/// +/// Models produce ordinal judgements more reliably than calibrated numbers: asked for a +/// probability they cluster on a few round values and, given a stated cutoff, write a number +/// just under it. The ordinal scale asks for one rung of a fixed ladder and maps it to the +/// band's midpoint, so the threshold policy is unchanged and a threshold between two rungs +/// selects exactly the rungs below it. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VerdictScale { + /// `p_solve` as a number in `[0, 1]`. + #[default] + Probability, + /// `confidence` as one of eight named rungs, from `surely` to `almost_surely_not`. + Ordinal, +} + +impl VerdictScale { + fn templates(self) -> (&'static str, &'static str) { + match self { + Self::Probability => (PROMPT_TEMPLATE, SCHEMA_TEMPLATE), + Self::Ordinal => (ORDINAL_PROMPT_TEMPLATE, ORDINAL_SCHEMA_TEMPLATE), + } + } +} + +/// Ladder rungs and the midpoint of the natural-frequency band each one names. +const CONFIDENCE_RUNGS: [(&str, f64); 8] = [ + ("surely", 0.97), + ("extremely_likely", 0.90), + ("very_likely", 0.78), + ("likely", 0.62), + ("uncertain", 0.50), + ("unlikely", 0.38), + ("very_unlikely", 0.22), + ("almost_surely_not", 0.08), +]; + +fn rung_midpoint(rung: &str) -> Option { + CONFIDENCE_RUNGS + .iter() + .find(|(name, _)| *name == rung) + .map(|(_, midpoint)| *midpoint) +} + +/// The numeric threshold that keeps `rung` and every rung above it on the efficient tier and +/// sends every rung below it to the capable tier: halfway between `rung`'s midpoint and the +/// next lower rung's (or halfway to zero for the lowest rung). `None` for an unknown rung. +pub(crate) fn rung_threshold(rung: &str) -> Option { + let index = CONFIDENCE_RUNGS + .iter() + .position(|(name, _)| *name == rung)?; + let lower = CONFIDENCE_RUNGS + .get(index + 1) + .map(|(_, midpoint)| *midpoint) + .unwrap_or(0.0); + Some((CONFIDENCE_RUNGS[index].1 + lower) / 2.0) +} /// Telemetry label for this algorithm's spans, metrics, and logs. const ALGORITHM_NAME: &str = "llm_task_classifier"; @@ -52,13 +117,28 @@ struct TaskClassifierVerdict { crux: String, primary_rule: String, capability_boundary: String, - p_solve: f64, + /// Probability scale: the forecast as a number. + #[serde(default)] + p_solve: Option, + /// Ordinal scale: the forecast as a ladder rung. Exactly one of the two is present. + #[serde(default)] + confidence: Option, } impl TaskClassifierVerdict { + /// The forecast as a number on either scale: `p_solve` itself, or the midpoint of the + /// band the reported rung names. + fn p_solve(&self) -> Option { + match (self.p_solve, self.confidence.as_deref()) { + (Some(p), None) => Some(p), + (None, Some(rung)) => rung_midpoint(rung), + _ => None, + } + } + /// Rejects malformed or internally inconsistent verdicts before policy evaluation. fn is_valid(&self) -> bool { - (0.0..=1.0).contains(&self.p_solve) + self.p_solve().is_some_and(|p| (0.0..=1.0).contains(&p)) && !self.crux.trim().is_empty() && matches!( ( @@ -233,9 +313,9 @@ impl JudgePolicy for TaskClassifierPolicy { let Some(threshold) = self.threshold(verdict) else { return Ok(Classification::Ambiguous(vec![])); }; - let category = if verdict.p_solve >= threshold - || (threshold - verdict.p_solve).abs() <= f64::EPSILON - { + // `is_valid` guarantees the forecast is present on exactly one scale. + let p_solve = verdict.p_solve().unwrap_or(0.0); + let category = if p_solve >= threshold || (threshold - p_solve).abs() <= f64::EPSILON { Category::Efficient } else { Category::Capable @@ -269,16 +349,23 @@ fn capability_evidence( "reason_code": "invalid_verdict", })); }; - Some(serde_json::json!({ + let mut evidence = serde_json::json!({ "source": "llm-classifier", - "score": verdict.p_solve, + "score": verdict.p_solve(), "threshold": threshold, - })) + }); + if let (Some(evidence), Some(rung)) = (evidence.as_object_mut(), verdict.confidence.as_deref()) + { + evidence.insert("confidence".to_string(), Value::String(rung.to_string())); + } + Some(evidence) } #[derive(Clone, Debug)] /// Settings that control capability classifier prompting and routing. pub struct TaskClassifierConfig { + /// Whether the forecaster reports a probability or a ladder rung. + pub verdict_scale: VerdictScale, /// Lowest solve probability that routes a supported task to the efficient target. pub base_threshold: f64, /// Amount added per capability-boundary step. @@ -322,6 +409,8 @@ struct TaskClassifierConfigWire { response_format_type: ClassifierResponseFormat, #[serde(default = "default_judge_max_output_tokens")] max_output_tokens: u64, + #[serde(default)] + verdict_scale: VerdictScale, } impl<'de> Deserialize<'de> for TaskClassifierConfig { @@ -336,6 +425,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { } contract = contract.with_response_format_type(wire.response_format_type); Ok(Self { + verdict_scale: wire.verdict_scale, base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, classify_trigger: wire.classify_trigger, @@ -354,6 +444,7 @@ const fn default_judge_max_output_tokens() -> u64 { impl Default for TaskClassifierConfig { fn default() -> Self { Self { + verdict_scale: VerdictScale::default(), base_threshold: 0.0, threshold_step: 0.0, classify_trigger: ClassifyTrigger::default(), @@ -604,7 +695,7 @@ impl LlmTaskClassifier { fn build_capability(config: TaskClassifierConfig) -> Result { config.validate()?; - let contract = Self::load_capability_contract(&config.contract)?; + let contract = Self::load_capability_contract(&config.contract, config.verdict_scale)?; let classify_trigger = config.classify_trigger; let message_hash_fallback = config.message_hash_fallback; let classifier: Arc> = Arc::new( @@ -682,9 +773,13 @@ impl LlmTaskClassifier { }) } - /// Loads the packaged capability-classifier contract. - fn load_capability_contract(config: &ClassifierContractConfig) -> Result { - ClassifierContract::from_config(config, PROMPT_TEMPLATE, SCHEMA_TEMPLATE) + /// Loads the packaged capability-classifier contract for the configured verdict scale. + fn load_capability_contract( + config: &ClassifierContractConfig, + scale: VerdictScale, + ) -> Result { + let (prompt, schema) = scale.templates(); + ClassifierContract::from_config(config, prompt, schema) } /// Keeps affinity and fallback ordering identical across judge-backed modes. @@ -718,6 +813,48 @@ impl LlmTaskClassifier { } } +/// Builds the up-front capability gate an escalation route runs once per session: the packaged +/// capability forecaster, judged through the escalation judge's target, mapped to a tier by the +/// same threshold policy capability mode uses. The gate reads the task framing only +/// (`recent_turn_window` unset), so it works before any trajectory exists. +pub(super) fn build_capability_gate( + gate: &EscalationGateConfig, + response_format_type: ClassifierResponseFormat, + max_output_tokens: u64, +) -> Result>> { + let mut contract_config = + ClassifierContractConfig::default().with_response_format_type(response_format_type); + if let Some(prompt) = &gate.prompt { + contract_config = contract_config.with_prompt(prompt.clone()); + } + let (prompt, schema) = gate.verdict_scale.templates(); + let contract = ClassifierContract::from_config(&contract_config, prompt, schema)?; + let config = TaskClassifierConfig { + verdict_scale: gate.verdict_scale, + base_threshold: gate.threshold()?, + threshold_step: gate.threshold_step, + ..TaskClassifierConfig::default() + }; + let mut classifier = JudgeClassifier::new( + StructuredJudge::new( + TaskInput { + recent_turn_window: None, + }, + contract, + SerdeDecoder::new(), + JudgeRuntimeConfig::new(max_output_tokens)?, + ), + TaskClassifierPolicy::new(&config), + ) + .with_evidence(capability_evidence); + if gate.classifier_target.is_some() { + classifier = classifier.with_judge_category(Category::Named(Arc::from( + escalation_util::GATE_JUDGE_CATEGORY, + ))); + } + Ok(Arc::new(classifier)) +} + #[async_trait] impl Classifier for LlmTaskClassifier { async fn score( @@ -765,6 +902,52 @@ mod tests { const TEST_THRESHOLD: f64 = 0.5; + #[test] + fn ordinal_verdicts_map_to_band_midpoints_and_probability_verdicts_pass_through() { + let ordinal: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"framing","primary_rule":"LIM-2","capability_boundary":"unsupported","confidence":"unlikely"}"#, + ) + .expect("ordinal verdict parses"); + assert!(ordinal.is_valid()); + assert_eq!(ordinal.p_solve(), Some(0.38)); + + let probability: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"local change","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + ) + .expect("probability verdict parses"); + assert_eq!(probability.p_solve(), Some(0.9)); + + // An unknown rung, or both scales at once, is not a usable verdict. + let unknown: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"x","primary_rule":"SUP-1","capability_boundary":"supported","confidence":"maybe"}"#, + ) + .expect("parses"); + assert!(!unknown.is_valid()); + let both: TaskClassifierVerdict = serde_json::from_str( + r#"{"crux":"x","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.5,"confidence":"likely"}"#, + ) + .expect("parses"); + assert!(!both.is_valid()); + } + + #[test] + fn rung_thresholds_sit_between_adjacent_rungs() { + // Keeping `uncertain` (0.50) and above means latching `unlikely` (0.38) and below. + let threshold = rung_threshold("uncertain").expect("known rung"); + assert!(threshold < 0.50 && threshold > 0.38); + assert!(rung_threshold("almost_surely_not").is_some_and(|t| t > 0.0 && t < 0.08)); + assert!(rung_threshold("maybe").is_none()); + } + + #[test] + fn ordinal_scale_loads_the_ladder_prompt_and_schema() { + let (prompt, schema) = VerdictScale::Ordinal.templates(); + assert!(prompt.contains("almost_surely_not") && !prompt.contains("p_solve")); + assert!(schema.contains("\"confidence\"") && !schema.contains("p_solve")); + let (prompt, schema) = VerdictScale::Probability.templates(); + assert!(prompt.contains("p_solve") && schema.contains("p_solve")); + } + type CapabilityJudge = StructuredJudge>; fn test_config(base_threshold: f64) -> TaskClassifierConfig { @@ -804,7 +987,8 @@ mod tests { crux: "test crux".to_string(), primary_rule: primary_rule.to_string(), capability_boundary: capability_boundary.to_string(), - p_solve, + p_solve: Some(p_solve), + confidence: None, } } @@ -1318,7 +1502,10 @@ mod tests { fn capability_judge(recent_turn_window: Option) -> Result { Ok(StructuredJudge::new( TaskInput { recent_turn_window }, - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, + LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, )) @@ -1693,8 +1880,10 @@ mod tests { /// rejecting every production verdict. #[test] fn every_schema_property_round_trips_through_the_judge_parser() -> Result<()> { - let contract = - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?; + let contract = LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?; let schema = contract.response_format(); let reply = schema_shaped_verdict(schema)?; let judge: CapabilityJudge = StructuredJudge::new( @@ -1709,14 +1898,16 @@ mod tests { let verdict = judge.parse(&text_response(None, reply))?; assert!(verdict.is_valid()); - assert!((0.0..=1.0).contains(&verdict.p_solve)); + assert!(verdict.p_solve().is_some_and(|p| (0.0..=1.0).contains(&p))); Ok(()) } #[test] fn packaged_prompt_keeps_the_schema_in_the_structured_request() -> Result<()> { - let contract = - LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?; + let contract = LlmTaskClassifier::load_capability_contract( + &ClassifierContractConfig::default(), + VerdictScale::Probability, + )?; let prompt = contract.system_prompt(); let schema_name = contract .response_format() diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b0979e286..2716f915e 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,6 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use crate::algorithms::llm_class::{self, VerdictScale}; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; @@ -61,6 +62,100 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Optional up-front capability gate. When set, the first request of a session is judged + /// from the task framing alone with the packaged capability forecaster, and a solve + /// probability below the threshold latches the session to the capable tier before the + /// efficient tier has spent anything. The trajectory judge takes over afterwards. + pub gate: Option, +} + +/// Runtime model category the gate's forecaster is taken from when `gate.classifier_target` is +/// set. The deployment loader registers that target under this named category; unset, the gate +/// shares the route's `Judge` category. +pub const GATE_JUDGE_CATEGORY: &str = "escalation_gate_judge"; + +/// Numeric threshold for the escalation route's up-front capability gate. +/// +/// Prose in the trajectory-judge prompt cannot set a split reliably: a task-level bar worded +/// as "spans several modules" or "the hardest minority" latches almost every multi-file task. +/// The gate reuses the capability classifier's forecast (`p_solve`) and threshold policy so the +/// operator dials the split with a number, the same way capability mode does. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EscalationGateConfig { + /// Lowest solve probability that keeps the session on the efficient tier. In `[0, 1]`. + /// Exactly one of `base_threshold` and `min_confidence` must be set. + #[serde(default)] + pub base_threshold: Option, + /// The threshold as a ladder rung: the efficient tier keeps this rung and every rung above + /// it, and every rung below latches to the capable tier. Pairs naturally with + /// `verdict_scale = "ordinal"`, so an ordinal configuration carries no numbers at all. + #[serde(default)] + pub min_confidence: Option, + /// Added once for uncertain or unmatched verdicts and twice for unsupported verdicts, as in + /// capability mode. `base_threshold + 2 * threshold_step` must be at most `1`. + #[serde(default)] + pub threshold_step: f64, + /// Replaces the packaged capability-classifier prompt for the gate call only. + #[serde(default)] + pub prompt: Option, + /// Target the gate forecast is called through, by target name. Defaults to the route's + /// `classifier_target`. The gate runs once per session and benefits from a strong forecaster, + /// while the trajectory judge runs on every weak turn and is better served by a cheap model; + /// naming them separately lets a deployment pay for each where it matters. Resolved by the + /// deployment loader, which is why it is a name here rather than a model id. + #[serde(default)] + pub classifier_target: Option, + /// Whether the forecaster reports `p_solve` as a number (default) or `confidence` as one + /// of eight ladder rungs, mapped to band midpoints before the threshold is applied. + #[serde(default)] + pub verdict_scale: VerdictScale, +} + +impl EscalationGateConfig { + /// The numeric threshold the gate applies, from whichever form the operator wrote. + pub(crate) fn threshold(&self) -> Result { + let reject = |message: String| Err(LibsyError::AlgorithmError { message }); + match (self.base_threshold, self.min_confidence.as_deref()) { + (Some(threshold), None) => Ok(threshold), + (None, Some(rung)) => llm_class::rung_threshold(rung).ok_or_else(|| { + LibsyError::AlgorithmError { + message: format!( + "gate.min_confidence must be a ladder rung (surely, extremely_likely, very_likely, likely, uncertain, unlikely, very_unlikely, almost_surely_not), got {rung:?}" + ), + } + }), + (Some(_), Some(_)) => reject( + "gate.base_threshold and gate.min_confidence cannot both be set".to_string(), + ), + (None, None) => reject( + "gate needs base_threshold or min_confidence".to_string(), + ), + } + } + + fn validate(&self) -> Result<()> { + let reject = |message: String| Err(LibsyError::AlgorithmError { message }); + let threshold = self.threshold()?; + if !(0.0..=1.0).contains(&threshold) { + return reject(format!( + "gate.base_threshold must be between 0 and 1, got {threshold}" + )); + } + if !self.threshold_step.is_finite() || self.threshold_step < 0.0 { + return reject(format!( + "gate.threshold_step must be finite and non-negative, got {}", + self.threshold_step + )); + } + let unsupported_threshold = threshold + 2.0 * self.threshold_step; + if unsupported_threshold > 1.0 { + return reject(format!( + "gate.base_threshold + 2 * gate.threshold_step must be at most 1, got {unsupported_threshold}" + )); + } + Ok(()) + } } impl EscalationJudgeConfig { @@ -70,6 +165,9 @@ impl EscalationJudgeConfig { if self.confirmations == 0 { return reject("confirmations must be at least 1".to_string()); } + if let Some(gate) = &self.gate { + gate.validate()?; + } if self.recent_turn_window == 0 { return reject("recent_turn_window must be at least 1".to_string()); } @@ -89,6 +187,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + gate: None, } } } diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index f12c7a0dd..73e899071 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -217,6 +217,9 @@ where judge: J, policy: P, evidence: Option>, + /// Runtime category the judge models are taken from; `Judge` unless a caller needs a + /// second judge alongside the route's own (the escalation gate). + judge_category: Category, } impl JudgeClassifier @@ -230,9 +233,16 @@ where judge, policy, evidence: None, + judge_category: Category::Judge, } } + /// Takes the judge models from `category` instead of [`Category::Judge`]. + pub(crate) fn with_judge_category(mut self, category: Category) -> Self { + self.judge_category = category; + self + } + /// Enables bounded evidence for built-in judges without widening the public policy trait. pub(crate) fn with_evidence(mut self, evidence: EvidenceFn) -> Self { self.evidence = Some(evidence); @@ -242,7 +252,7 @@ where /// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision. fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) { let judge_target = driver - .first_model_for(&Category::Judge) + .first_model_for(&self.judge_category) .map(|c| c.as_str()) .unwrap_or("missing"); report_fail_open(judge_target, error, reason); @@ -347,10 +357,13 @@ where request: &mut Request, driver: &Driver, ) -> Result<(Classification, Option)> { - let judge_models = driver.models_for(&Category::Judge); + let judge_models = driver.models_for(&self.judge_category); if judge_models.is_empty() { return Err(LibsyError::AlgorithmError { - message: "no models available for category Judge".to_string(), + message: format!( + "no models available for category {}", + self.judge_category.as_str() + ), }); } let verdict = self.verdict(state, request, driver, judge_models).await; diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index e2ca8e488..676eb5689 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -226,6 +226,12 @@ impl Driver { *self.evidence.lock() = Some(evidence); } + /// Returns a copy of the evidence recorded so far, so a component that wraps another can + /// read the inner decision back (for example to log a judge's score and threshold). + pub(crate) fn evidence(&self) -> Option { + self.evidence.lock().clone() + } + /// Supply fallback evidence without replacing a decision made earlier in the cascade. pub(crate) fn set_evidence_if_empty(&self, evidence: Value) { let mut current = self.evidence.lock(); diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 6e84c456a..6066cb720 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -21,7 +21,7 @@ pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger}; pub use algorithms::composite::{CompositeRouter, CompositeRouterConfig}; pub use algorithms::llm_class::{ CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier, - TaskClassifierConfig, + TaskClassifierConfig, VerdictScale, }; pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; @@ -33,6 +33,7 @@ pub use algorithms::util::classifier_contract::{ ClassifierContractConfig, ClassifierResponseFormat, }; pub use algorithms::util::escalation::EscalationJudgeConfig; +pub use algorithms::util::escalation::GATE_JUDGE_CATEGORY; pub use algorithms::util::prompts::append_note; pub use algorithms::util::subagent::{SubagentGate, SubagentOverride}; pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignals}; diff --git a/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md b/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md new file mode 100644 index 000000000..9929d83b4 --- /dev/null +++ b/crates/libsy/src/prompts/capability-classifier/prompt-ordinal.md @@ -0,0 +1,69 @@ +You are a task-level probability forecaster for a model router. You receive the +task's opening instruction and, when present, its latest user follow-up, plus +the qualitative capability card below. + +Forecast one binary event: + +SUCCESS means that the efficient agent completes the whole task correctly on +one fresh run under the actual harness, tools, and budget, as judged by the +final verifier. FAILURE means any other outcome. The two outcomes are +exhaustive. + +Use only evidence in the instruction and the capability card. Do not assume +hidden repository state, unmentioned tools, validators, documentation, access, +or future work habits. Do not invent empirical counts, success rates, or base +rates. The capability card is qualitative evidence, not a measured prior. + +# Assessment procedure + +1. State the crux: the hardest material requirement for whole-task success. +2. Select the one capability rule that best describes the crux. Use + primary_rule=none and capability_boundary=unmatched when no rule applies. + Rule ids are opaque labels. Do not infer a boundary from an id's spelling. +3. Privately identify the strongest instruction-visible reasons for SUCCESS + and FAILURE, then imagine the most likely concrete failure. +4. Privately consider material unknowns. Missing information should limit + extreme rungs, but it is not evidence that the answer must be uncertain. +5. Choose the rung last. It describes the chance of whole-task SUCCESS, not + confidence in this assessment, a route recommendation, or a cost judgment. + +Report the forecast as one rung of a fixed ladder rather than a number. Each +rung names a band of natural frequencies: over 100 comparable fresh runs, + +- surely: 95 or more succeed +- extremely_likely: 85 to 95 +- very_likely: 70 to 85 +- likely: 55 to 70 +- uncertain: 45 to 55 +- unlikely: 30 to 45 +- very_unlikely: 15 to 30 +- almost_surely_not: fewer than 15 + +Use the whole ladder when justified. Supported does not mean surely, and +unsupported does not mean almost_surely_not. The downstream routing threshold +is not part of this forecast. + +# Efficient-agent capability card + +The route verbs in this source card are inherited qualitative descriptions. +They do not ask you to output a route and do not assign a fixed probability to +any boundary. + +- SUP-1 [supported]: Route to the Efficient model when the task provides a complete output contract and a deterministic local validator that covers the material requirements. +- SUP-2 [supported]: Route to the Efficient model when all required inputs are available, the target environment can be inspected, and correctness can be verified end-to-end without inaccessible external state. +- SUP-3 [supported]: Route to the Efficient model when mathematical behavior, interfaces, shapes, data types, tolerances, and performance requirements are explicit and exercised by a representative harness. +- SUP-4 [supported]: Route to the Efficient model when the required mechanism is identified, the relevant search space is bounded, and the success condition is executable. Do not infer this rule merely from the task's technical domain. +- SUP-5 [supported]: Route to the Efficient model when reconstruction or behavioral reproduction is constrained by an executable reference, parser, format specification, or checker strong enough to distinguish correct from merely plausible output. +- UNC-1 [uncertain]: Treat the route as uncertain when multiple reasonable interpretations of preprocessing, representation, indexing, naming, or output placement would produce different results and neither the instructions nor a validator resolve the choice. +- UNC-2 [uncertain]: Treat the route as uncertain when success requires finding every relevant item across heterogeneous inputs or environment state, but the task does not define the search boundary or provide a completeness check. +- LIM-1 [unsupported]: Prefer the Capable model when correctness depends primarily on extracting precise information from noisy visual, temporal, or rendered media and no machine-checkable extraction or replay mechanism is available. +- LIM-2 [unsupported]: Prefer the Capable model when success depends on reproducing undocumented reference behavior, hidden intermediate state, or an unknown configuration, and small deviations fail despite satisfying the visible specification. + +# Output + +Return exactly one JSON object matching the response schema supplied with the +request. Do not include markdown or commentary. + +confidence must be one of the ladder rungs above, spelled exactly as listed. +Do not output a probability, recommended_route, abstain, counts, task totals, +empirical rates, or any other field. diff --git a/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json b/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json new file mode 100644 index 000000000..5d2a6099e --- /dev/null +++ b/crates/libsy/src/prompts/capability-classifier/schema-ordinal.json @@ -0,0 +1,52 @@ +{ + "type": "json_schema", + "json_schema": { + "name": "CapabilityClassifierOrdinalDecision", + "strict": true, + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "crux", + "primary_rule", + "capability_boundary", + "confidence" + ], + "properties": { + "crux": {"type": "string", "minLength": 1}, + "primary_rule": { + "type": "string", + "enum": [ + "SUP-1", + "SUP-2", + "SUP-3", + "SUP-4", + "SUP-5", + "UNC-1", + "UNC-2", + "LIM-1", + "LIM-2", + "none" + ] + }, + "capability_boundary": { + "type": "string", + "enum": ["supported", "uncertain", "unsupported", "unmatched"] + }, + "confidence": { + "type": "string", + "enum": [ + "surely", + "extremely_likely", + "very_likely", + "likely", + "uncertain", + "unlikely", + "very_unlikely", + "almost_surely_not" + ] + } + } + } + } +} diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index eea067f41..ff1329e71 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -16,7 +16,7 @@ use switchyard_libsy::{ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter, StageRouterConfig, - Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, + Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, VerdictScale, }; use switchyard_protocol::{ Category, LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, @@ -133,6 +133,8 @@ impl PyEscalationClassifierConfig { confirmations, recent_turn_window, window_message_chars, + // The up-front capability gate is not exposed to Python yet. + gate: None, }, max_output_tokens, }) @@ -278,6 +280,8 @@ impl PyTaskClassifierConfig { ) -> PyResult { Ok(Self { inner: TaskClassifierConfig { + // The ordinal confidence scale is not exposed to Python yet. + verdict_scale: VerdictScale::default(), base_threshold, threshold_step, classify_trigger: classify_trigger(session_affinity), diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 626c150af..fe0df3810 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -12,10 +12,10 @@ use std::sync::Arc; use libsy::{ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, - CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, - StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, - ToolSemantics, + CustomClassifierPolicy, EscalationJudgeConfig, GATE_JUDGE_CATEGORY, GateTrigger, + HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, + PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, + TaskClassifierConfig, ToolSemantics, VerdictScale, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -109,6 +109,7 @@ struct CapabilityClassifierRouteConfig { prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, + verdict_scale: VerdictScale, } #[derive(Clone, Debug)] @@ -253,6 +254,10 @@ pub struct LlmClassifierRouteConfig { /// Most completion tokens the judge verdict may use. #[serde(default = "default_classifier_max_output_tokens")] pub max_output_tokens: u64, + /// Capability mode: whether the forecaster reports `p_solve` as a probability or + /// `confidence` as one rung of the fixed ladder (the gate has its own `gate.verdict_scale`). + #[serde(default)] + pub verdict_scale: VerdictScale, /// Escalation mode: how many escalate verdicts latch the session, and how /// much of the transcript the judge sees. pub escalation: Option, @@ -511,6 +516,7 @@ impl StageClassifierConfig { contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, + verdict_scale: VerdictScale::default(), } } } @@ -608,6 +614,14 @@ impl AlgorithmSpec { ); } else { names.push(&config.classifier_target); + if let Some(gate_target) = config + .escalation + .as_ref() + .and_then(|escalation| escalation.gate.as_ref()) + .and_then(|gate| gate.classifier_target.as_deref()) + { + names.push(gate_target); + } } } Self::StageRouter { @@ -787,15 +801,31 @@ fn classifier_runtime_model_names( vec![config.weak_target, config.strong_target], ), ]), - LlmClassifierModeConfig::Escalation(config) => category_models([ - (Category::Judge, vec![config.classifier_target]), - (Category::Efficient, vec![config.weak_target.clone()]), - (Category::Capable, vec![config.strong_target.clone()]), - ( - Category::Any, - vec![config.strong_target, config.weak_target], - ), - ]), + LlmClassifierModeConfig::Escalation(config) => { + let mut models = category_models([ + (Category::Judge, vec![config.classifier_target]), + (Category::Efficient, vec![config.weak_target.clone()]), + (Category::Capable, vec![config.strong_target.clone()]), + ( + Category::Any, + vec![config.strong_target, config.weak_target], + ), + ]); + // The gate may name its own forecaster; it lives under a named category so the + // per-turn judge and the once-per-session gate can be different models. + if let Some(gate_target) = config + .judge + .gate + .as_ref() + .and_then(|gate| gate.classifier_target.clone()) + { + models.insert( + Category::Named(Arc::from(GATE_JUDGE_CATEGORY)), + vec![gate_target], + ); + } + models + } LlmClassifierModeConfig::Custom(config) => custom_runtime_model_names(&config.models), } } @@ -847,6 +877,7 @@ impl LlmClassifierRouteConfig { prompt, response_format_type, max_output_tokens, + verdict_scale, escalation, models, default_target, @@ -902,6 +933,7 @@ impl LlmClassifierRouteConfig { prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, + verdict_scale: *verdict_scale, }, )) } @@ -1179,6 +1211,7 @@ fn build_algorithm( contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, + verdict_scale: config.verdict_scale, }; LlmTaskClassifier::new(LlmClassifierConfig::Capability { config: classifier_config, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 316754723..e847dfab1 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1109,6 +1109,24 @@ new = ["send_message"] Ok(()) } + /// The gate may name its own judge target; it is resolved like `classifier_target` and an + /// unknown name is rejected at load time. + #[test] + fn an_escalation_gate_can_name_its_own_judge_target() -> RunnerResult<()> { + let gated = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1, gate = { base_threshold = 0.4, classifier_target = \"strong\" } }", + ); + runner_from_toml(&gated)?; + + let unknown = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1, gate = { base_threshold = 0.4, classifier_target = \"nobody\" } }", + ); + assert!(error_message(&unknown).contains("nobody")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7d4b8b914..856bea7c1 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -203,6 +203,7 @@ Capability mode classifies before serving. See | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | +| `verdict_scale` | No | `probability` | `probability` asks for `p_solve`; `ordinal` asks for `confidence` as one of eight named rungs mapped to band midpoints, so `base_threshold` selects rungs. See `escalation.gate.verdict_scale` for the ladder. | Escalation mode serves the weak target first and judges the completed turn. See [Escalation-Router Routing](../routing_algorithms/escalation_router_routing.md). @@ -215,6 +216,13 @@ Escalation mode serves the weak target first and judges the completed turn. See | `escalation.confirmations` | No | `2` | Consecutive escalate verdicts required to latch. Above `1` needs a session ID. | | `escalation.recent_turn_window` | No | `28` | Trailing messages shown to the judge. | | `escalation.window_message_chars` | No | `500` | Per-message cap inside that window. | +| `escalation.gate` | No | unset | Up-front capability gate: judges the first request of a session from the task framing with the packaged capability forecaster and latches to `strong_target` when `p_solve` is below the threshold, before `weak_target` is called. Needs a session ID. | +| `escalation.gate.base_threshold` | One of the two, when `gate` is set | — | Lowest solve probability that keeps the session on `weak_target`. In `[0, 1]`. | +| `escalation.gate.min_confidence` | One of the two, when `gate` is set | — | The threshold as a ladder rung: `weak_target` keeps this rung and every rung above it, every rung below latches to `strong_target`. Use with `verdict_scale = "ordinal"` for a configuration with no numbers. | +| `escalation.gate.threshold_step` | No | `0.0` | Added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `escalation.gate.prompt` | No | packaged capability prompt | Replaces the capability-forecaster prompt for the gate call only. | +| `escalation.gate.classifier_target` | No | the route's `classifier_target` | Target the once-per-session gate forecast is called through, so a strong forecaster can gate while a cheap model judges every weak turn. | +| `escalation.gate.verdict_scale` | No | `probability` | `probability` asks the forecaster for `p_solve` in `[0, 1]`; `ordinal` asks for `confidence` as one of eight rungs (`surely`, `extremely_likely`, `very_likely`, `likely`, `uncertain`, `unlikely`, `very_unlikely`, `almost_surely_not`), mapped to band midpoints (0.97, 0.90, 0.78, 0.62, 0.50, 0.38, 0.22, 0.08) before the threshold applies. | Existing configurations that contain `escalation` but omit `mode` remain valid. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..e1779abd0 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -90,6 +90,54 @@ A judge that times out, errors, or returns an unparseable verdict fails open: th turn serves the buffered weak reply and the existing streak is held rather than cleared. A judge failure never creates a strong-tier latch. +## Up-front capability gate + +The trajectory judge only sees trouble once it has happened, and a hand-off +mid-task makes the strong tier redo the work, so for tasks that were never going +to fit the weak tier the cheapest moment to escalate is the first request. +`escalation.gate` adds that moment as a numeric threshold: + +```toml +escalation = { confirmations = 1, gate = { base_threshold = 0.4 } } +``` + +On the first request of a session the route calls the judge with the packaged +capability-classifier prompt, reading the task framing alone, and applies the +capability threshold policy to its `p_solve` forecast. Below `base_threshold` +the session latches to `strong_target` immediately and the weak tier is never +called. At or above it the request proceeds as usual and the trajectory judge +takes over for the rest of the session. The gate runs once per session, so it +needs a session ID, like `confirmations` above `1`. An unusable gate verdict or +a failed gate call falls open to the weak tier. + +Why a number rather than a task-level rule in the judge prompt: in practice a +prose bar ("spans several modules", "the hardest minority of tasks") latches +almost every multi-file task, because the judge has no reference distribution +to calibrate against. A threshold on a forecast is what capability mode already +uses to set its split, and the gate reuses that policy unchanged, including +`threshold_step` for uncertain and unsupported verdicts and `prompt` to replace +the packaged forecaster prompt. `gate.classifier_target` lets the gate call a +different target from the per-turn judge: the forecast runs once per session +and rewards a strong model, while the trajectory judge runs on every weak turn +and is where a cheap model belongs. + +`gate.verdict_scale = "ordinal"` asks the forecaster for one rung of a fixed +ladder (`surely` down to `almost_surely_not`) instead of a probability. Models +produce ordinal judgements more reliably than calibrated numbers: asked for a +probability they cluster on a few round values, and given a stated cutoff they +write a number just under it. Each rung maps to the midpoint of the frequency +band it names, so `base_threshold` still selects the split. Better, write the +threshold as a rung too: + +```toml +escalation = { confirmations = 1, gate = { verdict_scale = "ordinal", min_confidence = "uncertain" } } +``` + +keeps `uncertain` and everything above it on the weak tier and latches +`unlikely` and below, with no number anywhere in the prompt, the answer or the +configuration. The chosen rung is recorded in the evidence next to the score. The gate verdict is recorded in the route's +evidence as `{"source": "escalation", "verdict": "gate"}`. + ## Judge model compatibility The trajectory judge uses the same response contract and provider/model