From 6f013037bdf812a9288d2c1058613905785e0c90 Mon Sep 17 00:00:00 2001 From: Graham King Date: Fri, 18 Sep 2026 15:58:20 -0400 Subject: [PATCH 1/2] fix(stage): recognize failed Responses patch and shell outputs Stage Router ignored failures reported by OpenAI Responses built-in patch and shell tools. Repeated failures could leave recovery work on the efficient model instead of switching to the capable model. Recognize failed patches, nonzero shell exit codes, and shell timeouts. Also check shell output text for errors. Keep the original tool history unchanged. Add one regression test covering failures and successful results. Assisted-by: Pi:GPT 6 Astra medium Reviewed-by: Pi:GLM 5.3 high Signed-off-by: Graham King --- .../libsy/src/algorithms/util/tool_signals.rs | 160 +++++++++++++++++- 1 file changed, 156 insertions(+), 4 deletions(-) diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index ba469cfdf..bb30934d6 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -17,7 +17,7 @@ use std::path::Path; use async_trait::async_trait; use serde::Deserialize; use serde_json::Value; -use switchyard_protocol::{ContentBlock, Request, Role}; +use switchyard_protocol::{ContentBlock, Request, Role, WireFormat}; use crate::{LibsyError, Result}; @@ -704,9 +704,7 @@ fn extract_tool_signals_with_window_and_semantics( recent_window: usize, semantics: &ToolSemantics, ) -> ToolSignals { - // Read the decoded conversation, not the raw body: every inbound format lands - // in the same shape here, so the signals do not depend on knowing which one it - // arrived as. + // Read the decoded conversation, including preserved built-in tool outputs. let messages = &request.llm_request.messages; let mut tool_texts: Vec<(String, bool)> = Vec::new(); let mut tool_calls: Vec = Vec::new(); @@ -741,6 +739,64 @@ fn extract_tool_signals_with_window_and_semantics( tool_texts.push((text, is_error)); } } + // Built-in tool history stays opaque so it can be replayed unchanged. + ContentBlock::Unknown { provider, raw } + if provider.as_str() == WireFormat::OpenAiResponses.as_str() + && raw.get("type").and_then(Value::as_str) + == Some("apply_patch_call_output") => + { + tool_result_count += 1; + let text = raw + .get("output") + .and_then(Value::as_str) + .unwrap_or_default(); + let is_error = raw.get("status").and_then(Value::as_str) == Some("failed"); + if !text.is_empty() || is_error { + tool_texts.push((text.to_owned(), is_error)); + } + } + ContentBlock::Unknown { provider, raw } + if provider.as_str() == WireFormat::OpenAiResponses.as_str() + && raw.get("type").and_then(Value::as_str) == Some("shell_call_output") => + { + tool_result_count += 1; + let mut texts = Vec::new(); + let mut is_error = false; + if let Some(outputs) = raw.get("output").and_then(Value::as_array) { + for output in outputs { + for field in ["stdout", "stderr"] { + if let Some(text) = output.get(field).and_then(Value::as_str) + && !text.is_empty() + { + texts.push(text); + } + } + if let Some(outcome) = output.get("outcome") { + is_error |= match outcome.get("type").and_then(Value::as_str) { + Some("timeout") => true, + Some("exit") + if outcome + .get("exit_code") + .and_then(Value::as_i64) + .is_some_and(|code| code != 0) => + { + // Keep plain nonzero exits SOFT, including empty output. + texts.push("returned non-zero"); + output + .get("stderr") + .and_then(Value::as_str) + .is_some_and(|text| !text.trim().is_empty()) + } + _ => false, + }; + } + } + } + let text = texts.join("\n"); + if !text.is_empty() || is_error { + tool_texts.push((text, is_error)); + } + } // Compaction is detected anywhere in the conversation: the summary // stays in the prefix on every later turn, so this self-latches // once it fires. @@ -1511,6 +1567,102 @@ mod tests { assert_eq!(sig.write_count, 1); } + #[test] + fn responses_builtin_tool_failures_escalate() { + use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier}; + + let mut cases = Vec::new(); + for (status, output) in [ + ( + "failed", + "Synthetic dependency unavailable; retry with the recovery path.", + ), + ( + "completed", + "Synthetic dependency unavailable; retry with the recovery path.", + ), + ("failed", ""), + ] { + cases.push(( + json!({ + "type": "apply_patch_call_output", + "status": status, + "output": output, + }), + if status == "failed" { HARD } else { 0.0 }, + )); + } + for (outcome, stdout, stderr, severity) in [ + (json!({"type": "exit", "exit_code": 1}), "", "", SOFT), + (json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT), + ( + json!({"type": "exit", "exit_code": 1}), + "", + "command failed", + HARD, + ), + (json!({"type": "timeout"}), "", "", HARD), + (json!({"type": "exit", "exit_code": 0}), "done", "", 0.0), + ( + json!({"type": "exit", "exit_code": 0}), + "Traceback (most recent call last):", + "", + HARD, + ), + ( + json!({"type": "exit", "exit_code": 0}), + "", + "Traceback (most recent call last):", + HARD, + ), + ] { + cases.push(( + json!({ + "type": "shell_call_output", + "output": [ + {"stdout": stdout, "stderr": stderr, "outcome": outcome}, + {"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}} + ], + }), + severity, + )); + } + for (raw, severity) in cases { + let is_error = severity >= HARD; + let request = with_messages( + ["call_1", "call_2"] + .into_iter() + .map(|call_id| { + let mut raw = raw.clone(); + raw["call_id"] = json!(call_id); + Message { + role: Role::User, + content: vec![ContentBlock::Unknown { + provider: WireFormat::OpenAiResponses.into(), + raw, + }], + } + }) + .collect(), + ); + let signal = ToolSignals::from_request(&request, Some(3)); + assert_eq!(signal.severity, severity, "{raw}"); + assert_eq!(signal.repeated_failure, is_error, "{raw}"); + assert_eq!(signal.tool_result_count, 2); + assert_eq!( + matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 0.5), + PickOutcome::Resolved { + tier: Tier::Capable, + .. + } + ), + is_error, + "{raw}" + ); + } + } + #[test] fn conversation_counts_are_per_block_and_role_aware() { // A batched user message (Anthropic shape) contributes one count per From c517e09a9830cc7ad7e50dc01ee679acdbc2944a Mon Sep 17 00:00:00 2001 From: Graham King Date: Fri, 18 Sep 2026 16:10:38 -0400 Subject: [PATCH 2/2] Feedback. Thanks Code Rabbit. Signed-off-by: Graham King --- .../libsy/src/algorithms/util/tool_signals.rs | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index bb30934d6..3cecf06a3 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -751,9 +751,7 @@ fn extract_tool_signals_with_window_and_semantics( .and_then(Value::as_str) .unwrap_or_default(); let is_error = raw.get("status").and_then(Value::as_str) == Some("failed"); - if !text.is_empty() || is_error { - tool_texts.push((text.to_owned(), is_error)); - } + tool_texts.push((text.to_owned(), is_error)); } ContentBlock::Unknown { provider, raw } if provider.as_str() == WireFormat::OpenAiResponses.as_str() @@ -793,9 +791,7 @@ fn extract_tool_signals_with_window_and_semantics( } } let text = texts.join("\n"); - if !text.is_empty() || is_error { - tool_texts.push((text, is_error)); - } + tool_texts.push((text, is_error)); } // Compaction is detected anywhere in the conversation: the summary // stays in the prefix on every later turn, so this self-latches @@ -1629,7 +1625,7 @@ mod tests { } for (raw, severity) in cases { let is_error = severity >= HARD; - let request = with_messages( + let mut request = with_messages( ["call_1", "call_2"] .into_iter() .map(|call_id| { @@ -1660,6 +1656,32 @@ mod tests { is_error, "{raw}" ); + + let mut success = match raw["type"].as_str() { + Some("apply_patch_call_output") => json!({ + "type": "apply_patch_call_output", "status": "completed", "output": "" + }), + Some("shell_call_output") => json!({ + "type": "shell_call_output", + "output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}] + }), + _ => unreachable!(), + }; + for index in 0..3 { + success["call_id"] = json!(format!("success_{index}")); + request.llm_request.messages.push(Message { + role: Role::User, + content: vec![ContentBlock::Unknown { + provider: WireFormat::OpenAiResponses.into(), + raw: success.clone(), + }], + }); + } + let recovered = ToolSignals::from_request(&request, Some(3)); + assert_eq!(recovered.severity, 0.0, "{raw}"); + assert!(!recovered.repeated_failure, "{raw}"); + assert!(recovered.no_error_streak >= 3, "{raw}"); + assert_eq!(recovered.tool_result_count, 5); } }