diff --git a/docs/pyagentspec/source/changelog.rst b/docs/pyagentspec/source/changelog.rst index c9ceb2b5..f6008515 100644 --- a/docs/pyagentspec/source/changelog.rst +++ b/docs/pyagentspec/source/changelog.rst @@ -17,6 +17,14 @@ Improvements Bug fixes ^^^^^^^^^ +* **LangGraph structured-output guard** + + LangGraph agents with declared structured outputs now fail promptly with a clear error when + their model repeatedly returns prose or finishes without producing the declared output, rather + than looping until the recursion limit or silently returning empty outputs. + + We thank @spichen for the contribution! + Breaking Changes ^^^^^^^^^^^^^^^^ diff --git a/pyagentspec/src/pyagentspec/adapters/_utils.py b/pyagentspec/src/pyagentspec/adapters/_utils.py index e221d613..a69bf07e 100644 --- a/pyagentspec/src/pyagentspec/adapters/_utils.py +++ b/pyagentspec/src/pyagentspec/adapters/_utils.py @@ -184,6 +184,16 @@ def _build_type_from_schema( return mapping.get(t, Any) +def is_single_string_output(expected_outputs: List[AgentSpecProperty]) -> bool: + """Whether the declared outputs are a single string property. + + Such an output is the model's free text, so an adapter can read it from the final + message instead of asking for structured generation. Lives here rather than in one + adapter because it is a property of the declared outputs. + """ + return len(expected_outputs) == 1 and expected_outputs[0].type == "string" + + def create_pydantic_model_from_properties( model_name: str, properties: List[AgentSpecProperty] ) -> type[BaseModel]: diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_agent_output_guard.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_agent_output_guard.py new file mode 100644 index 00000000..30a8b968 --- /dev/null +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_agent_output_guard.py @@ -0,0 +1,129 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Makes an agent fail loudly when its model will not produce structured output. + +``create_agent(response_format=...)`` ties the agent's control flow to getting a +parseable structured response. A model that never produces one fails in one of two +ways, depending on whether the agent has real tools: + +* no tools: the agent loops. ``create_agent`` sets ``recursion_limit=9999``, so that is + thousands of model calls before ``GraphRecursionError``. +* with tools: routing treats a turn with no tool calls as "done", so the agent exits + with no ``structured_response`` and the declared outputs come back empty, with no + error at all (upstream langchain issue #36349). + +:class:`StructuredOutputGuard` covers both. An agent whose only declared output is a +string avoids the problem entirely, since ``response_format`` is skipped for it (see +:func:`pyagentspec.adapters._utils.is_single_string_output`). +""" + +from functools import lru_cache +from typing import Annotated, Any, Dict, List, NoReturn, Optional + +from typing_extensions import NotRequired + +from pyagentspec.adapters.langgraph._types import ( + AgentMiddleware, + AgentState, + langchain_middleware_types, +) + +# A compliant run spends at most one turn without a structured response, so a small +# bound separates "model is working on it" from "model will never comply". +DEFAULT_MAX_STRUCTURED_OUTPUT_ATTEMPTS = 3 + +# Must match the private field on _StructuredOutputAttemptState. +_ATTEMPTS_STATE_KEY = "_pyagentspec_structured_output_attempts" + + +class StructuredOutputNotProducedError(RuntimeError): + """Raised when a model will not produce the structured response an agent declares. + + Subclasses ``RuntimeError`` so existing handlers around agent execution still catch it. + """ + + +class _StructuredOutputAttemptState(AgentState): + # On the state rather than the instance, so the count is per run and not shared + # between concurrent ones. LangChain omits PrivateStateAttr fields from user input + # and output schemas. + _pyagentspec_structured_output_attempts: NotRequired[int] + + +@lru_cache +def _structured_output_attempt_state_schema() -> type: + """Add LangChain's private-state marker when the guard is used.""" + + _StructuredOutputAttemptState.__annotations__[_ATTEMPTS_STATE_KEY] = NotRequired[ + Annotated[int, langchain_middleware_types.PrivateStateAttr] + ] + + return _StructuredOutputAttemptState + + +class StructuredOutputGuard(AgentMiddleware): + """Fails an agent that cannot produce the structured output it declares. + + Install only when ``response_format`` is set. Without it, a turn with no tool calls + just means the agent is done and an empty ``structured_response`` is fine. + """ + + def __init__( + self, + *, + agent_name: str, + output_titles: List[str], + model_id: str, + max_attempts: int = DEFAULT_MAX_STRUCTURED_OUTPUT_ATTEMPTS, + ) -> None: + super().__init__() + self.state_schema = _structured_output_attempt_state_schema() + self.agent_name = agent_name + self.output_titles = output_titles + self.model_id = model_id + self.max_attempts = max_attempts + + def _fail(self, detail: str) -> NoReturn: + raise StructuredOutputNotProducedError( + f"Agent {self.agent_name!r} did not produce a structured response matching its " + f"declared outputs ({', '.join(self.output_titles)}). {detail} Model " + f"{self.model_id!r} may not support structured output. Declare a single string " + f"output to get the model's free text instead, or use a model that supports " + f"structured output." + ) + + # No async variants: LangGraph routes async runs to the sync hooks, and both are pure. + def after_model(self, state: Any, runtime: Any) -> Optional[Dict[str, int]]: + # Runs after every model turn. The last message decides between three cases: + # + # * an AIMessage with tool calls: progress, counter resets; + # * an AIMessage without tool calls: prose instead of the structured response, + # counter increments and the run fails at the bound; + # * anything else: in a plain run the model's AIMessage is always last here, so + # this means another middleware rewrote the history and the turn cannot be + # classified. The counter is left untouched (None updates nothing) rather than + # reset: unclassifiable is not evidence of progress, and resetting would let + # such a middleware clear real evidence of a stuck model every turn. + messages = state.get("messages") or [] + last = messages[-1] if messages else None + if last is None or getattr(last, "type", None) != "ai": + return None + if getattr(last, "tool_calls", None): + # A tool call is progress, so the bound applies to consecutive failures only. + # Under ToolStrategy the structured response is itself a tool call. + return {_ATTEMPTS_STATE_KEY: 0} + attempts = (state.get(_ATTEMPTS_STATE_KEY) or 0) + 1 + if attempts >= self.max_attempts: + self._fail(f"It answered in prose {attempts} times instead.") + return {_ATTEMPTS_STATE_KEY: attempts} + + def after_agent(self, state: Any, runtime: Any) -> None: + # after_model never sees the silent case: an agent with tools exits on its first + # prose turn, well before the bound. + if "structured_response" in state: + return None + self._fail("It ended its run without one.") diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py index 0ca55cb0..f4ae9a98 100644 --- a/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py @@ -35,7 +35,9 @@ SchemaRegistry, _build_type_from_schema, create_pydantic_model_from_properties, + is_single_string_output, ) +from pyagentspec.adapters.langgraph._agent_output_guard import StructuredOutputGuard from pyagentspec.adapters.langgraph._execution_span import patch_with_execution_span from pyagentspec.adapters.langgraph._managerworkers import ( _MANAGER_NODE_KEY, @@ -70,10 +72,7 @@ langgraph_graph, langgraph_swarm, ) -from pyagentspec.adapters.langgraph.mcp_utils import ( - _HttpxClientFactory, - run_async_in_sync, -) +from pyagentspec.adapters.langgraph.mcp_utils import _HttpxClientFactory, run_async_in_sync from pyagentspec.adapters.langgraph.tracing import ( AgentSpecLlmCallbackHandler, AgentSpecToolCallbackHandler, @@ -107,10 +106,7 @@ ) from pyagentspec.llms.ocigenaiconfig import OciGenAiConfig from pyagentspec.llms.ollamaconfig import OllamaConfig -from pyagentspec.llms.openaicompatibleconfig import ( - OpenAIAPIType, - OpenAiCompatibleConfig, -) +from pyagentspec.llms.openaicompatibleconfig import OpenAIAPIType, OpenAiCompatibleConfig from pyagentspec.llms.openaiconfig import OpenAiConfig from pyagentspec.llms.vllmconfig import VllmConfig from pyagentspec.managerworkers import ManagerWorkers as AgentSpecManagerWorkers @@ -591,9 +587,7 @@ def _input_message_node_convert_to_langgraph( self, node: AgentSpecInputMessageNode, ) -> "NodeExecutor": - from pyagentspec.adapters.langgraph._node_execution import ( - InputMessageNodeExecutor, - ) + from pyagentspec.adapters.langgraph._node_execution import InputMessageNodeExecutor return InputMessageNodeExecutor(node) @@ -601,9 +595,7 @@ def _output_message_node_convert_to_langgraph( self, node: AgentSpecOutputMessageNode, ) -> "NodeExecutor": - from pyagentspec.adapters.langgraph._node_execution import ( - OutputMessageNodeExecutor, - ) + from pyagentspec.adapters.langgraph._node_execution import OutputMessageNodeExecutor return OutputMessageNodeExecutor(node) @@ -668,9 +660,7 @@ def _catch_exception_node_convert_to_langgraph( config: RunnableConfig, middleware: List[Any], ) -> "NodeExecutor": - from pyagentspec.adapters.langgraph._node_execution import ( - CatchExceptionNodeExecutor, - ) + from pyagentspec.adapters.langgraph._node_execution import CatchExceptionNodeExecutor subflow = self.convert( catch_node.subflow, @@ -1235,8 +1225,9 @@ def _create_react_agent_with_given_info( state_schema: Optional[Any] = None response_format: Any = None - # Build response (output) model (used for response_format) - if outputs: + # A single string output is read from the final message instead, so it needs no + # response_format. Same rule as LlmNodeExecutor. + if outputs and not is_single_string_output(outputs): output_model = create_pydantic_model_from_properties("AgentOutputModel", outputs) # Explicitly use ToolStrategy instead of letting LangChain select a provider # strategy. OpenAI-compatible models do not necessarily support provider-native @@ -1266,6 +1257,17 @@ def _create_react_agent_with_given_info( response_format=response_format, state_schema=state_schema, ) + if output_model is not None: + # Rebuild rather than append: `middleware` is shared across every agent of a + # swarm / manager-workers graph. + middleware = [ + *middleware, + StructuredOutputGuard( + agent_name=name, + output_titles=[output.title for output in outputs], + model_id=llm_config.model_id, + ), + ] if middleware: create_agent_kwargs["middleware"] = middleware compiled_graph: CompiledStateGraph[Any, Any, Any] = langchain_agents.create_agent( diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_node_execution.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_node_execution.py index a1398df9..80e1141b 100644 --- a/pyagentspec/src/pyagentspec/adapters/langgraph/_node_execution.py +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_node_execution.py @@ -16,7 +16,11 @@ maybe_warn_about_unrestricted_templated_url, validate_url_against_allow_list, ) -from pyagentspec.adapters._utils import render_nested_object_template, render_template +from pyagentspec.adapters._utils import ( + is_single_string_output, + render_nested_object_template, + render_template, +) from pyagentspec.adapters.langgraph._types import ( BaseChatModel, BaseMessage, @@ -606,20 +610,13 @@ def __init__(self, node: AgentSpecLlmNode, llm: BaseChatModel) -> None: super().__init__(node) if not isinstance(self.node, AgentSpecLlmNode): raise TypeError("LlmNodeExecutor can only be initialized with LlmNode") - outputs = self.node.outputs - if outputs is not None and len(outputs) == 1 and outputs[0].type == "string": - self.requires_structured_generation = False - else: - self.requires_structured_generation = True if not isinstance(llm, BaseChatModel): raise TypeError("Llm can only be initialized with a BaseChatModel") self.llm: BaseChatModel = llm node_outputs = self.node.outputs or [] - self.requires_structured_generation = not ( - len(node_outputs) == 1 and node_outputs[0].type == "string" - ) + self.requires_structured_generation = not is_single_string_output(node_outputs) self.structured_llm: Any = None @@ -933,7 +930,7 @@ def extract_outputs_from_invoke_result( # Extracts the outputs from the return value of an invoke call made on an agent # The outputs are typically exposed as part of the `structured_response`, or as entries in the result directly. # We give priority to the latter. - return { + outputs = { # Defaults if available **{ output.title: output.default @@ -949,3 +946,14 @@ def extract_outputs_from_invoke_result( if output.title in result }, } + # No response_format is requested for a single string output, so read it from the + # final message. + if is_single_string_output(expected_outputs): + title = expected_outputs[0].title + if title not in outputs: + messages = result.get("messages") + if messages: + content = getattr(messages[-1], "content", None) + if content is not None: + outputs[title] = content + return outputs diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py index 1b44d0ad..c538dfb9 100644 --- a/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py @@ -15,11 +15,12 @@ # Otherwise, importing the module when they are not installed would lead to an import error. import langchain.agents as langchain_agents + import langchain.agents.middleware.types as langchain_middleware_types import langchain_ollama import langchain_openai import langgraph.graph as langgraph_graph import langgraph_swarm - from langchain.agents.middleware.types import AgentState + from langchain.agents.middleware.types import AgentMiddleware, AgentState from langchain_core.callbacks import BaseCallbackHandler from langchain_core.language_models import BaseChatModel from langchain_core.messages import BaseMessage, SystemMessage, ToolMessage @@ -39,6 +40,7 @@ langchain_openai = LazyLoader("langchain_openai") langgraph_graph = LazyLoader("langgraph.graph") langchain_agents = LazyLoader("langchain.agents") + langchain_middleware_types = LazyLoader("langchain.agents.middleware.types") BaseTool = LazyType("langchain_core.tools", "BaseTool") StructuredTool = LazyType("langchain_core.tools", "StructuredTool") Checkpointer = LazyType("langgraph.types", "Checkpointer") @@ -59,6 +61,7 @@ GenerationChunk = LazyType("langchain_core.outputs", "GenerationChunk") LLMResult = LazyType("langchain_core.outputs", "LLMResult") AgentState = LazyType("langchain.agents.middleware.types", "AgentState") + AgentMiddleware = LazyType("langchain.agents.middleware.types", "AgentMiddleware") LangGraphTool: TypeAlias = Union[BaseTool, Callable[..., Any]] @@ -108,6 +111,7 @@ class FlowOutputSchema(TypedDict): __all__ = [ "langgraph_graph", "langchain_agents", + "langchain_middleware_types", "langchain_ollama", "langchain_openai", "LangGraphTool", @@ -136,6 +140,7 @@ class FlowOutputSchema(TypedDict): "ToolMessage", "BaseChatModel", "AgentState", + "AgentMiddleware", "Checkpointer", "interrupt", "RunnableConfig", diff --git a/pyagentspec/tests/adapters/langgraph/flows/test_agentnode.py b/pyagentspec/tests/adapters/langgraph/flows/test_agentnode.py index e5b4ba12..2dcf97ea 100644 --- a/pyagentspec/tests/adapters/langgraph/flows/test_agentnode.py +++ b/pyagentspec/tests/adapters/langgraph/flows/test_agentnode.py @@ -96,6 +96,80 @@ def test_agentnode_can_be_imported_and_executed(agent_flow: Flow) -> None: assert "car" in outputs +def test_single_string_output_taken_from_final_message_without_structured_generation() -> None: + """The stub model has no structured output, so if the converter still attached a + ``response_format`` the output would come back empty instead of holding "42". + """ + from unittest.mock import patch + + from langchain_core.language_models.fake_chat_models import ( + FakeMessagesListChatModel, + ) + from langchain_core.messages import AIMessage + from langchain_openai import ChatOpenAI + from langgraph.checkpoint.memory import MemorySaver + + from pyagentspec.adapters.langgraph import AgentSpecLoader + from pyagentspec.adapters.langgraph._langgraphconverter import ( + AgentSpecToLangGraphConverter, + ) + from pyagentspec.llms.openaicompatibleconfig import OpenAiCompatibleConfig + + class _FakeModel(FakeMessagesListChatModel, ChatOpenAI): + pass + + fake_llm = _FakeModel(responses=[AIMessage(content="42")]) + + answer = StringProperty(title="answer") + agent = Agent( + name="agent", + llm_config=OpenAiCompatibleConfig(name="agent_llm", model_id="fake", url="null"), + system_prompt="Answer the question.", + outputs=[answer], + ) + agent_node = AgentNode(name="agent_node", agent=agent) + start_node = StartNode(name="start") + end_node = EndNode(name="end", outputs=[answer]) + flow = Flow( + name="flow", + start_node=start_node, + nodes=[start_node, agent_node, end_node], + control_flow_connections=[ + ControlFlowEdge(name="start_to_node", from_node=start_node, to_node=agent_node), + ControlFlowEdge(name="node_to_end", from_node=agent_node, to_node=end_node), + ], + data_flow_connections=[ + DataFlowEdge( + name="answer_edge", + source_node=agent_node, + source_output=answer.title, + destination_node=end_node, + destination_input=answer.title, + ), + ], + outputs=[answer], + ) + + loader = AgentSpecLoader(tool_registry={}, checkpointer=MemorySaver()) + with patch.object( + AgentSpecToLangGraphConverter, + "_llm_convert_to_langgraph", + autospec=True, + side_effect=lambda self_obj, llm_config, *a, **k: fake_llm, + ), patch.object( + FakeMessagesListChatModel, + "bind_tools", + new=lambda self_obj, *a, **k: self_obj, + ): + compiled = loader.load_component(flow) + result = compiled.invoke( + {"inputs": {}, "messages": [{"role": "user", "content": "What is 6*7?"}]}, + {"configurable": {"thread_id": "agentnode-single-string"}}, + ) + + assert result["outputs"]["answer"] == "42" + + @pytest.mark.anyio @retry_test(max_attempts=3, wait_between_tries=2) async def test_agentnode_can_be_executed_async(agent_flow: Flow) -> None: diff --git a/pyagentspec/tests/adapters/langgraph/test_agent_output_guard.py b/pyagentspec/tests/adapters/langgraph/test_agent_output_guard.py new file mode 100644 index 00000000..b03ad82e --- /dev/null +++ b/pyagentspec/tests/adapters/langgraph/test_agent_output_guard.py @@ -0,0 +1,206 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""How :class:`StructuredOutputGuard` bounds an agent that declares structured output. + +These build no LLM config, so unlike the AgentNode flow tests they run rather than skip +under ``SKIP_LLM_TESTS=1``. +""" + +from typing import Any, get_args + +import pytest + +from pyagentspec.adapters.langgraph._agent_output_guard import ( + StructuredOutputGuard, + StructuredOutputNotProducedError, +) + +# ─── after_model budget ─────────────────────────────────────────────────────── + + +def _guard(max_attempts: int = 3) -> StructuredOutputGuard: + return StructuredOutputGuard( + agent_name="researcher", + output_titles=["summary", "confidence"], + model_id="some-model", + max_attempts=max_attempts, + ) + + +def _prose(attempts: int = 0) -> dict: + """A model turn that answered in prose, so no structured response.""" + from langchain_core.messages import AIMessage, HumanMessage + + return { + "messages": [HumanMessage(content="q"), AIMessage(content="42")], + "_pyagentspec_structured_output_attempts": attempts, + } + + +def _tool_turn(attempts: int = 0) -> dict: + from langchain_core.messages import AIMessage + + return { + "messages": [AIMessage(content="", tool_calls=[{"name": "s", "args": {}, "id": "c1"}])], + "_pyagentspec_structured_output_attempts": attempts, + } + + +def test_prose_turn_increments_until_the_limit() -> None: + """Prose never satisfies response_format, so the agent cannot exit on its own.""" + guard = _guard(max_attempts=3) + + assert guard.after_model(_prose(0), runtime=None) == { + "_pyagentspec_structured_output_attempts": 1 + } + assert guard.after_model(_prose(1), runtime=None) == { + "_pyagentspec_structured_output_attempts": 2 + } + with pytest.raises(StructuredOutputNotProducedError): + guard.after_model(_prose(2), runtime=None) + + +def test_error_message_names_agent_fields_and_model() -> None: + """Failing fast is only useful if the message says what to change.""" + with pytest.raises(StructuredOutputNotProducedError) as excinfo: + _guard(max_attempts=1).after_model(_prose(), runtime=None) + + message = str(excinfo.value) + assert "researcher" in message + assert "summary, confidence" in message + assert "some-model" in message + # It should point at the actionable escape hatch, not just complain. + assert "single string output" in message + + +def test_tool_calling_turn_resets_rather_than_counting() -> None: + """A tool call is progress. Under ToolStrategy the structured response is itself a + tool call, so counting these would fail every compliant agent. The reset also keeps a + long run of mixed prose and tool calls from adding up to a false failure.""" + guard = _guard(max_attempts=2) + # max_attempts=2 with 1 attempt already banked would raise if this counted. + assert guard.after_model(_tool_turn(attempts=1), runtime=None) == { + "_pyagentspec_structured_output_attempts": 0 + } + + +def test_non_ai_or_empty_last_message_is_ignored() -> None: + """Only a model turn can be a failed structured response.""" + from langchain_core.messages import ToolMessage + + guard = _guard(max_attempts=1) + assert guard.after_model({"messages": []}, runtime=None) is None + assert guard.after_model({}, runtime=None) is None + assert ( + guard.after_model({"messages": [ToolMessage(content="r", tool_call_id="c1")]}, runtime=None) + is None + ) + + +@pytest.mark.anyio +async def test_budget_applies_on_async_runs_via_the_sync_hook() -> None: + """The guard has no ``aafter_model`` and relies on LangGraph routing async runs to the + sync hook. Agents do run async, so a silent no-op there would bring the hang back.""" + from langchain.agents import create_agent + from langchain_core.language_models.fake_chat_models import GenericFakeChatModel + from langchain_core.messages import AIMessage, HumanMessage + from pydantic import BaseModel + + class _Answer(BaseModel): + summary: str + confidence: str + + class _ProseOnlyModel(GenericFakeChatModel): + """Accepts the structured-output tool binding, then ignores it.""" + + def bind_tools(self, tools: Any, **kwargs: Any) -> Any: + return self + + # A model that only ever answers in prose, so no structured response is produced. + # `response_format` is what makes that a hang rather than a clean exit. + model = _ProseOnlyModel(messages=iter([AIMessage(content="42")] * 50)) + agent = create_agent( + model=model, + tools=[], + system_prompt="Answer.", + response_format=_Answer, + middleware=[_guard(max_attempts=3)], + ) + + with pytest.raises(StructuredOutputNotProducedError): + await agent.ainvoke({"messages": [HumanMessage(content="6*7?")]}) + + +# ─── after_agent: the silent-exit case the budget cannot see ────────────────── + + +def test_after_agent_accepts_a_run_that_produced_the_response() -> None: + """The guard has to stay out of the way of a compliant agent.""" + assert _guard().after_agent({"structured_response": object()}, runtime=None) is None + + +def test_after_agent_rejects_a_run_that_produced_nothing() -> None: + """An agent with tools exits after one prose turn, spending no budget, so this hook is + the only place that failure shows up.""" + from langchain_core.messages import AIMessage + + with pytest.raises(StructuredOutputNotProducedError) as excinfo: + _guard().after_agent({"messages": [AIMessage(content="42")]}, runtime=None) + + message = str(excinfo.value) + assert "researcher" in message + assert "summary, confidence" in message + assert "some-model" in message + assert "single string output" in message + + +def test_agent_with_tools_raises_instead_of_losing_outputs_silently() -> None: + """Regression for langchain#36349. With at least one real tool, routing treats a + turn with no tool calls as "done", so the agent used to return no + ``structured_response`` and the declared outputs came back empty.""" + from langchain.agents import create_agent + from langchain_core.language_models.fake_chat_models import GenericFakeChatModel + from langchain_core.messages import AIMessage, HumanMessage + from langchain_core.tools import tool + from pydantic import BaseModel + + class _Answer(BaseModel): + summary: str + confidence: str + + @tool + def search(q: str) -> str: + """Search for something.""" + return "a result" + + class _ProseOnlyModel(GenericFakeChatModel): + def bind_tools(self, tools: Any, **kwargs: Any) -> Any: + return self + + model = _ProseOnlyModel(messages=iter([AIMessage(content="42")] * 50)) + agent = create_agent( + model=model, + tools=[search], + system_prompt="Answer.", + response_format=_Answer, + middleware=[_guard()], + ) + + with pytest.raises(StructuredOutputNotProducedError, match="ended its run without one"): + agent.invoke({"messages": [HumanMessage(content="6*7?")]}) + + +def test_limiter_state_schema_declares_the_counter() -> None: + """The counter belongs on the state schema, not the instance, so concurrent runs do + not share it.""" + from langchain.agents.middleware.types import PrivateStateAttr + + schema: Any = _guard().state_schema + key = "_pyagentspec_structured_output_attempts" + assert key in schema.__annotations__ + assert key in schema.__optional_keys__ + assert PrivateStateAttr in get_args(get_args(schema.__annotations__[key])[0]) diff --git a/pyagentspec/tests/adapters/test_utils.py b/pyagentspec/tests/adapters/test_utils.py new file mode 100644 index 00000000..ab8c86a8 --- /dev/null +++ b/pyagentspec/tests/adapters/test_utils.py @@ -0,0 +1,19 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for the shared adapter helpers in ``pyagentspec.adapters._utils``.""" + +from pyagentspec.adapters._utils import is_single_string_output +from pyagentspec.property import IntegerProperty, StringProperty + + +def test_is_single_string_output() -> None: + """A lone string output is free text, not a structured field, so adapters read it + from the final message instead of requesting structured generation.""" + assert is_single_string_output([StringProperty(title="x")]) is True + assert is_single_string_output([]) is False + assert is_single_string_output([IntegerProperty(title="n")]) is False + assert is_single_string_output([StringProperty(title="a"), StringProperty(title="b")]) is False