diff --git a/docs-website/docs/pipeline-components/agents-1/agent.mdx b/docs-website/docs/pipeline-components/agents-1/agent.mdx index 49c9953b2c5..0480e3633f1 100644 --- a/docs-website/docs/pipeline-components/agents-1/agent.mdx +++ b/docs-website/docs/pipeline-components/agents-1/agent.mdx @@ -63,10 +63,14 @@ print(response["exit_reason"]) # "text" The `exit_reason` output tells you why the agent stopped, which makes it easy to route the agent's output downstream — for example, with a [`ConditionalRouter`](../routers/conditionalrouter.mdx). It is one of: -- `"text"`: the model returned a reply with no tool calls. +- `"text"`: the model returned a complete reply with no tool calls. +- `"length"`: the model reached its output-token limit. `last_message` may contain a partial response. +- `"content_filter"`: a content filter stopped the model response. `last_message` may contain a partial response. - the name of the tool that satisfied a tool exit condition. In this case `last_message` is that tool's result — a tool-result `ChatMessage` whose `text` is empty — so `exit_reason` tells you how to consume it. - `"max_agent_steps"`: the agent reached `max_agent_steps` before meeting an exit condition. +The Agent stops by default on `"length"` and `"content_filter"` so it does not repeatedly submit an unchanged request. These reasons do not make recovery impossible: an `on_exit` hook can inspect the reason, rewrite the messages, and set `continue_run` to `True`. Alternatively, route the result downstream to retry with different generation settings or request human review. + Because `exit_reason` is available on the live `State`, an `after_run` [hook](./hooks.mdx) can read it to react to how the run ended — for example, appending a fallback answer when the step budget is exhausted before the agent finished: ```python @@ -84,6 +88,23 @@ def fallback_on_max_steps(state: State) -> None: ) ``` +For example, this hook requests one shorter continuation after an output-limit exit. The Agent's existing `max_agent_steps` setting still bounds the run. + +```python +@hook +def recover_from_output_limit(state: State) -> None: + recovery_prompt = "Continue with a shorter answer." + recovery_attempted = any( + message.text == recovery_prompt for message in state.get("messages", []) + ) + if state.get("exit_reason") == "length" and not recovery_attempted: + state.set( + "messages", + [ChatMessage.from_user(recovery_prompt)], + ) + state.set("continue_run", True) +``` + ## Parameters `chat_generator` is the only mandatory parameter — an instance of a Chat Generator that supports tools. All other parameters are optional. diff --git a/docs-website/docs/pipeline-components/agents-1/state.mdx b/docs-website/docs/pipeline-components/agents-1/state.mdx index ad82d9ed071..72a93c0e0aa 100644 --- a/docs-website/docs/pipeline-components/agents-1/state.mdx +++ b/docs-website/docs/pipeline-components/agents-1/state.mdx @@ -70,7 +70,7 @@ If you don't specify a handler, State automatically assigns a default based on t :::info Reserved keys The `Agent` manages some state keys itself and rejects them in a user-provided `state_schema` with a `ValueError`: -- The run-metadata keys `step_count`, `token_usage`, `tool_call_counts`, and `exit_reason`, which the Agent populates automatically during a run: tools and hooks can read them from the live `State`, and they are returned in the result dictionary. `exit_reason` reports why the Agent stopped (`"text"`, the name of the tool that triggered a tool exit condition, or `"max_agent_steps"`). +- The run-metadata keys `step_count`, `token_usage`, `tool_call_counts`, and `exit_reason`, which the Agent populates automatically during a run: tools and hooks can read them from the live `State`, and they are returned in the result dictionary. `exit_reason` reports why the Agent stopped (`"text"`, `"length"`, `"content_filter"`, the name of the tool that triggered a tool exit condition, or `"max_agent_steps"`). - The hook-facing keys `continue_run` (set by an `on_exit` hook to keep the Agent running), `tools` (the tools available in the current step, for hooks to inspect), `hook_context` (the request-scoped resources passed to `Agent.run(hook_context={...})`), and `context_tokens` (an approximate count of the tokens currently in the context window, refreshed after each LLM call for hooks to read — for example, to trigger context compaction). Unlike the run-metadata keys, these are not returned in the result dictionary. If one of your state keys clashes, rename it (for example, `my_token_usage`). diff --git a/docs-website/docs/tools/agenttool.mdx b/docs-website/docs/tools/agenttool.mdx index 5e95165249c..fb08226b86a 100644 --- a/docs-website/docs/tools/agenttool.mdx +++ b/docs-website/docs/tools/agenttool.mdx @@ -36,7 +36,7 @@ If the specialist needs more than a task, for example a variable in its system p - `name` is mandatory and specifies the tool name. - `description` is mandatory. It should tell the calling LLM what the wrapped Agent is specialized in and when to delegate to it. - `parameters` is optional and lets you override the generated JSON schema for the tool's inputs. It must cover every mandatory input of the wrapped Agent that is not supplied through `inputs_from_state`, otherwise a `ValueError` is raised. -- `outputs_to_string` is optional and controls how the wrapped Agent's output is converted to a string for the calling LLM. By default, the text of the final reply is returned, or the serialized message if the reply has no text. A warning is appended if the Agent stopped because it reached `max_agent_steps`. +- `outputs_to_string` is optional and controls how the wrapped Agent's output is converted to a string for the calling LLM. By default, the text of the final reply is returned, or the serialized message if the reply has no text. A warning is appended if the Agent stopped because it reached `max_agent_steps`, reached the model's output limit, or had its response stopped by a content filter. - `inputs_from_state` is optional and maps the calling Agent's state keys to inputs of the wrapped Agent. Example: `{"subject": "topic"}` passes the state value at `"subject"` as the wrapped Agent's `"topic"` input. Inputs mapped this way are not added to the generated schema, since the calling Agent provides them. - `outputs_to_state` is optional and maps the wrapped Agent's output keys to the calling Agent's state keys. Example: `{"notes": {"source": "last_message"}}` writes the wrapped Agent's `"last_message"` output to `"notes"` in state. diff --git a/haystack/components/agents/agent.py b/haystack/components/agents/agent.py index 7dc4dd583a9..e797d56c4e2 100644 --- a/haystack/components/agents/agent.py +++ b/haystack/components/agents/agent.py @@ -65,9 +65,12 @@ logger = logging.getLogger(__name__) -# `exit_reason` values the Agent sets when it stops without a tool exit condition: a tool-call-free reply, or the -# `max_agent_steps` budget running out. A tool exit condition instead reports the tool's name. +# `exit_reason` values the Agent sets when it stops without a tool exit condition: a tool-call-free reply, an +# incomplete model generation, or the `max_agent_steps` budget running out. A tool exit condition instead reports the +# tool's name. _EXIT_REASON_TEXT = "text" +_EXIT_REASON_LENGTH = "length" +_EXIT_REASON_CONTENT_FILTER = "content_filter" _EXIT_REASON_MAX_STEPS = "max_agent_steps" # Run-metadata state keys the Agent populates automatically during a run. Users may not define them in their own @@ -147,17 +150,27 @@ def _consume_continue_run(state: State) -> bool: return should_continue -def _is_text_exit(messages: list[ChatMessage]) -> bool: +def _get_model_exit_reason(messages: list[ChatMessage]) -> str | None: """ - Return whether `messages` end in a plain assistant text reply with no tool calls anywhere in the batch. + Return the exit reason for a terminal assistant reply without tool calls. - This is the "no tool call" exit for the model's own replies. The last message must be a non-empty assistant text - message, so an invalid response (e.g. one with no tool calls and no text) does not trigger an exit. + Incomplete generation reasons take precedence over text so callers can distinguish a partial response from a + complete answer. An empty response without a recognized terminal reason does not trigger an exit, preserving the + Agent's recovery behavior for malformed tool calls that a Chat Generator discarded. """ - if not messages: - return False + if not messages or any(message.tool_call for message in messages): + return None + last = messages[-1] - return not any(m.tool_call for m in messages) and last.is_from(ChatRole.ASSISTANT) and bool(last.text) + if not last.is_from(ChatRole.ASSISTANT): + return None + if last.meta.get("finish_reason") == _EXIT_REASON_LENGTH: + return _EXIT_REASON_LENGTH + if last.meta.get("finish_reason") == _EXIT_REASON_CONTENT_FILTER: + return _EXIT_REASON_CONTENT_FILTER + if last.text: + return _EXIT_REASON_TEXT + return None def _pending_tool_call_messages_from_state(state: State) -> list[ChatMessage]: @@ -838,9 +851,11 @@ def run( `meta["usage"]`. - "tool_call_counts": Mapping of tool name to the number of times that tool was invoked. - "exit_reason": Why the Agent stopped, useful for routing the output downstream (e.g. with a - `ConditionalRouter`). One of: `"text"` (the model returned a reply with no tool calls), the name of - the tool that satisfied a tool exit condition (in which case `last_message` is that tool's result), - or `"max_agent_steps"` (the Agent hit `max_agent_steps` before meeting an exit condition). + `ConditionalRouter`). One of: `"text"` (the model returned a complete reply with no tool calls), + `"length"` or `"content_filter"` (the model returned an incomplete reply, which may contain partial + text), the name of the tool that satisfied a tool exit condition (in which case `last_message` is that + tool's result), or `"max_agent_steps"` (the Agent hit `max_agent_steps` before meeting an exit + condition). - Any additional keys defined in the `state_schema`. """ agent_inputs = {"messages": messages, "streaming_callback": streaming_callback, **kwargs} @@ -922,9 +937,11 @@ async def run_async( `meta["usage"]`. - "tool_call_counts": Mapping of tool name to the number of times that tool was invoked. - "exit_reason": Why the Agent stopped, useful for routing the output downstream (e.g. with a - `ConditionalRouter`). One of: `"text"` (the model returned a reply with no tool calls), the name of - the tool that satisfied a tool exit condition (in which case `last_message` is that tool's result), - or `"max_agent_steps"` (the Agent hit `max_agent_steps` before meeting an exit condition). + `ConditionalRouter`). One of: `"text"` (the model returned a complete reply with no tool calls), + `"length"` or `"content_filter"` (the model returned an incomplete reply, which may contain partial + text), the name of the tool that satisfied a tool exit condition (in which case `last_message` is that + tool's result), or `"max_agent_steps"` (the Agent hit `max_agent_steps` before meeting an exit + condition). - Any additional keys defined in the `state_schema`. """ agent_inputs = {"messages": messages, "streaming_callback": streaming_callback, **kwargs} @@ -993,11 +1010,12 @@ def _run_step(self, exe_context: _ExecutionContext, agent_span: tracing.Span) -> _record_llm_usage(state=exe_context.state, llm_messages=llm_messages) _record_context_tokens(state=exe_context.state, llm_messages=llm_messages) - # Stop on the "no tool call" exit: no tools available, or a plain assistant text reply (see _is_text_exit). - if not current_tools or _is_text_exit(messages=llm_messages): + # Stop when there are no tools, or the model produced a terminal reply without tool calls. + model_exit_reason = _get_model_exit_reason(messages=llm_messages) + if not current_tools or model_exit_reason is not None: exe_context.counter += 1 exe_context.state.set("step_count", exe_context.counter) - exe_context.state.set("exit_reason", _EXIT_REASON_TEXT) + exe_context.state.set("exit_reason", model_exit_reason or _EXIT_REASON_TEXT) return self._continue_after_exit_hooks(exe_context=exe_context) _run_hooks(hooks=self.hooks, hook_point=BEFORE_TOOL, state=exe_context.state) @@ -1058,11 +1076,12 @@ async def _run_step_async(self, exe_context: _ExecutionContext, agent_span: trac _record_llm_usage(state=exe_context.state, llm_messages=llm_messages) _record_context_tokens(state=exe_context.state, llm_messages=llm_messages) - # Stop on the "no tool call" exit: no tools available, or a plain assistant text reply (see _is_text_exit). - if not current_tools or _is_text_exit(messages=llm_messages): + # Stop when there are no tools, or the model produced a terminal reply without tool calls. + model_exit_reason = _get_model_exit_reason(messages=llm_messages) + if not current_tools or model_exit_reason is not None: exe_context.counter += 1 exe_context.state.set("step_count", exe_context.counter) - exe_context.state.set("exit_reason", _EXIT_REASON_TEXT) + exe_context.state.set("exit_reason", model_exit_reason or _EXIT_REASON_TEXT) return await self._continue_after_exit_hooks_async(exe_context=exe_context) await _run_hooks_async(hooks=self.hooks, hook_point=BEFORE_TOOL, state=exe_context.state) diff --git a/haystack/tools/agent_tool.py b/haystack/tools/agent_tool.py index f07fcb08068..0a711da6908 100644 --- a/haystack/tools/agent_tool.py +++ b/haystack/tools/agent_tool.py @@ -7,7 +7,7 @@ from typing import Any from haystack.components.agents import Agent -from haystack.components.agents.agent import _EXIT_REASON_MAX_STEPS +from haystack.components.agents.agent import _EXIT_REASON_CONTENT_FILTER, _EXIT_REASON_LENGTH, _EXIT_REASON_MAX_STEPS from haystack.tools.component_tool import ComponentTool from haystack.tools.tool import _deserialize_outputs_to_state, _deserialize_outputs_to_string from haystack.utils.deserialization import deserialize_component_inplace @@ -38,8 +38,13 @@ def agent_result_to_string(result: dict[str, Any]) -> str: """Default `outputs_to_string` handler""" last_message = result["last_message"] text = last_message.text or json.dumps(last_message.to_dict()) - if result["exit_reason"] == _EXIT_REASON_MAX_STEPS: + exit_reason = result["exit_reason"] + if exit_reason == _EXIT_REASON_MAX_STEPS: text += "\n\n[The Agent reached max_agent_steps and stopped, so this result may be incomplete.]" + elif exit_reason == _EXIT_REASON_LENGTH: + text += "\n\n[The Agent reached the model's output limit and stopped, so this result may be incomplete.]" + elif exit_reason == _EXIT_REASON_CONTENT_FILTER: + text += "\n\n[The Agent's response was stopped by a content filter, so this result may be incomplete.]" return text @@ -121,7 +126,8 @@ def __init__( :param outputs_to_string: Optional dictionary defining how tool outputs should be converted into string(s) or results. If not provided, the tool result is the text of the Agent's final reply, or the serialized message if - the reply has no text. A warning is appended if the Agent stopped because it reached `max_agent_steps`. + the reply has no text. A warning is appended if the Agent stopped because it reached `max_agent_steps`, + reached the model's output limit, or had its response stopped by a content filter. `outputs_to_string` supports two formats: diff --git a/releasenotes/notes/agent-incomplete-exit-reasons-059b44c730340561.yaml b/releasenotes/notes/agent-incomplete-exit-reasons-059b44c730340561.yaml new file mode 100644 index 00000000000..758d93030f2 --- /dev/null +++ b/releasenotes/notes/agent-incomplete-exit-reasons-059b44c730340561.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + ``Agent`` now reports ``exit_reason="length"`` or ``"content_filter"`` when a tool-call-free model reply ends + for either reason, including replies containing partial text. These incomplete generations stop by default but + can be recovered with an ``on_exit`` hook. ``AgentTool`` also warns the calling model that such results may be + incomplete. diff --git a/test/components/agents/test_agent.py b/test/components/agents/test_agent.py index 568c0486c05..9ee37c728c4 100644 --- a/test/components/agents/test_agent.py +++ b/test/components/agents/test_agent.py @@ -17,7 +17,7 @@ from openai.types.chat import ChatCompletionChunk, chat_completion_chunk from haystack import Document, Pipeline, component -from haystack.components.agents.agent import Agent +from haystack.components.agents.agent import Agent, _get_model_exit_reason from haystack.components.agents.state import State, merge_lists, replace_values from haystack.components.agents.tool_calling import _run_tool from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder @@ -574,6 +574,32 @@ def test_clone_with_additional_state_schema_and_tools(self, weather_tool, compon assert clone.state_schema == {"foo": {"type": str}, "notes": {"type": str}} +class TestGetModelExitReason: + @pytest.mark.parametrize( + ("message", "expected"), + [ + (ChatMessage.from_assistant("Complete answer."), "text"), + (ChatMessage.from_assistant("", meta={"finish_reason": "length"}), "length"), + (ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": "length"}), "length"), + (ChatMessage.from_assistant("", meta={"finish_reason": "content_filter"}), "content_filter"), + (ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": "content_filter"}), "content_filter"), + (ChatMessage.from_assistant(""), None), + (ChatMessage.from_user("Not an assistant reply."), None), + ], + ) + def test_classifies_tool_call_free_model_replies(self, message, expected): + assert _get_model_exit_reason([message]) == expected + + def test_empty_message_batch_is_not_an_exit(self): + assert _get_model_exit_reason([]) is None + + def test_tool_call_in_batch_takes_precedence(self): + tool_call = ChatMessage.from_assistant(tool_calls=[ToolCall(tool_name="weather", arguments={})]) + incomplete = ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": "length"}) + + assert _get_model_exit_reason([tool_call, incomplete]) is None + + class TestAgentRun: def test_agent_with_no_tools(self): agent = Agent(chat_generator=MockChatGenerator("Berlin"), tools=[], max_agent_steps=3) @@ -1087,6 +1113,31 @@ def test_does_not_exit_on_empty_assistant_message(self, weather_tool): assert result["step_count"] == 2 assert result["last_message"].text == "The weather is sunny." + @pytest.mark.parametrize("finish_reason", ["length", "content_filter"]) + @pytest.mark.parametrize("text", ["", "Partial answer."]) + def test_incomplete_model_reply_exits_with_specific_reason(self, weather_tool, finish_reason, text): + replies = [ + ChatMessage.from_assistant(text=text, meta={"finish_reason": finish_reason}), + "Recovered answer that must not be generated.", + ] + agent = Agent(chat_generator=MockChatGenerator(replies), tools=[weather_tool]) + + result = agent.run([ChatMessage.from_user("What's the weather?")]) + + assert result["step_count"] == 1 + assert result["exit_reason"] == finish_reason + assert result["last_message"].text == text + + @pytest.mark.parametrize("finish_reason", ["length", "content_filter"]) + def test_incomplete_model_reply_without_tools_preserves_reason(self, finish_reason): + reply = ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": finish_reason}) + agent = Agent(chat_generator=MockChatGenerator(reply)) + + result = agent.run([ChatMessage.from_user("Question")]) + + assert result["step_count"] == 1 + assert result["exit_reason"] == finish_reason + @pytest.mark.asyncio async def test_does_not_exit_on_empty_assistant_message_async(self, weather_tool): replies = [ChatMessage.from_assistant(text=""), "The weather is sunny."] @@ -1097,6 +1148,22 @@ async def test_does_not_exit_on_empty_assistant_message_async(self, weather_tool assert result["step_count"] == 2 assert result["last_message"].text == "The weather is sunny." + @pytest.mark.asyncio + @pytest.mark.parametrize("finish_reason", ["length", "content_filter"]) + @pytest.mark.parametrize("text", ["", "Partial answer."]) + async def test_incomplete_model_reply_exits_with_specific_reason_async(self, weather_tool, finish_reason, text): + replies = [ + ChatMessage.from_assistant(text=text, meta={"finish_reason": finish_reason}), + "Recovered answer that must not be generated.", + ] + agent = Agent(chat_generator=MockChatGenerator(replies), tools=[weather_tool]) + + result = await agent.run_async([ChatMessage.from_user("What's the weather?")]) + + assert result["step_count"] == 1 + assert result["exit_reason"] == finish_reason + assert result["last_message"].text == text + def test_text_exit(self, weather_tool): """A plain assistant reply with no tool calls reports the `"text"` exit reason.""" agent = Agent(chat_generator=MockChatGenerator("Berlin is sunny."), tools=[weather_tool]) diff --git a/test/components/agents/test_agent_hooks.py b/test/components/agents/test_agent_hooks.py index 1ff08dd5069..0b2cd5c67a2 100644 --- a/test/components/agents/test_agent_hooks.py +++ b/test/components/agents/test_agent_hooks.py @@ -489,6 +489,32 @@ def record(state: State) -> None: assert agent.chat_generator.run.call_count == 1 assert fired == [1] + def test_incomplete_generation_can_recover(self): + seen_reasons = [] + + def request_shorter_answer(state: State) -> None: + seen_reasons.append(state.get("exit_reason")) + if state.get("exit_reason") == "length": + state.set("messages", [ChatMessage.from_user("Continue with a shorter answer.")]) + state.set("continue_run", True) + + agent = _agent(MockChatGenerator(), hooks={"on_exit": [hook(request_shorter_answer)]}) + agent.chat_generator.run = MagicMock( + side_effect=[ + {"replies": [ChatMessage.from_assistant("Partial", meta={"finish_reason": "length"})]}, + {"replies": [ChatMessage.from_assistant("Recovered answer")]}, + ] + ) + + result = agent.run(messages=[ChatMessage.from_user("hi")]) + + assert agent.chat_generator.run.call_count == 2 + assert seen_reasons == ["length", "text"] + assert result["exit_reason"] == "text" + assert result["step_count"] == 2 + second_call_messages = agent.chat_generator.run.call_args_list[1].kwargs["messages"] + assert second_call_messages[-1].text == "Continue with a shorter answer." + def test_critique_on_tool_based_exit(self): agent = _agent( MockChatGenerator(), tools=[final_answer], exit_conditions=["final_answer"], hooks={"on_exit": [critique]} @@ -510,6 +536,17 @@ def test_max_agent_steps_bounds_always_cancel_hook(self): agent.run(messages=[ChatMessage.from_user("hi")]) assert agent.chat_generator.run.call_count == 3 + def test_max_agent_steps_bounds_repeated_incomplete_generation(self): + agent = _agent(MockChatGenerator(), max_agent_steps=3, hooks={"on_exit": [always_continue]}) + agent.chat_generator.run = MagicMock( + return_value={"replies": [ChatMessage.from_assistant("", meta={"finish_reason": "length"})]} + ) + + result = agent.run(messages=[ChatMessage.from_user("hi")]) + + assert agent.chat_generator.run.call_count == 3 + assert result["exit_reason"] == "max_agent_steps" + def test_continue_run_from_before_llm_hook_does_not_force_continuation(self): def leak_continue(state: State) -> None: state.set("continue_run", True) @@ -646,6 +683,31 @@ async def require_save_async(state: State) -> None: assert agent.chat_generator.run_async.call_count == 3 assert result["tool_call_counts"]["save"] == 1 + @pytest.mark.asyncio + async def test_incomplete_generation_can_recover(self): + seen_reasons = [] + + async def request_safe_answer(state: State) -> None: + seen_reasons.append(state.get("exit_reason")) + if state.get("exit_reason") == "content_filter": + state.set("messages", [ChatMessage.from_user("Answer at a safe, high level.")]) + state.set("continue_run", True) + + agent = _agent(MockChatGenerator(), hooks={"on_exit": [hook(request_safe_answer)]}) + agent.chat_generator.run_async = AsyncMock( + side_effect=[ + {"replies": [ChatMessage.from_assistant("Partial", meta={"finish_reason": "content_filter"})]}, + {"replies": [ChatMessage.from_assistant("Safe recovered answer")]}, + ] + ) + + result = await agent.run_async(messages=[ChatMessage.from_user("hi")]) + + assert agent.chat_generator.run_async.call_count == 2 + assert seen_reasons == ["content_filter", "text"] + assert result["exit_reason"] == "text" + assert result["step_count"] == 2 + class TestAgentHookLifecycle: def test_init_does_not_warm_up(self): diff --git a/test/tools/test_agent_tool.py b/test/tools/test_agent_tool.py index 09611e41bd3..884666b1667 100644 --- a/test/tools/test_agent_tool.py +++ b/test/tools/test_agent_tool.py @@ -65,6 +65,19 @@ def test_exit_reason_max_agent_steps(self): "\n\n[The Agent reached max_agent_steps and stopped, so this result may be incomplete.]" ) + def test_exit_reason_length(self): + result = {"last_message": ChatMessage.from_assistant("Partial response"), "exit_reason": "length"} + assert agent_result_to_string(result=result) == "Partial response" + ( + "\n\n[The Agent reached the model's output limit and stopped, so this result may be incomplete.]" + ) + + def test_exit_reason_content_filter(self): + message = ChatMessage.from_assistant("") + result = {"last_message": message, "exit_reason": "content_filter"} + assert agent_result_to_string(result=result) == json.dumps(message.to_dict()) + ( + "\n\n[The Agent's response was stopped by a content filter, so this result may be incomplete.]" + ) + class TestAgentTool: def test_init(self):