From 247257bc22fc8302df633626746f15b4b4c702c9 Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:48:38 +0100 Subject: [PATCH 1/3] feat: trace effective tools and handoffs per turn --- src/agents/run_internal/run_loop.py | 57 +++++++++++++++++------- src/agents/tracing/span_data.py | 10 ++++- tests/test_agent_tracing.py | 69 +++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 17 deletions(-) diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index ba0c02f351..8e34d1bbd7 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -85,10 +85,18 @@ dispose_resolved_computers, ) from ..tool_guardrails import ToolInputGuardrailResult, ToolOutputGuardrailResult -from ..tracing import Span, SpanError, agent_span, get_current_trace, task_span, turn_span +from ..tracing import ( + Span, + SpanError, + agent_span, + get_current_span, + get_current_trace, + task_span, + turn_span, +) from ..tracing.config import include_task_and_turn_spans from ..tracing.model_tracing import get_model_tracing_impl -from ..tracing.span_data import AgentSpanData, TaskSpanData +from ..tracing.span_data import AgentSpanData, TaskSpanData, TurnSpanData from ..usage import ( Usage, _extract_raw_usage_snapshot, @@ -319,6 +327,27 @@ def _agent_diagnostic_extra(agent: Agent[Any]) -> dict[str, object]: return {"agent_name": agent.name} +def _record_effective_model_capabilities( + agent_span: Span[AgentSpanData] | None, + tools: list[Tool], + handoffs: list[Handoff], +) -> None: + """Record the resolved model-visible capability set for the agent and current turn.""" + handoff_names = [handoff.agent_name for handoff in handoffs] + tool_names = [ + tool_name for tool in tools if (tool_name := get_tool_trace_name_for_tool(tool)) is not None + ] + + if agent_span is not None: + agent_span.span_data.handoffs = handoff_names + agent_span.span_data.tools = tool_names + + current_span = get_current_span() + if current_span is not None and isinstance(current_span.span_data, TurnSpanData): + current_span.span_data.handoffs = list(handoff_names) + current_span.span_data.tools = list(tool_names) + + async def _should_persist_stream_items( *, session: Session | None, @@ -2105,13 +2134,11 @@ async def raise_if_input_guardrail_tripwire_known() -> None: handoffs, collision_policy=run_config.tool_name_collision_policy, ) - if agent_span is not None: - agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] - agent_span.span_data.tools = [ - tool_name - for tool in all_tools - if (tool_name := get_tool_trace_name_for_tool(tool)) is not None - ] + _record_effective_model_capabilities( + agent_span=agent_span, + tools=all_tools, + handoffs=handoffs, + ) model = get_model(execution_agent, run_config) tool_use_tracker.record_model(model) @@ -2432,13 +2459,11 @@ async def run_single_turn( handoffs, collision_policy=run_config.tool_name_collision_policy, ) - if agent_span is not None: - agent_span.span_data.handoffs = [handoff.agent_name for handoff in handoffs] - agent_span.span_data.tools = [ - tool_name - for tool in all_tools - if (tool_name := get_tool_trace_name_for_tool(tool)) is not None - ] + _record_effective_model_capabilities( + agent_span=agent_span, + tools=all_tools, + handoffs=handoffs, + ) output_schema = get_output_schema(execution_agent) if server_conversation_tracker is not None: diff --git a/src/agents/tracing/span_data.py b/src/agents/tracing/span_data.py index 872388a736..8d98fb1c16 100644 --- a/src/agents/tracing/span_data.py +++ b/src/agents/tracing/span_data.py @@ -98,7 +98,7 @@ def export(self) -> dict[str, Any]: class TurnSpanData(SpanData): """Represents one agent loop turn.""" - __slots__ = ("turn", "agent_name", "usage", "metadata") + __slots__ = ("turn", "agent_name", "usage", "metadata", "tools", "handoffs") def __init__( self, @@ -106,11 +106,15 @@ def __init__( agent_name: str, usage: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None, + tools: list[str] | None = None, + handoffs: list[str] | None = None, ): self.turn = turn self.agent_name = agent_name self.usage = usage self.metadata = metadata + self.tools = tools + self.handoffs = handoffs @property def type(self) -> str: @@ -122,6 +126,10 @@ def export(self) -> dict[str, Any]: "turn": self.turn, "agent_name": self.agent_name, } + if self.tools is not None: + data["tools"] = self.tools + if self.handoffs is not None: + data["handoffs"] = self.handoffs if self.usage is not None: data["usage"] = self.usage diff --git a/tests/test_agent_tracing.py b/tests/test_agent_tracing.py index c094439947..62718ef3c2 100644 --- a/tests/test_agent_tracing.py +++ b/tests/test_agent_tracing.py @@ -8,6 +8,7 @@ from openai.types.responses.response_usage import InputTokensDetails from agents import Agent, RunConfig, Runner, RunState, custom_span, function_tool, trace +from agents.handoffs import handoff from agents.sandbox.runtime import SandboxRuntime from agents.testing import ScriptedModel from agents.usage import Usage @@ -130,6 +131,70 @@ def second_lookup() -> str: assert agent_spans[0].span_data.handoffs == expected_handoffs +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.asyncio +async def test_turn_span_snapshots_effective_capabilities_per_model_turn( + streamed: bool, +) -> None: + phase = {"value": "draft"} + + def draft_enabled(_ctx: object, _agent: object) -> bool: + return phase["value"] == "draft" + + def send_enabled(_ctx: object, _agent: object) -> bool: + return phase["value"] == "send" + + @function_tool(name_override="draft_invoice", is_enabled=draft_enabled) + def draft_invoice() -> str: + phase["value"] = "send" + return "drafted" + + @function_tool(name_override="send_invoice", is_enabled=send_enabled) + def send_invoice() -> str: + return "sent" + + model = ScriptedModel(emit_traces=True) + model.extend( + [ + [get_function_tool_call("draft_invoice", "{}", call_id="call-1")], + [get_text_message("done")], + ] + ) + agent = Agent( + name="test_agent", + model=model, + tools=[draft_invoice, send_invoice], + handoffs=[ + handoff(Agent(name="Draft Review"), is_enabled=draft_enabled), + handoff(Agent(name="Send Review"), is_enabled=send_enabled), + ], + ) + + if streamed: + result = Runner.run_streamed(agent, input="test") + async for _ in result.stream_events(): + pass + else: + await Runner.run(agent, input="test") + + spans = fetch_ordered_spans() + turn_spans = [span for span in spans if span.span_data.type == "turn"] + assert len(turn_spans) == 2 + assert [span.span_data.tools for span in turn_spans] == [ + ["draft_invoice"], + ["send_invoice"], + ] + assert [span.span_data.handoffs for span in turn_spans] == [ + ["Draft Review"], + ["Send Review"], + ] + + agent_spans = [span for span in spans if span.span_data.type == "agent"] + assert len(agent_spans) == 1 + assert agent_spans[0].span_data.tools == ["send_invoice"] + assert agent_spans[0].span_data.handoffs == ["Send Review"] + + @pytest.mark.asyncio async def test_task_and_turn_spans_export_aggregate_usage(): @function_tool @@ -205,6 +270,8 @@ def foo_tool() -> str: "sdk_span_type": "turn", "turn": 1, "agent_name": "test_agent", + "tools": ["foo_tool"], + "handoffs": [], "usage": { "input_tokens": 10, "output_tokens": 3, @@ -220,6 +287,8 @@ def foo_tool() -> str: "sdk_span_type": "turn", "turn": 2, "agent_name": "test_agent", + "tools": ["foo_tool"], + "handoffs": [], "usage": { "input_tokens": 10, "output_tokens": 3, From adab4fc25dd5b314c7d4ecd984363b7f518a11de Mon Sep 17 00:00:00 2001 From: Sylvester Kaczmarek <16242628+sylvesterkaczmarek@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:30:01 +0100 Subject: [PATCH 2/3] fix: use model-visible identities in turn tracing --- src/agents/run_internal/run_loop.py | 29 ++++++++++---- tests/test_agent_tracing.py | 4 +- tests/test_turn_capability_identities.py | 51 ++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 tests/test_turn_capability_identities.py diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 8e34d1bbd7..99bb5f7ae7 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -22,6 +22,7 @@ from openai.types.responses.response_prompt_param import ResponsePromptParam from .._tool_identity import ( + get_function_tool_trace_name, get_tool_trace_name_for_tool, resolve_tool_name_collisions, ) @@ -333,19 +334,31 @@ def _record_effective_model_capabilities( handoffs: list[Handoff], ) -> None: """Record the resolved model-visible capability set for the agent and current turn.""" - handoff_names = [handoff.agent_name for handoff in handoffs] - tool_names = [ - tool_name for tool in tools if (tool_name := get_tool_trace_name_for_tool(tool)) is not None + agent_handoff_names = [handoff.agent_name for handoff in handoffs] + turn_handoff_names = [handoff.tool_name for handoff in handoffs] + agent_tool_names = [ + tool_name + for tool in tools + if (tool_name := get_tool_trace_name_for_tool(tool)) is not None + ] + turn_tool_names = [ + trace_name + for tool in tools + if ( + trace_name := get_function_tool_trace_name(tool) + or get_tool_trace_name_for_tool(tool) + ) + is not None ] if agent_span is not None: - agent_span.span_data.handoffs = handoff_names - agent_span.span_data.tools = tool_names + agent_span.span_data.handoffs = agent_handoff_names + agent_span.span_data.tools = agent_tool_names current_span = get_current_span() if current_span is not None and isinstance(current_span.span_data, TurnSpanData): - current_span.span_data.handoffs = list(handoff_names) - current_span.span_data.tools = list(tool_names) + current_span.span_data.handoffs = turn_handoff_names + current_span.span_data.tools = turn_tool_names async def _should_persist_stream_items( @@ -1192,7 +1205,7 @@ async def _save_max_turns_items( response_id=response_id, reasoning_item_id_policy=streamed_result._reasoning_item_id_policy, store=store_setting, - wrapper=streamed_result.context_wrapper, + wrapper=context_wrapper, ) streamed_result._current_turn_persisted_item_count += saved_count except BaseException: diff --git a/tests/test_agent_tracing.py b/tests/test_agent_tracing.py index 62718ef3c2..9f8aa92527 100644 --- a/tests/test_agent_tracing.py +++ b/tests/test_agent_tracing.py @@ -185,8 +185,8 @@ def send_invoice() -> str: ["send_invoice"], ] assert [span.span_data.handoffs for span in turn_spans] == [ - ["Draft Review"], - ["Send Review"], + ["transfer_to_draft_review"], + ["transfer_to_send_review"], ] agent_spans = [span for span in spans if span.span_data.type == "agent"] diff --git a/tests/test_turn_capability_identities.py b/tests/test_turn_capability_identities.py new file mode 100644 index 0000000000..cdbde08059 --- /dev/null +++ b/tests/test_turn_capability_identities.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import pytest + +from agents import Agent, Runner, function_tool, tool_namespace +from agents.handoffs import handoff +from agents.testing import ScriptedModel + +from .test_responses import get_text_message +from .testing_processor import fetch_ordered_spans + + +@pytest.mark.asyncio +async def test_turn_span_uses_model_visible_capability_identities() -> None: + @function_tool + def lookup_invoice() -> str: + return "invoice" + + namespaced_tools = tool_namespace( + name="billing", + description="Billing lookup tools.", + tools=[lookup_invoice], + ) + billing_agent = Agent(name="Billing Review") + billing_handoff = handoff( + billing_agent, + tool_name_override="escalate_to_billing", + ) + model = ScriptedModel(emit_traces=True) + model.extend([[get_text_message("done")]]) + agent = Agent( + name="test_agent", + model=model, + tools=namespaced_tools, + handoffs=[billing_handoff], + ) + + await Runner.run(agent, input="test") + + spans = fetch_ordered_spans() + turn_spans = [span for span in spans if span.span_data.type == "turn"] + assert len(turn_spans) == 1 + exported = turn_spans[0].export() + assert exported is not None + assert exported["span_data"]["data"]["tools"] == ["billing.lookup_invoice"] + assert exported["span_data"]["data"]["handoffs"] == ["escalate_to_billing"] + + agent_spans = [span for span in spans if span.span_data.type == "agent"] + assert len(agent_spans) == 1 + assert agent_spans[0].span_data.tools == ["lookup_invoice"] + assert agent_spans[0].span_data.handoffs == ["Billing Review"] From f09c5861622d0775d7cc8ba9360fff42a49b33d7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 23:33:34 +0000 Subject: [PATCH 3/3] chore: keep turn tracing diff focused --- src/agents/run_internal/run_loop.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 99bb5f7ae7..95943f2629 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -337,17 +337,12 @@ def _record_effective_model_capabilities( agent_handoff_names = [handoff.agent_name for handoff in handoffs] turn_handoff_names = [handoff.tool_name for handoff in handoffs] agent_tool_names = [ - tool_name - for tool in tools - if (tool_name := get_tool_trace_name_for_tool(tool)) is not None + tool_name for tool in tools if (tool_name := get_tool_trace_name_for_tool(tool)) is not None ] turn_tool_names = [ trace_name for tool in tools - if ( - trace_name := get_function_tool_trace_name(tool) - or get_tool_trace_name_for_tool(tool) - ) + if (trace_name := get_function_tool_trace_name(tool) or get_tool_trace_name_for_tool(tool)) is not None ] @@ -1205,7 +1200,7 @@ async def _save_max_turns_items( response_id=response_id, reasoning_item_id_policy=streamed_result._reasoning_item_id_policy, store=store_setting, - wrapper=context_wrapper, + wrapper=streamed_result.context_wrapper, ) streamed_result._current_turn_persisted_item_count += saved_count except BaseException: