Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -2018,6 +2018,28 @@ async def test_agent_run_anthropic_reasoning_tokens(memory_logger):
# pylint: enable=unsupported-membership-test,unsubscriptable-object


@pytest.mark.vcr
@pytest.mark.asyncio
async def test_agent_run_openai_web_search_usage(memory_logger):
"""Capture native web-search usage reported in RequestUsage.details."""
if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest":
pytest.skip("Native web search usage is covered by the latest pydantic-ai cassette")

from pydantic_ai.capabilities import NativeTool
from pydantic_ai.native_tools import WebSearchTool

agent = Agent("openai-responses:gpt-4.1-mini", capabilities=[NativeTool(WebSearchTool())])
result = await agent.run("Search the web for the current weather in Paris and answer in one sentence.")
assert result.output

spans = memory_logger.pop()
chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None)
assert chat_span is not None, "chat span not found"
# pylint: disable=unsupported-membership-test,unsubscriptable-object
assert chat_span["metadata"]["pydantic_ai_usage"]["web_search_requests"] > 0
# pylint: enable=unsupported-membership-test,unsubscriptable-object


@pytest.mark.parametrize(
"details_key",
[
Expand Down
14 changes: 11 additions & 3 deletions py/src/braintrust/integrations/pydantic_ai/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1432,11 +1432,19 @@ def _extract_response_metrics(

def _extract_response_metadata(response: Any) -> dict[str, Any] | None:
usage = getattr(response, "usage", None)
metadata: dict[str, Any] = {}

audio_seconds = getattr(usage, "audio_seconds", None)
if audio_seconds is None:
return None
if audio_seconds is not None:
metadata["audio_seconds"] = float(audio_seconds)

details = getattr(usage, "details", None)
if isinstance(details, Mapping):
web_search_requests = details.get("web_search_requests")
if web_search_requests is not None:
metadata["web_search_requests"] = web_search_requests

return {"pydantic_ai_usage": {"audio_seconds": float(audio_seconds)}}
return {"pydantic_ai_usage": metadata} if metadata else None


class _ContextPropagatingAsyncContextManager(AbstractAsyncContextManager):
Expand Down
Loading