diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_openai_web_search_usage.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_openai_web_search_usage.yaml new file mode 100644 index 000000000..c4d06cc83 --- /dev/null +++ b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_openai_web_search_usage.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"model":"gpt-4.1-mini","input":"Search the web for the current weather + in Paris and answer in one sentence.","tool_choice":{"type":"web_search_preview"},"tools":[{"type":"web_search_preview","search_context_size":"low"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '222' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.24.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.24.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0a5ce544de4a3d6c006ac3b80a613487d29c971a49ac3f162f\",\n + \ \"object\": \"response\",\n \"created_at\": 1791211530,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1791211531,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4.1-mini-2025-04-14\",\n \"moderation\": null,\n \"output\": [\n {\n + \ \"id\": \"ws_0a5ce544de4a3d6c006ac3b80a91a887d2937c490d3ec400ab\",\n + \ \"type\": \"web_search_call\",\n \"status\": \"completed\",\n \"action\": + {\n \"type\": \"search\",\n \"queries\": [\n \"Search + the web for the current weather in Paris and answer in one sentence.\"\n ],\n + \ \"query\": \"Search the web for the current weather in Paris and answer + in one sentence.\"\n }\n },\n {\n \"id\": \"msg_0a5ce544de4a3d6c006ac3b80b2e3487d2b8436d40532bbc6c\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"As of 4:45 PM on October + 5, 2026, Paris is experiencing sunny weather with a temperature of 77\\u00b0F + (25\\u00b0C).\\n\\n## Weather for Paris, \\u00cele-de-France, France:\\nCurrent + Conditions: Sunny, 77\\u00b0F (25\\u00b0C)\\n\\nDaily Forecast:\\n* Monday, + October 5: Low: 51\\u00b0F (10\\u00b0C), High: 77\\u00b0F (25\\u00b0C), Description: + Mostly sunny, very warm and pleasant\\n* Tuesday, October 6: Low: 54\\u00b0F + (12\\u00b0C), High: 76\\u00b0F (25\\u00b0C), Description: Remaining very warm + and pleasant with sunshine and a few clouds\\n* Wednesday, October 7: Low: + 51\\u00b0F (10\\u00b0C), High: 65\\u00b0F (18\\u00b0C), Description: Cooler + with considerable cloudiness; showers in the afternoon\\n* Thursday, October + 8: Low: 47\\u00b0F (8\\u00b0C), High: 61\\u00b0F (16\\u00b0C), Description: + Partly sunny with a couple of showers in the afternoon\\n* Friday, October + 9: Low: 55\\u00b0F (13\\u00b0C), High: 60\\u00b0F (16\\u00b0C), Description: + Occasional rain and drizzle\\n* Saturday, October 10: Low: 50\\u00b0F (10\\u00b0C), + High: 65\\u00b0F (18\\u00b0C), Description: Cloudy; a couple of showers in + the morning followed by occasional rain and drizzle in the afternoon\\n* Sunday, + October 11: Low: 52\\u00b0F (11\\u00b0C), High: 66\\u00b0F (19\\u00b0C), Description: + Cloudy\\n \"\n }\n ],\n \"role\": \"assistant\"\n }\n + \ ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": + null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": \"in_memory\",\n + \ \"reasoning\": {\n \"context\": null,\n \"effort\": null,\n \"summary\": + null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"auto\",\n + \ \"store\": true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": + {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n },\n + \ \"tool_choice\": {\n \"type\": \"web_search_preview\"\n },\n \"tool_usage\": + {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": + 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": + 1\n }\n },\n \"tools\": [\n {\n \"type\": \"web_search_preview\",\n + \ \"search_content_types\": [\n \"text\"\n ],\n \"search_context_size\": + \"low\",\n \"user_location\": {\n \"type\": \"approximate\",\n + \ \"city\": null,\n \"country\": \"US\",\n \"region\": + null,\n \"timezone\": null\n }\n }\n ],\n \"top_logprobs\": + 0,\n \"top_p\": 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": + 323,\n \"input_tokens_details\": {\n \"cache_write_tokens\": 0,\n + \ \"cached_tokens\": 0\n },\n \"output_tokens\": 321,\n \"output_tokens_details\": + {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 644\n },\n + \ \"user\": null,\n \"metadata\": {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a45d35e01b12a3f1-YYZ + connection: + - keep-alive + content-length: + - '3989' + content-type: + - application/json + date: + - Mon, 05 Oct 2026 14:45:31 GMT + openai-processing-ms: + - '1548' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-request-id: + - req_e465943f119242d68d1dfdb66e1d7d22 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index cb5d1bc4a..7d517eb76 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -2018,6 +2018,28 @@ async def test_agent_run_anthropic_reasoning_tokens(memory_logger): # pylint: enable=unsupported-membership-test,unsubscriptable-object +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_agent_run_openai_web_search_usage(memory_logger): + """Capture native web-search usage reported in RequestUsage.details.""" + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": + pytest.skip("Native web search usage is covered by the latest pydantic-ai cassette") + + from pydantic_ai.capabilities import NativeTool + from pydantic_ai.native_tools import WebSearchTool + + agent = Agent("openai-responses:gpt-4.1-mini", capabilities=[NativeTool(WebSearchTool())]) + result = await agent.run("Search the web for the current weather in Paris and answer in one sentence.") + assert result.output + + spans = memory_logger.pop() + chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) + assert chat_span is not None, "chat span not found" + # pylint: disable=unsupported-membership-test,unsubscriptable-object + assert chat_span["metadata"]["pydantic_ai_usage"]["web_search_requests"] > 0 + # pylint: enable=unsupported-membership-test,unsubscriptable-object + + @pytest.mark.parametrize( "details_key", [ diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index d38064e5f..61256c4f2 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -1432,11 +1432,19 @@ def _extract_response_metrics( def _extract_response_metadata(response: Any) -> dict[str, Any] | None: usage = getattr(response, "usage", None) + metadata: dict[str, Any] = {} + audio_seconds = getattr(usage, "audio_seconds", None) - if audio_seconds is None: - return None + if audio_seconds is not None: + metadata["audio_seconds"] = float(audio_seconds) + + details = getattr(usage, "details", None) + if isinstance(details, Mapping): + web_search_requests = details.get("web_search_requests") + if web_search_requests is not None: + metadata["web_search_requests"] = web_search_requests - return {"pydantic_ai_usage": {"audio_seconds": float(audio_seconds)}} + return {"pydantic_ai_usage": metadata} if metadata else None class _ContextPropagatingAsyncContextManager(AbstractAsyncContextManager):