From dfae3b45509b901fac9ee9a77273133a9775c038 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 15:59:08 -0400 Subject: [PATCH 1/3] fix(pydantic-ai): record audio duration usage --- py/src/braintrust/integrations/pydantic_ai/tracing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index c704a6f85..5ec1af426 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -1400,6 +1400,10 @@ def _extract_response_metrics( if hasattr(usage, "input_audio_tokens") and usage.input_audio_tokens is not None: metrics["prompt_audio_tokens"] = float(usage.input_audio_tokens) + audio_seconds = getattr(usage, "audio_seconds", None) + if audio_seconds is not None: + metrics["audio_seconds"] = float(audio_seconds) + if hasattr(usage, "output_audio_tokens") and usage.output_audio_tokens is not None: metrics["completion_audio_tokens"] = float(usage.output_audio_tokens) From 1cc7a8ec87e34fe6b1b842037b9322090f7aecea Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 16:21:34 -0400 Subject: [PATCH 2/3] fix(pydantic-ai): keep audio duration in metadata --- .../test_pydantic_ai_integration.py | 30 +++++++++++++++++++ .../integrations/pydantic_ai/tracing.py | 20 +++++++++---- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index ef751745f..5c47389e3 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -180,6 +180,36 @@ def test_extract_response_metrics_leaf_fields(): # pylint: enable=unsupported-membership-test,unsubscriptable-object +def test_audio_seconds_is_recorded_as_metadata(): + """Keep Pydantic AI's nonstandard audio duration out of canonical metrics.""" + from braintrust.integrations.pydantic_ai.tracing import _extract_response_metadata, _extract_response_metrics + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.usage import RequestUsage + + supports_audio_seconds = hasattr(RequestUsage(), "audio_seconds") + if supports_audio_seconds: + usage = RequestUsage(input_tokens=10, input_audio_tokens=7, audio_seconds=2.5) + else: + usage = RequestUsage(input_tokens=10, input_audio_tokens=7) + + response = ModelResponse( + parts=[TextPart(content="spoken response")], + usage=usage, + ) + metrics = _extract_response_metrics(response, start_time=1.0, end_time=2.0) + metadata = _extract_response_metadata(response) + + assert metrics is not None + # pylint: disable=unsupported-membership-test,unsubscriptable-object + assert "audio_seconds" not in metrics + assert metrics["prompt_audio_tokens"] == 7 + if supports_audio_seconds: + assert metadata == {"pydantic_ai_usage": {"audio_seconds": 2.5}} + else: + assert metadata is None + # pylint: enable=unsupported-membership-test,unsubscriptable-object + + @pytest.mark.vcr @pytest.mark.asyncio async def test_direct_model_request_creates_nested_chat_span_without_class_scan(memory_logger, direct): diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 5ec1af426..d38064e5f 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -480,8 +480,9 @@ async def model_request_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: output = _shape_model_response(result) metrics = _extract_response_metrics(result, start_time, end_time) + response_metadata = _extract_response_metadata(result) - span.log(output=output, metrics=metrics) + span.log(output=output, metadata=response_metadata, metrics=metrics) return result def model_request_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): @@ -774,9 +775,11 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): metrics = _extract_response_metrics( final_response, self.start_time, end_time, self._first_token_time ) + response_metadata = _extract_response_metadata(final_response) else: metrics = _wrapper_span_metrics(self.start_time, end_time, self._first_token_time) - self.span_cm.log(output=output, metrics=metrics) + response_metadata = None + self.span_cm.log(output=output, metadata=response_metadata, metrics=metrics) except Exception as e: logger.debug(f"Failed to extract stream output/metrics: {e}") @@ -1400,10 +1403,6 @@ def _extract_response_metrics( if hasattr(usage, "input_audio_tokens") and usage.input_audio_tokens is not None: metrics["prompt_audio_tokens"] = float(usage.input_audio_tokens) - audio_seconds = getattr(usage, "audio_seconds", None) - if audio_seconds is not None: - metrics["audio_seconds"] = float(audio_seconds) - if hasattr(usage, "output_audio_tokens") and usage.output_audio_tokens is not None: metrics["completion_audio_tokens"] = float(usage.output_audio_tokens) @@ -1431,6 +1430,15 @@ def _extract_response_metrics( return metrics if metrics else None +def _extract_response_metadata(response: Any) -> dict[str, Any] | None: + usage = getattr(response, "usage", None) + audio_seconds = getattr(usage, "audio_seconds", None) + if audio_seconds is None: + return None + + return {"pydantic_ai_usage": {"audio_seconds": float(audio_seconds)}} + + class _ContextPropagatingAsyncContextManager(AbstractAsyncContextManager): def __init__(self, cm: Any, ctx: contextvars.Context): self._cm = cm From 418bfd37ad567de62d77e36f90739aa6c3c89c97 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 17:45:51 -0400 Subject: [PATCH 3/3] test(pydantic-ai): feature-detect audio usage field --- .../pydantic_ai/test_pydantic_ai_integration.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 5c47389e3..cb5d1bc4a 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -186,11 +186,10 @@ def test_audio_seconds_is_recorded_as_metadata(): from pydantic_ai.messages import ModelResponse, TextPart from pydantic_ai.usage import RequestUsage - supports_audio_seconds = hasattr(RequestUsage(), "audio_seconds") + usage = RequestUsage(input_tokens=10, input_audio_tokens=7) + supports_audio_seconds = hasattr(usage, "audio_seconds") if supports_audio_seconds: - usage = RequestUsage(input_tokens=10, input_audio_tokens=7, audio_seconds=2.5) - else: - usage = RequestUsage(input_tokens=10, input_audio_tokens=7) + setattr(usage, "audio_seconds", 2.5) response = ModelResponse( parts=[TextPart(content="spoken response")],