diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 8c9064e1f..cc25a21e9 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -6,6 +6,7 @@ import os import tempfile from pathlib import Path +from types import SimpleNamespace import pytest from braintrust import SpanCustomizer, logger, set_span_customizers @@ -58,6 +59,33 @@ def _single_span(logs, name): return matches[0] +def test_pipecat_observer_filters_metrics_from_other_processors(): + LLMTokenUsage = _import("pipecat.metrics.metrics.LLMTokenUsage") + LLMUsageMetricsData = _import("pipecat.metrics.metrics.LLMUsageMetricsData") + MetricsFrame = _import("pipecat.frames.frames.MetricsFrame") + + observer = BraintrustPipecatObserver() + processor = SimpleNamespace(name="OpenAILLMService#1") + frame = MetricsFrame( + data=[ + LLMUsageMetricsData( + processor=processor.name, + model="gpt-4o-mini", + value=LLMTokenUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + LLMUsageMetricsData( + processor="JevClassifier#1", + model="gpt-4o-mini", + value=LLMTokenUsage(prompt_tokens=100, completion_tokens=20, total_tokens=120), + ), + ] + ) + + observer._capture_metrics(frame, processor) + + assert observer._llm_metrics == {"prompt_tokens": 10, "completion_tokens": 5, "tokens": 15} + + @pytest.mark.asyncio async def test_span_customizer_redacts_incremental_tts_input(memory_logger): TTSStartedFrame = _import("pipecat.frames.frames.TTSStartedFrame") diff --git a/py/src/braintrust/integrations/pipecat/tracing.py b/py/src/braintrust/integrations/pipecat/tracing.py index a60c04d2f..d84d94466 100644 --- a/py/src/braintrust/integrations/pipecat/tracing.py +++ b/py/src/braintrust/integrations/pipecat/tracing.py @@ -456,9 +456,13 @@ def _discard_tts_audio(self, context_id: str) -> None: self._tts_audio_metadata.pop(context_id, None) def _capture_metrics(self, frame: Any, processor: Any) -> None: + processor_name = _processor_name(processor) for metric in getattr(frame, "data", []) or []: metric_type = type(metric).__name__ if metric_type == "LLMUsageMetricsData": + metric_processor = getattr(metric, "processor", None) + if processor_name is not None and metric_processor is not None and metric_processor != processor_name: + continue self._llm_metrics.update(_llm_usage_metrics(getattr(metric, "value", None))) self._llm_metadata.update(_metadata_from_metric(metric)) elif metric_type == "TTFBMetricsData" and self._llm_span is not None: