From c2e93f4082a24f404c5983c1270e33fc5d428cb5 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 8 Oct 2026 18:14:36 +0000 Subject: [PATCH 1/2] fix(adk): let Google GenAI own LLM spans for ADK model calls With auto_instrument(), each Gemini request made through Google ADK produced two nested `llm` spans with identical usage: ADK's `llm_call [...]` and Google GenAI's `generate_content`. Trace summaries counted the request's tokens and LLM-call count twice. ADK always dispatches model calls through a separately instrumented provider client, so that integration owns the `llm` leaf and its usage, matching the JS SDK's ADK plugin and the pydantic-ai, DSPy, and LlamaIndex integrations: - Type ADK's `llm_call [...]` span as `task` and stop logging token metrics and usage metadata on it. It keeps its input, output, model/provider/tools metadata, `llm_call_type`, and time to first token. - Have `setup_adk()` also enable the Google GenAI integration, so users who only call `setup_adk()` still get a Gemini `llm` span with usage. `auto_instrument()` already enables both and still respects `google_genai=False`. Tests replay the existing cassettes; no re-recording was needed: - `test_adk_captures_metrics` covers `setup_adk()`: one usage-bearing `generate_content` span under a `task` `llm_call`. - `test_auto_adk.py` covers `auto_instrument()`, the reported scenario. - The other ADK tests assert the same ownership rule. Fixes SDK-525. Co-Authored-By: Claude Opus 5.5 --- .../braintrust/integrations/adk/__init__.py | 9 ++ .../braintrust/integrations/adk/test_adk.py | 141 +++++++++--------- py/src/braintrust/integrations/adk/tracing.py | 45 +----- .../auto_test_scripts/test_auto_adk.py | 60 ++++++++ 4 files changed, 149 insertions(+), 106 deletions(-) diff --git a/py/src/braintrust/integrations/adk/__init__.py b/py/src/braintrust/integrations/adk/__init__.py index 5421ac1f9..a556f49eb 100644 --- a/py/src/braintrust/integrations/adk/__init__.py +++ b/py/src/braintrust/integrations/adk/__init__.py @@ -43,6 +43,12 @@ def setup_adk( """ Setup Braintrust integration with Google ADK. Will automatically patch Google ADK agents, runners, flows, and MCP tools for automatic tracing. + Also enables the Google GenAI integration, which ADK's Gemini models call + through. It owns the ``llm`` span and token usage for each Gemini request; + ADK's ``llm_call`` spans are ``task`` spans so usage is counted once. For + other ADK models (e.g. LiteLLM or Anthropic), enable that provider's + integration, or use ``braintrust.auto_instrument()``, to record usage. + If you prefer manual patching take a look at `wrap_agent`, `wrap_runner`, `wrap_workflow`, `wrap_flow`, and `wrap_mcp_tool`. Args: @@ -61,4 +67,7 @@ def setup_adk( if span == NOOP_SPAN: init_logger(project=project_name, api_key=api_key, project_id=project_id) + from braintrust.integrations.google_genai import GoogleGenAIIntegration + + GoogleGenAIIntegration.setup() return ADKIntegration.setup() diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index ed1f1a429..5a51cdbf5 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -8,6 +8,7 @@ from braintrust import logger from braintrust.integrations.adk import setup_adk from braintrust.integrations.adk.tracing import _create_thread_wrapper +from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.logger import Attachment from braintrust.test_helpers import init_test_logger from google.adk import Agent @@ -88,6 +89,24 @@ def _assert_llm_output_shape(output): assert output["avg_logprobs"] is not None +def _adk_model_call_spans(spans): + return [row for row in spans if row["span_attributes"]["name"].startswith("llm_call")] + + +def _assert_provider_owns_llm_spans(spans): + """Each ADK model call has one Google GenAI ``llm`` child, which alone carries usage.""" + model_call_ids = [row["span_id"] for row in _adk_model_call_spans(spans)] + llm_spans = [row for row in spans if row["span_attributes"]["type"] == "llm"] + assert model_call_ids + assert sorted(row["span_parents"][0] for row in llm_spans) == sorted(model_call_ids) + for row in llm_spans: + assert row["span_attributes"]["name"] == "generate_content" + assert row["context"]["span_origin"]["instrumentation"]["name"] == "google-genai-auto" + assert row["metrics"]["prompt_tokens"] > 0 + assert [row for row in spans if "prompt_tokens" in row.get("metrics", {})] == llm_spans + return llm_spans + + def get_weather(location: str): """Get the weather for a location.""" return { @@ -169,11 +188,13 @@ async def run_message(text: str) -> str: "What name did I tell you?", } - llm_spans = [row for row in spans if row["span_attributes"]["type"] == "llm"] - assert len(llm_spans) == 2 + assert len(_assert_provider_owns_llm_spans(spans)) == 2 + model_call_spans = _adk_model_call_spans(spans) follow_up_span = next( - span for span in llm_spans if "What name did I tell you?" in _extract_text_parts(span["input"]["contents"]) + span + for span in model_call_spans + if "What name did I tell you?" in _extract_text_parts(span["input"]["contents"]) ) follow_up_texts = _extract_text_parts(follow_up_span["input"]["contents"]) @@ -298,6 +319,7 @@ async def _run_async_impl(self, ctx: InvocationContext) -> AsyncGenerator[Event, "agent_run [child]", "call_llm", "llm_call [direct_response]", + "generate_content", } spans_by_name = {span["span_attributes"]["name"]: span for span in spans} assert len(spans) == len(expected_names) @@ -308,6 +330,7 @@ async def _run_async_impl(self, ctx: InvocationContext) -> AsyncGenerator[Event, assert parent_name_by_id[spans_by_name["agent_run [child]"]["span_parents"][0]] == "agent_run [custom_root]" assert parent_name_by_id[spans_by_name["call_llm"]["span_parents"][0]] == "agent_run [child]" assert parent_name_by_id[spans_by_name["llm_call [direct_response]"]["span_parents"][0]] == "call_llm" + assert parent_name_by_id[spans_by_name["generate_content"]["span_parents"][0]] == "llm_call [direct_response]" @pytest.mark.vcr @@ -365,11 +388,12 @@ async def test_adk_braintrust_integration(memory_logger): assert invocation_span["metadata"]["session_id"] == "test-session" # Verify LLM call spans - llm_spans = [row for row in spans if row["span_attributes"]["type"] == "llm"] - assert len(llm_spans) >= 2, "Should have at least 2 LLM calls (tool selection and response generation)" + _assert_provider_owns_llm_spans(spans) + model_call_spans = _adk_model_call_spans(spans) + assert len(model_call_spans) >= 2, "Should have at least 2 LLM calls (tool selection and response generation)" # Check tool selection LLM call - tool_selection_spans = [span for span in llm_spans if "tool_selection" in span["span_attributes"]["name"]] + tool_selection_spans = [span for span in model_call_spans if "tool_selection" in span["span_attributes"]["name"]] assert len(tool_selection_spans) > 0, "Missing tool selection LLM call" tool_selection_span = tool_selection_spans[0] @@ -382,16 +406,16 @@ async def test_adk_braintrust_integration(memory_logger): adk_spans = [row for row in spans if row["context"]["span_origin"]["instrumentation"]["name"] == "adk-auto"] span_types_by_origin = {row["span_attributes"]["type"] for row in adk_spans} - assert {"task", "llm", "tool"} <= span_types_by_origin, ( - f"adk-auto origin missing on task/llm/tool spans: {span_types_by_origin}" + assert span_types_by_origin == {"task", "tool"}, ( + f"adk-auto origin should be on task/tool spans only: {span_types_by_origin}" ) - for span in llm_spans: + for span in model_call_spans: meta = span["metadata"] assert meta.get("provider") == "google", ( f"Missing metadata.provider=google on {span['span_attributes']['name']}" ) - assert meta.get("model"), "Missing metadata.model on llm span" + assert meta.get("model"), "Missing metadata.model on llm_call span" assert meta.get("tools"), "metadata.tools should be non-empty for a tool-using agent" tool_names = [ fn.get("name") for tool_entry in meta["tools"] for fn in (tool_entry.get("function_declarations") or []) @@ -400,7 +424,9 @@ async def test_adk_braintrust_integration(memory_logger): assert "tools" not in span["input"].get("config", {}), "tools should not be in input.config" # Check response generation LLM call - response_gen_spans = [span for span in llm_spans if "response_generation" in span["span_attributes"]["name"]] + response_gen_spans = [ + span for span in model_call_spans if "response_generation" in span["span_attributes"]["name"] + ] assert len(response_gen_spans) > 0, "Missing response generation LLM call" response_span = response_gen_spans[0] @@ -640,18 +666,18 @@ async def test_adk_max_tokens_captures_content(memory_logger): spans = memory_logger.pop() # Find the LLM call span - llm_spans = [row for row in spans if row["span_attributes"]["type"] == "llm"] - assert len(llm_spans) > 0, "Missing LLM call span" + model_call_spans = _adk_model_call_spans(spans) + assert len(model_call_spans) > 0, "Missing LLM call span" - llm_span = llm_spans[0] - assert "output" in llm_span, "Missing output in LLM span" + model_call_span = model_call_spans[0] + assert "output" in model_call_span, "Missing output in LLM span" # Sampling config from generate_content_config is captured in input.config - config = llm_span["input"]["config"] + config = model_call_span["input"]["config"] assert config["max_output_tokens"] == 50 assert config["temperature"] == 0.7 - output = llm_span["output"] + output = model_call_span["output"] # When MAX_TOKENS is hit, we should still have content captured # The integration should merge content from earlier events if the final event lacks it @@ -749,8 +775,8 @@ async def test_adk_binary_data_attachment_conversion(memory_logger): assert "89504e47" not in span_str.lower(), "Raw binary data (hex) should not be in logged span" # Find LLM spans and verify they also don't contain raw binary - llm_spans = [row for row in spans if row["span_attributes"]["type"] == "llm"] - assert len(llm_spans) > 0, "Should have LLM spans" + llm_spans = [row for row in spans if row["span_attributes"]["name"].startswith(("llm_call", "generate_content"))] + assert len(llm_spans) == 2, "Should have ADK and Google GenAI model-call spans" for llm_span in llm_spans: if "input" in llm_span and "contents" in llm_span["input"]: @@ -790,7 +816,7 @@ async def test_adk_usage_metadata_metrics(memory_logger): assert usage_metadata.thoughts_token_count > 0 spans = memory_logger.pop() - llm_spans = [row for row in spans if row["span_attributes"].get("type") == "llm"] + llm_spans = _assert_provider_owns_llm_spans(spans) assert len(llm_spans) == 1 llm_span = llm_spans[0] metrics = llm_span["metrics"] @@ -833,49 +859,29 @@ async def test_adk_captures_metrics(memory_logger): spans = memory_logger.pop() - # Find LLM spans - llm_spans = [row for row in spans if row["span_attributes"].get("type") == "llm"] - assert len(llm_spans) > 0, "Should have LLM spans" - - # Verify metrics are present in at least one LLM span - llm_span_with_metrics = None - for llm_span in llm_spans: - if "metrics" in llm_span and llm_span["metrics"]: - llm_span_with_metrics = llm_span - break - - assert llm_span_with_metrics is not None, "At least one LLM span should have metrics" + # Google GenAI owns the provider call: exactly one usage-bearing ``llm`` span + # per Gemini request, so trace totals count its usage once. + (llm_span,) = _assert_provider_owns_llm_spans(spans) + assert llm_span["metadata"]["model"] == ADK_MODEL + assert llm_span["metadata"]["provider"] == "google" - metrics = llm_span_with_metrics["metrics"] - - # Verify core token metrics are present - assert "prompt_tokens" in metrics, "Metrics should include prompt_tokens" - assert "completion_tokens" in metrics, "Metrics should include completion_tokens" - assert "tokens" in metrics, "Metrics should include total tokens" - - # Verify token counts are reasonable - assert metrics["prompt_tokens"] > 0, "prompt_tokens should be greater than 0" - assert metrics["completion_tokens"] > 0, "completion_tokens should be greater than 0" - assert metrics["tokens"] > 0, "total tokens should be greater than 0" - assert metrics["tokens"] == metrics["prompt_tokens"] + metrics["completion_tokens"], ( - "total tokens should equal prompt + completion tokens" + assert_metrics_are_valid(llm_span["metrics"]) + assert ( + llm_span["metrics"]["tokens"] + == llm_span["metrics"]["prompt_tokens"] + llm_span["metrics"]["completion_tokens"] ) - # Verify time to first token is captured for streaming responses - assert "time_to_first_token" in metrics, "Metrics should include time_to_first_token" - assert metrics["time_to_first_token"] > 0, "time_to_first_token should be greater than 0" - assert metrics["time_to_first_token"] < 10, "time_to_first_token should be reasonable (< 10 seconds)" - - # Verify model name is captured in metadata - metadata = llm_span_with_metrics.get("metadata", {}) - assert "model" in metadata, "Metadata should include model name" - assert metadata["model"] == ADK_MODEL, "Model name should match the agent's model" - assert metadata.get("provider") == "google", "Metadata should include provider=google" - - # No tools configured — _determine_llm_call_type should mark this as direct_response - assert "direct_response" in llm_span_with_metrics["span_attributes"]["name"], ( - f"Expected direct_response call type, got {llm_span_with_metrics['span_attributes']['name']}" - ) + # ADK's model-call span keeps its readable request/response and call type, + # but as a task parent of the provider span. + (adk_model_span,) = _adk_model_call_spans(spans) + assert adk_model_span["span_attributes"]["type"] == "task" + assert adk_model_span["span_attributes"]["name"] == "llm_call [direct_response]" + assert adk_model_span["context"]["span_origin"]["instrumentation"]["name"] == "adk-auto" + assert adk_model_span["metadata"]["model"] == ADK_MODEL + assert adk_model_span["metadata"]["provider"] == "google" + assert "usage_metadata" in adk_model_span["output"] + assert "time_to_first_token" in adk_model_span["metrics"] + assert 0 < adk_model_span["metrics"]["time_to_first_token"] < 10 # _determine_llm_call_type paths are exercised through the VCR-backed @@ -1027,21 +1033,20 @@ async def test_adk_structured_output_schema(memory_logger, agent_kwargs, prompt, spans = memory_logger.pop() # Find LLM span with the schema - llm_spans_with_schema = [ + model_call_spans_with_schema = [ span - for span in spans - if span["span_attributes"]["type"] == "llm" - and "input" in span + for span in _adk_model_call_spans(spans) + if "input" in span and "config" in span["input"] and span["input"]["config"].get(schema_field) is not None ] - assert len(llm_spans_with_schema) > 0, f"Should have at least one LLM call with {schema_field}" + assert len(model_call_spans_with_schema) > 0, f"Should have at least one LLM call with {schema_field}" - llm_span = llm_spans_with_schema[0] + model_call_span = model_call_spans_with_schema[0] # Assert complete input structure - assert llm_span["input"] == { + assert model_call_span["input"] == { "model": ADK_MODEL, "contents": [ { @@ -1057,7 +1062,7 @@ async def test_adk_structured_output_schema(memory_logger, agent_kwargs, prompt, "live_connect_config": ANY, } - _assert_llm_output_shape(llm_span["output"]) + _assert_llm_output_shape(model_call_span["output"]) class TestAutoInstrumentADK: diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index ce6f4b352..239725638 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -12,11 +12,7 @@ from typing import Any from braintrust.bt_json import bt_safe_deep_copy -from braintrust.integrations.utils import ( - _extract_google_usage_metadata_metrics, - _extract_google_usage_metadata_provider_metadata, - _materialize_attachment, -) +from braintrust.integrations.utils import _materialize_attachment from braintrust.logger import start_span as _bt_start_span @@ -177,19 +173,6 @@ def _capture_config(config: Any) -> dict[str, Any] | Any: return captured or config -def _extract_metrics(response: Any) -> dict[str, Any] | None: - """Extract token usage metrics from Google GenAI response.""" - if not response: - return None - - usage_metadata = getattr(response, "usage_metadata", None) - if not usage_metadata: - return None - - metrics = _extract_google_usage_metadata_metrics(usage_metadata) - return metrics if metrics else None - - def _extract_model_name(response: Any, llm_request: Any, instance: Any) -> str | None: """Extract model name from Google GenAI response, request, or flow instance.""" # Try to get from response first @@ -508,12 +491,14 @@ async def _trace(): # Create span BEFORE execution so child spans (like mcp_tool) have proper parent # Start with generic name - we'll update it after we see the response + # A task, not an llm span: the provider client ADK calls (google-genai, + # litellm, ...) owns the llm span and its usage. with _start_stream_span( name="llm_call", - type=SpanTypeAttribute.LLM, + type=SpanTypeAttribute.TASK, input=captured_request, metadata=metadata, - ) as llm_span: + ) as model_call_span: # Execute the LLM call and yield events while span is active last_event = None event_with_content = None @@ -536,32 +521,16 @@ async def _trace(): # We need to check if we should merge content from an earlier event. output = _event_output_with_content(last_event, event_with_content) - # Extract metrics from response - metrics = _extract_metrics(last_event) - - # Add time to first token if we captured it - if first_token_time is not None: - if metrics is None: - metrics = {} - metrics["time_to_first_token"] = first_token_time - start_time - # Determine the actual call type based on the response call_type = _determine_llm_call_type(llm_request, last_event) # Update span name with the specific call type now that we know it - llm_span.set_attributes( + model_call_span.set_attributes( name=f"llm_call [{call_type}]", span_attributes={"llm_call_type": call_type}, ) - # Log output, metrics, and provider-specific usage details. - llm_span.log( - output=output, - metrics=metrics, - metadata=_extract_google_usage_metadata_provider_metadata( - getattr(last_event, "usage_metadata", None) - ), - ) + model_call_span.log(output=output, metrics={"time_to_first_token": first_token_time - start_time}) async with aclosing(_trace()) as agen: async for event in agen: diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py index 0e0928415..779d40cfc 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_adk.py @@ -1,15 +1,19 @@ """Test auto_instrument for Google ADK.""" +import asyncio import importlib +import os from importlib.metadata import version as pkg_version from braintrust.auto import auto_instrument +from braintrust.conftest import get_vcr_config from braintrust.integrations.adk.patchers import ( AgentRunAsyncPatcher, _RunnerRunAsyncSubPatcher, _ThreadBridgePlatformSubPatcher, _ThreadBridgeRunnersSubPatcher, ) +from braintrust.integrations.test_utils import assert_metrics_are_valid, autoinstrument_test_context from google.adk import runners as adk_runners from google.adk.agents import BaseAgent from google.adk.runners import Runner @@ -47,5 +51,61 @@ def is_patched(target, patcher): assert is_patched(BaseAgent.run_async, AgentRunAsyncPatcher) assert is_patched(Runner.run_async, _RunnerRunAsyncSubPatcher) assert not is_patched(Runner.run, _RunnerRunAsyncSubPatcher) +assert results2.get("google_genai") == True + + +# 5. With ADK and Google GenAI both instrumented, a Gemini request emits one +# usage-bearing ``llm`` span. +async def run_agent(): + from google.adk import Agent + from google.adk.sessions import InMemorySessionService + from google.genai import types + + agent = Agent( + name="metrics_agent", + model=( + "gemini-2.5-flash-lite" + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") in {"latest", "2.6.3"} + else "gemini-2.0-flash" + ), + instruction="You are a helpful assistant.", + ) + session_service = InMemorySessionService() + await session_service.create_session( + app_name="metrics_app", user_id="test-user", session_id="test-session-metrics" + ) + runner = Runner(agent=agent, app_name="metrics_app", session_service=session_service) + message = types.Content(role="user", parts=[types.Part(text="Say hello in 3 words")]) + async for _ in runner.run_async(user_id="test-user", session_id="test-session-metrics", new_message=message): + pass + + +os.environ.setdefault("GOOGLE_API_KEY", "test-google-api-key") + + +def uppercase_method(request): + # google-genai sends lowercase HTTP methods; cassettes record them uppercased. + request.method = request.method.upper() + return request + + +vcr_config = {**get_vcr_config(), "before_record_request": uppercase_method} +with autoinstrument_test_context( + "test_adk_captures_metrics", integration="adk", vcr_config=vcr_config +) as memory_logger: + asyncio.run(run_agent()) + spans = memory_logger.pop() + +usage_spans = [span for span in spans if "prompt_tokens" in span.get("metrics", {})] +assert [span["span_attributes"]["name"] for span in usage_spans] == ["generate_content"], usage_spans +(llm_span,) = usage_spans +assert llm_span["span_attributes"]["type"] == "llm" +assert llm_span["context"]["span_origin"]["instrumentation"]["name"] == "google-genai-auto" +assert_metrics_are_valid(llm_span["metrics"]) + +(adk_model_span,) = [span for span in spans if span["span_attributes"]["name"].startswith("llm_call")] +assert adk_model_span["span_attributes"]["type"] == "task" +assert llm_span["span_parents"] == [adk_model_span["span_id"]] +assert [span for span in spans if span["span_attributes"]["type"] == "llm"] == [llm_span] print("SUCCESS") From e80ebcc3b4eebe769e1edd3eeb74672c81a08121 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 9 Oct 2026 16:23:39 +0000 Subject: [PATCH 2/2] test(adk): drop assertions Google GenAI now owns With Google GenAI owning the `llm` span and usage for ADK model calls, some ADK tests only re-checked GenAI behavior: - Remove `test_adk_usage_metadata_metrics` and its cassettes. ADK no longer computes usage, and `test_google_search_grounding` already covers tool-use prompt tokens, reasoning tokens and `usage_by_modality` for the same Google Search response shape in four call modes. - Trim `test_adk_captures_metrics` to the ownership rule and the ADK `llm_call` span's shape (task type, call type, time to first token). - Check only ADK's own model-call spans for raw binary in `test_adk_binary_data_attachment_conversion`; GenAI's attachment handling is covered in its own tests. Co-Authored-By: Claude Opus 5.5 --- .../test_adk_usage_metadata_metrics.yaml | 133 ----------------- .../test_adk_usage_metadata_metrics.yaml | 135 ------------------ .../test_adk_usage_metadata_metrics.yaml | 134 ----------------- .../braintrust/integrations/adk/test_adk.py | 90 ++---------- 4 files changed, 13 insertions(+), 479 deletions(-) delete mode 100644 py/src/braintrust/integrations/adk/cassettes/1.14.1/test_adk_usage_metadata_metrics.yaml delete mode 100644 py/src/braintrust/integrations/adk/cassettes/2.6.3/test_adk_usage_metadata_metrics.yaml delete mode 100644 py/src/braintrust/integrations/adk/cassettes/latest/test_adk_usage_metadata_metrics.yaml diff --git a/py/src/braintrust/integrations/adk/cassettes/1.14.1/test_adk_usage_metadata_metrics.yaml b/py/src/braintrust/integrations/adk/cassettes/1.14.1/test_adk_usage_metadata_metrics.yaml deleted file mode 100644 index bf068b1d6..000000000 --- a/py/src/braintrust/integrations/adk/cassettes/1.14.1/test_adk_usage_metadata_metrics.yaml +++ /dev/null @@ -1,133 +0,0 @@ -interactions: -- request: - body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, - Japan? Answer in one sentence."}], "role": "user"}], "systemInstruction": {"parts": - [{"text": "Use Google Search to answer the user''s question accurately and concisely.\n\nYou - are an agent. Your internal name is \"usage_metadata_agent\"."}], "role": "user"}, - "tools": [{"googleSearch": {}}], "generationConfig": {}}' - headers: - Content-Type: - - application/json - user-agent: - - google-genai-sdk/1.75.0 gl-python/3.14.6 google-adk/1.14.1 gl-python/3.14.6 - x-goog-api-client: - - google-genai-sdk/1.75.0 gl-python/3.14.6 google-adk/1.14.1 gl-python/3.14.6 - method: POST - uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent - response: - body: - string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": - [\n {\n \"text\": \"As of 2026, the estimated total population - of Tokyo is 14,270,748.\"\n }\n ],\n \"role\": \"model\"\n - \ },\n \"finishReason\": \"STOP\",\n \"index\": 0,\n \"groundingMetadata\": - {\n \"searchEntryPoint\": {\n \"renderedContent\": \"\\u003cstyle\\u003e\\n.container - {\\n align-items: center;\\n border-radius: 8px;\\n display: flex;\\n font-family: - Google Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n - \ padding: 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: - solid 1px;\\n border-radius: 16px;\\n min-width: 14px;\\n padding: 5px - 16px;\\n text-align: center;\\n user-select: none;\\n margin: 0 8px;\\n - \ -webkit-tap-highlight-color: transparent;\\n}\\n.carousel {\\n overflow: - auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n margin-right: - -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: 4px;\\n}\\n.gradient-container - {\\n position: relative;\\n}\\n.gradient {\\n position: absolute;\\n transform: - translate(3px, -9px);\\n height: 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: - light) {\\n .container {\\n background-color: #fafafa;\\n box-shadow: - 0 0 0 1px #0000000f;\\n }\\n .headline-label {\\n color: #1f1f1f;\\n - \ }\\n .chip {\\n background-color: #ffffff;\\n border-color: #d2d2d2;\\n - \ color: #5e5e5e;\\n text-decoration: none;\\n }\\n .chip:hover {\\n - \ background-color: #f2f2f2;\\n }\\n .chip:focus {\\n background-color: - #f2f2f2;\\n }\\n .chip:active {\\n background-color: #d8d8d8;\\n border-color: - #b6b6b6;\\n }\\n .logo-dark {\\n display: none;\\n }\\n .gradient {\\n - \ background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\\n }\\n}\\n@media - (prefers-color-scheme: dark) {\\n .container {\\n background-color: #1f1f1f;\\n - \ box-shadow: 0 0 0 1px #ffffff26;\\n }\\n .headline-label {\\n color: - #fff;\\n }\\n .chip {\\n background-color: #2c2c2c;\\n border-color: - #3c4043;\\n color: #fff;\\n text-decoration: none;\\n }\\n .chip:hover - {\\n background-color: #353536;\\n }\\n .chip:focus {\\n background-color: - #353536;\\n }\\n .chip:active {\\n background-color: #464849;\\n border-color: - #53575b;\\n }\\n .logo-light {\\n display: none;\\n }\\n .gradient - {\\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n - \ }\\n}\\n\\u003c/style\\u003e\\n\\u003cdiv class=\\\"container\\\"\\u003e\\n - \ \\u003cdiv class=\\\"headline\\\"\\u003e\\n \\u003csvg class=\\\"logo-light\\\" - width=\\\"18\\\" height=\\\"18\\\" viewBox=\\\"9 9 35 35\\\" fill=\\\"none\\\" - xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 - 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 - 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z\\\" - fill=\\\"#4285F4\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 - 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 - 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 - 43.8555 26.3109 43.8555V43.8555Z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath - fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M16.6559 29.8904C16.3111 - 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 - 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 - 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z\\\" - fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 - 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 - 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 - 16.2386 26.3109 16.2386V16.2386Z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n \\u003c/svg\\u003e\\n - \ \\u003csvg class=\\\"logo-dark\\\" width=\\\"18\\\" height=\\\"18\\\" - viewBox=\\\"0 0 48 48\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n - \ \\u003ccircle cx=\\\"24\\\" cy=\\\"23\\\" fill=\\\"#FFF\\\" r=\\\"22\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 - 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 - 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 - 4.06-2.85h.8z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath d=\\\"M15.59 - 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 - 1.53-6.64l1-.22 3.81 2.98.22 1.08z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath - d=\\\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 - 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n - \ \\u003c/svg\\u003e\\n \\u003cdiv class=\\\"gradient-container\\\"\\u003e\\u003cdiv - class=\\\"gradient\\\"\\u003e\\u003c/div\\u003e\\u003c/div\\u003e\\n \\u003c/div\\u003e\\n - \ \\u003cdiv class=\\\"carousel\\\"\\u003e\\n \\u003ca class=\\\"chip\\\" - href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQH9hELTUaoQVeZdpYUpyo2tui_DlXnI201Tl6CbR859pFwuXQXu9JMa6Ejb9nAs7U8l0gU0sGaV9jF3syljqRMrpOd0_xxZ8Rz4izTjYtjVyWHajQXjbafTrDXlY2mW-CMpUeEm5Wg0q63vt6C4pSR5PEDbueY7gXfgStqeNhtLgaYeNaS6WvgCa1hAoUWWZOQMeKChF_006JwKqoWXuWISSkomnQ==\\\"\\u003ecurrent - population of Tokyo, Japan\\u003c/a\\u003e\\n \\u003c/div\\u003e\\n\\u003c/div\\u003e\\n\"\n - \ },\n \"groundingChunks\": [\n {\n \"web\": - {\n \"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQEpi7ONwA9RPjVU3PSHeR1t9uzsnQEIRM4YkGBYScGS_rM2QOBJVYBMgVwlDCatTBo8TbZds6lbK2jnOLtT41q5botXEYDZavDBzTCjnhjhkeSzOcyqukPUFfUk6M4maEBDIurGF0FCSoCt_vzk\",\n - \ \"title\": \"wikipedia.org\"\n }\n }\n ],\n - \ \"groundingSupports\": [\n {\n \"segment\": {\n - \ \"endIndex\": 66,\n \"text\": \"As of 2026, the - estimated total population of Tokyo is 14,270,748.\"\n },\n \"groundingChunkIndices\": - [\n 0\n ]\n }\n ],\n \"webSearchQueries\": - [\n \"current population of Tokyo, Japan\"\n ]\n }\n - \ }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 48,\n \"candidatesTokenCount\": - 44,\n \"totalTokenCount\": 760,\n \"promptTokensDetails\": [\n {\n - \ \"modality\": \"TEXT\",\n \"tokenCount\": 48\n }\n ],\n - \ \"toolUsePromptTokenCount\": 100,\n \"toolUsePromptTokensDetails\": - [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 100\n - \ }\n ],\n \"thoughtsTokenCount\": 568,\n \"serviceTier\": \"standard\"\n - \ },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"AhByarLrMtzH-8YP3uDUcQ\"\n}\n" - headers: - Alt-Svc: - - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 - Content-Type: - - application/json; charset=UTF-8 - Date: - - Tue, 04 Aug 2026 16:15:02 GMT - Server: - - scaffolding on HTTPServer2 - Server-Timing: - - gfet4t7; dur=4018 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-Gemini-Service-Tier: - - standard - X-XSS-Protection: - - '0' - content-length: - - '6819' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/adk/cassettes/2.6.3/test_adk_usage_metadata_metrics.yaml b/py/src/braintrust/integrations/adk/cassettes/2.6.3/test_adk_usage_metadata_metrics.yaml deleted file mode 100644 index 18df209cd..000000000 --- a/py/src/braintrust/integrations/adk/cassettes/2.6.3/test_adk_usage_metadata_metrics.yaml +++ /dev/null @@ -1,135 +0,0 @@ -interactions: -- request: - body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, - Japan? Answer in one sentence."}], "role": "user"}], "systemInstruction": {"parts": - [{"text": "Use Google Search to answer the user''s question accurately and concisely.\n\nYou - are an agent. Your internal name is \"usage_metadata_agent\"."}], "role": "user"}, - "tools": [{"googleSearch": {}}], "generationConfig": {}}' - headers: - Content-Type: - - application/json - user-agent: - - google-genai-sdk/2.22.0 gl-python/3.14.6 google-adk/2.6.3 gl-python/3.14.6 - x-goog-api-client: - - google-genai-sdk/2.22.0 gl-python/3.14.6 google-adk/2.6.3 gl-python/3.14.6 - method: POST - uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent - response: - body: - string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": - [\n {\n \"text\": \"The estimated total population of - Tokyo, Japan, is approximately 14.27 million as of October 1, 2025.\"\n }\n - \ ],\n \"role\": \"model\"\n },\n \"finishReason\": - \"STOP\",\n \"index\": 0,\n \"groundingMetadata\": {\n \"searchEntryPoint\": - {\n \"renderedContent\": \"\\u003cstyle\\u003e\\n.container {\\n - \ align-items: center;\\n border-radius: 8px;\\n display: flex;\\n font-family: - Google Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n - \ padding: 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: - solid 1px;\\n border-radius: 16px;\\n min-width: 14px;\\n padding: 5px - 16px;\\n text-align: center;\\n user-select: none;\\n margin: 0 8px;\\n - \ -webkit-tap-highlight-color: transparent;\\n}\\n.carousel {\\n overflow: - auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n margin-right: - -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: 4px;\\n}\\n.gradient-container - {\\n position: relative;\\n}\\n.gradient {\\n position: absolute;\\n transform: - translate(3px, -9px);\\n height: 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: - light) {\\n .container {\\n background-color: #fafafa;\\n box-shadow: - 0 0 0 1px #0000000f;\\n }\\n .headline-label {\\n color: #1f1f1f;\\n - \ }\\n .chip {\\n background-color: #ffffff;\\n border-color: #d2d2d2;\\n - \ color: #5e5e5e;\\n text-decoration: none;\\n }\\n .chip:hover {\\n - \ background-color: #f2f2f2;\\n }\\n .chip:focus {\\n background-color: - #f2f2f2;\\n }\\n .chip:active {\\n background-color: #d8d8d8;\\n border-color: - #b6b6b6;\\n }\\n .logo-dark {\\n display: none;\\n }\\n .gradient {\\n - \ background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\\n }\\n}\\n@media - (prefers-color-scheme: dark) {\\n .container {\\n background-color: #1f1f1f;\\n - \ box-shadow: 0 0 0 1px #ffffff26;\\n }\\n .headline-label {\\n color: - #fff;\\n }\\n .chip {\\n background-color: #2c2c2c;\\n border-color: - #3c4043;\\n color: #fff;\\n text-decoration: none;\\n }\\n .chip:hover - {\\n background-color: #353536;\\n }\\n .chip:focus {\\n background-color: - #353536;\\n }\\n .chip:active {\\n background-color: #464849;\\n border-color: - #53575b;\\n }\\n .logo-light {\\n display: none;\\n }\\n .gradient - {\\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n - \ }\\n}\\n\\u003c/style\\u003e\\n\\u003cdiv class=\\\"container\\\"\\u003e\\n - \ \\u003cdiv class=\\\"headline\\\"\\u003e\\n \\u003csvg class=\\\"logo-light\\\" - width=\\\"18\\\" height=\\\"18\\\" viewBox=\\\"9 9 35 35\\\" fill=\\\"none\\\" - xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 - 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 - 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z\\\" - fill=\\\"#4285F4\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 - 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 - 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 - 43.8555 26.3109 43.8555V43.8555Z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath - fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M16.6559 29.8904C16.3111 - 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 - 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 - 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z\\\" - fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 - 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 - 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 - 16.2386 26.3109 16.2386V16.2386Z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n \\u003c/svg\\u003e\\n - \ \\u003csvg class=\\\"logo-dark\\\" width=\\\"18\\\" height=\\\"18\\\" - viewBox=\\\"0 0 48 48\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n - \ \\u003ccircle cx=\\\"24\\\" cy=\\\"23\\\" fill=\\\"#FFF\\\" r=\\\"22\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 - 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 - 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 - 4.06-2.85h.8z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath d=\\\"M15.59 - 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 - 1.53-6.64l1-.22 3.81 2.98.22 1.08z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath - d=\\\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 - 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n - \ \\u003c/svg\\u003e\\n \\u003cdiv class=\\\"gradient-container\\\"\\u003e\\u003cdiv - class=\\\"gradient\\\"\\u003e\\u003c/div\\u003e\\u003c/div\\u003e\\n \\u003c/div\\u003e\\n - \ \\u003cdiv class=\\\"carousel\\\"\\u003e\\n \\u003ca class=\\\"chip\\\" - href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQFPWbv4wU6cpBYTIkncpS1DTtW3XTFlHJvZ1jHwkr4D8lYvKDh2nxa6nV-KJHynfDBFY98G1E6FvCi8TO-3c750xHb-P5KglDFiV-61fiQn_rmPGoH49lWWidHDoi510I-Mw5LuSMFkkMtPhElCKtPTZHWiPPLxu8CtXoQ3gilkn3kDtBHDSbLx8Q131EgofyIQMaH6_KncsNVyQoH433g-RHWm\\\"\\u003ecurrent - population of Tokyo Japan\\u003c/a\\u003e\\n \\u003c/div\\u003e\\n\\u003c/div\\u003e\\n\"\n - \ },\n \"groundingChunks\": [\n {\n \"web\": - {\n \"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQE-M_yDJfZXuKrCBsz-hIxkCguUwmJNGcPUgrwDvNyB02TmUnl9VXhgT7A9p4wlzyK8qAfJHFtHp-VT82EBUifI-2uss1QEuMm-MUQlpmNZNfjr834iAvIyNoNPWw9bsgsS0B1h8FvJIP3-96OgWDnZ\",\n - \ \"title\": \"tokyo.lg.jp\"\n }\n }\n ],\n - \ \"groundingSupports\": [\n {\n \"segment\": {\n - \ \"endIndex\": 101,\n \"text\": \"The estimated - total population of Tokyo, Japan, is approximately 14.27 million as of October - 1, 2025.\"\n },\n \"groundingChunkIndices\": [\n 0\n - \ ]\n }\n ],\n \"webSearchQueries\": [\n - \ \"current population of Tokyo Japan\"\n ]\n }\n }\n - \ ],\n \"usageMetadata\": {\n \"promptTokenCount\": 48,\n \"candidatesTokenCount\": - 46,\n \"totalTokenCount\": 508,\n \"promptTokensDetails\": [\n {\n - \ \"modality\": \"TEXT\",\n \"tokenCount\": 48\n }\n ],\n - \ \"toolUsePromptTokenCount\": 93,\n \"toolUsePromptTokensDetails\": - [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 93\n - \ }\n ],\n \"thoughtsTokenCount\": 321,\n \"serviceTier\": \"standard\"\n - \ },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"XPyaaq2WD5fB1MkP_PKU-AE\"\n}\n" - headers: - Alt-Svc: - - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 - Content-Type: - - application/json; charset=UTF-8 - Date: - - Fri, 04 Sep 2026 17:14:07 GMT - Server: - - scaffolding on HTTPServer2 - Server-Timing: - - gfet4t7; dur=2981 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-Gemini-Service-Tier: - - standard - X-XSS-Protection: - - '0' - content-length: - - '6885' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/adk/cassettes/latest/test_adk_usage_metadata_metrics.yaml b/py/src/braintrust/integrations/adk/cassettes/latest/test_adk_usage_metadata_metrics.yaml deleted file mode 100644 index 72fdfd462..000000000 --- a/py/src/braintrust/integrations/adk/cassettes/latest/test_adk_usage_metadata_metrics.yaml +++ /dev/null @@ -1,134 +0,0 @@ -interactions: -- request: - body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, - Japan? Answer in one sentence."}], "role": "user"}], "systemInstruction": {"parts": - [{"text": "Use Google Search to answer the user''s question accurately and concisely.\n\nYou - are an agent. Your internal name is \"usage_metadata_agent\"."}], "role": "user"}, - "tools": [{"googleSearch": {}}], "generationConfig": {}}' - headers: - Content-Type: - - application/json - user-agent: - - google-genai-sdk/2.28.0 gl-python/3.14.7 google-adk/2.11.0 gl-python/3.14.7 - x-goog-api-client: - - google-genai-sdk/2.28.0 gl-python/3.14.7 google-adk/2.11.0 gl-python/3.14.7 - method: POST - uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent - response: - body: - string: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": - [\n {\n \"text\": \"The current projected population of - Tokyo, Japan, is 14,270,748 as of 2026.\"\n }\n ],\n \"role\": - \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0,\n - \ \"groundingMetadata\": {\n \"searchEntryPoint\": {\n \"renderedContent\": - \"\\u003cstyle\\u003e\\n.container {\\n align-items: center;\\n border-radius: - 8px;\\n display: flex;\\n font-family: Google Sans, Roboto, sans-serif;\\n - \ font-size: 14px;\\n line-height: 20px;\\n padding: 8px 12px;\\n}\\n.chip - {\\n display: inline-block;\\n border: solid 1px;\\n border-radius: 16px;\\n - \ min-width: 14px;\\n padding: 5px 16px;\\n text-align: center;\\n user-select: - none;\\n margin: 0 8px;\\n -webkit-tap-highlight-color: transparent;\\n}\\n.carousel - {\\n overflow: auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n - \ margin-right: -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: - 4px;\\n}\\n.gradient-container {\\n position: relative;\\n}\\n.gradient {\\n - \ position: absolute;\\n transform: translate(3px, -9px);\\n height: 36px;\\n - \ width: 9px;\\n}\\n@media (prefers-color-scheme: light) {\\n .container - {\\n background-color: #fafafa;\\n box-shadow: 0 0 0 1px #0000000f;\\n - \ }\\n .headline-label {\\n color: #1f1f1f;\\n }\\n .chip {\\n background-color: - #ffffff;\\n border-color: #d2d2d2;\\n color: #5e5e5e;\\n text-decoration: - none;\\n }\\n .chip:hover {\\n background-color: #f2f2f2;\\n }\\n .chip:focus - {\\n background-color: #f2f2f2;\\n }\\n .chip:active {\\n background-color: - #d8d8d8;\\n border-color: #b6b6b6;\\n }\\n .logo-dark {\\n display: - none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #fafafa - 15%, #fafafa00 100%);\\n }\\n}\\n@media (prefers-color-scheme: dark) {\\n - \ .container {\\n background-color: #1f1f1f;\\n box-shadow: 0 0 0 1px - #ffffff26;\\n }\\n .headline-label {\\n color: #fff;\\n }\\n .chip - {\\n background-color: #2c2c2c;\\n border-color: #3c4043;\\n color: - #fff;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: - #353536;\\n }\\n .chip:focus {\\n background-color: #353536;\\n }\\n - \ .chip:active {\\n background-color: #464849;\\n border-color: #53575b;\\n - \ }\\n .logo-light {\\n display: none;\\n }\\n .gradient {\\n background: - linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n }\\n}\\n\\u003c/style\\u003e\\n\\u003cdiv - class=\\\"container\\\"\\u003e\\n \\u003cdiv class=\\\"headline\\\"\\u003e\\n - \ \\u003csvg class=\\\"logo-light\\\" width=\\\"18\\\" height=\\\"18\\\" - viewBox=\\\"9 9 35 35\\\" fill=\\\"none\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n - \ \\u003cpath fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M42.8622 - 27.0064C42.8622 25.7839 42.7525 24.6084 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 - 32.3041 33.9596 34.1222 32.1258 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 - 32.2571 42.8622 27.0064V27.0064Z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n \\u003cpath - fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 43.8555C30.9659 - 43.8555 34.8687 42.3195 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 - 37.0061 26.3109 37.0061C21.8282 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 - 39.9841 19.5712 43.8555 26.3109 43.8555V43.8555Z\\\" fill=\\\"#34A853\\\"/\\u003e\\n - \ \\u003cpath fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M16.6559 - 29.8904C16.3111 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 - 24.3733 16.6559 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 - 26.6146C9.06992 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 - 29.8904V29.8904Z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" - clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 - 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 - 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 - 16.2386 26.3109 16.2386V16.2386Z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n \\u003c/svg\\u003e\\n - \ \\u003csvg class=\\\"logo-dark\\\" width=\\\"18\\\" height=\\\"18\\\" - viewBox=\\\"0 0 48 48\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n - \ \\u003ccircle cx=\\\"24\\\" cy=\\\"23\\\" fill=\\\"#FFF\\\" r=\\\"22\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 - 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n - \ \\u003cpath d=\\\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 - 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 - 4.06-2.85h.8z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath d=\\\"M15.59 - 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 - 1.53-6.64l1-.22 3.81 2.98.22 1.08z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath - d=\\\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 - 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n - \ \\u003c/svg\\u003e\\n \\u003cdiv class=\\\"gradient-container\\\"\\u003e\\u003cdiv - class=\\\"gradient\\\"\\u003e\\u003c/div\\u003e\\u003c/div\\u003e\\n \\u003c/div\\u003e\\n - \ \\u003cdiv class=\\\"carousel\\\"\\u003e\\n \\u003ca class=\\\"chip\\\" - href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGCMZSLCe6o-n74OvBG9BkfBhlq2t44W_0g735PNwINkVEgF1xbRe-75Pmu9dMBt5HnsHnq_5WBj5dOtx3zbGNBfzeN0CY96ko3lJPdgelkujepUEUrQ0QdrZBKIeUxqUWM4eZ48-uPOtfvvkuaTdmUQA0IfQAQzLXPDUuEEVibrp7JylxvzWOw6lMSfo7thsiPiEIUdeE3NoWIzYCPLpGe5HIC2w==\\\"\\u003ecurrent - population of Tokyo Japan\\u003c/a\\u003e\\n \\u003c/div\\u003e\\n\\u003c/div\\u003e\\n\"\n - \ },\n \"groundingChunks\": [\n {\n \"web\": - {\n \"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQHPHg0TAUbggJybYQZEaRzEe2ql-6neGdPBRIJMvJBy2tW-iuL62CS5sSET4oNdJbnmzkIdgBCpUAbjPGT3Ardy0kt-25DQ830CxCF9aQXUHZRocz_4zvq8Vjlij3UCxeeyagM93Yle7XpAIpahxQ==\",\n - \ \"title\": \"wikipedia.org\"\n }\n }\n ],\n - \ \"groundingSupports\": [\n {\n \"segment\": {\n - \ \"endIndex\": 75,\n \"text\": \"The current projected - population of Tokyo, Japan, is 14,270,748 as of 2026.\"\n },\n - \ \"groundingChunkIndices\": [\n 0\n ]\n - \ }\n ],\n \"webSearchQueries\": [\n \"current - population of Tokyo Japan\"\n ]\n }\n }\n ],\n \"usageMetadata\": - {\n \"promptTokenCount\": 48,\n \"candidatesTokenCount\": 45,\n \"totalTokenCount\": - 721,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n - \ \"tokenCount\": 48\n }\n ],\n \"toolUsePromptTokenCount\": - 93,\n \"toolUsePromptTokensDetails\": [\n {\n \"modality\": - \"TEXT\",\n \"tokenCount\": 93\n }\n ],\n \"thoughtsTokenCount\": - 535,\n \"serviceTier\": \"standard\"\n },\n \"modelVersion\": \"gemini-2.5-flash\",\n - \ \"responseId\": \"D7jDaqD5FZ2G39IPub35gAk\"\n}\n" - headers: - Alt-Svc: - - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 - Content-Type: - - application/json; charset=UTF-8 - Date: - - Mon, 05 Oct 2026 14:45:39 GMT - Server: - - scaffolding on HTTPServer2 - Server-Timing: - - gfet4t7; dur=3789 - Transfer-Encoding: - - chunked - Vary: - - Origin - - X-Origin - - Referer - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - SAMEORIGIN - X-Gemini-Service-Tier: - - standard - X-XSS-Protection: - - '0' - content-length: - - '6838' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index 5a51cdbf5..4321ecdfb 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -8,7 +8,6 @@ from braintrust import logger from braintrust.integrations.adk import setup_adk from braintrust.integrations.adk.tracing import _create_thread_wrapper -from braintrust.integrations.test_utils import assert_metrics_are_valid from braintrust.logger import Attachment from braintrust.test_helpers import init_test_logger from google.adk import Agent @@ -20,7 +19,6 @@ from google.adk.events.event import Event from google.adk.runners import Runner from google.adk.sessions import InMemorySessionService -from google.adk.tools import google_search from google.genai import types from pydantic import BaseModel, Field @@ -774,69 +772,21 @@ async def test_adk_binary_data_attachment_conversion(memory_logger): assert b"\x89PNG".hex() not in span_str, "Raw binary data should not be in logged span" assert "89504e47" not in span_str.lower(), "Raw binary data (hex) should not be in logged span" - # Find LLM spans and verify they also don't contain raw binary - llm_spans = [row for row in spans if row["span_attributes"]["name"].startswith(("llm_call", "generate_content"))] - assert len(llm_spans) == 2, "Should have ADK and Google GenAI model-call spans" + # Find model-call spans and verify they also don't contain raw binary + model_call_spans = _adk_model_call_spans(spans) + assert len(model_call_spans) > 0, "Should have model-call spans" - for llm_span in llm_spans: - if "input" in llm_span and "contents" in llm_span["input"]: - llm_str = str(llm_span["input"]) + for model_call_span in model_call_spans: + if "input" in model_call_span and "contents" in model_call_span["input"]: + llm_str = str(model_call_span["input"]) assert b"\x89PNG".hex() not in llm_str, "Raw binary data should not be in LLM span input" assert "89504e47" not in llm_str.lower(), "Raw binary data (hex) should not be in LLM span input" -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_adk_usage_metadata_metrics(memory_logger): - """Google Search usage includes tool prompts and reasoning in normalized totals.""" - assert not memory_logger.pop() - - agent = Agent( - name="usage_metadata_agent", - model="gemini-2.5-flash", - instruction="Use Google Search to answer the user's question accurately and concisely.", - tools=[google_search], - ) - app_name = "usage_metadata_app" - user_id = "test-user" - session_id = "test-session-usage-metadata" - runner = await _create_runner(agent, app_name=app_name, user_id=user_id, session_id=session_id) - user_msg = types.Content( - role="user", - parts=[types.Part(text="What is the current population of Tokyo, Japan? Answer in one sentence.")], - ) - - events = [event async for event in runner.run_async(user_id=user_id, session_id=session_id, new_message=user_msg)] - usage_metadata = next( - (event.usage_metadata for event in reversed(events) if getattr(event, "usage_metadata", None) is not None), - None, - ) - assert usage_metadata is not None - assert usage_metadata.tool_use_prompt_token_count > 0 - assert usage_metadata.thoughts_token_count > 0 - - spans = memory_logger.pop() - llm_spans = _assert_provider_owns_llm_spans(spans) - assert len(llm_spans) == 1 - llm_span = llm_spans[0] - metrics = llm_span["metrics"] - - assert metrics["prompt_tokens"] == (usage_metadata.prompt_token_count + usage_metadata.tool_use_prompt_token_count) - assert metrics["completion_tokens"] == ( - usage_metadata.candidates_token_count + usage_metadata.thoughts_token_count - ) - assert metrics["completion_reasoning_tokens"] == usage_metadata.thoughts_token_count - assert metrics["tokens"] == usage_metadata.total_token_count - assert metrics["tokens"] == metrics["prompt_tokens"] + metrics["completion_tokens"] - assert llm_span["metadata"]["usage_by_modality"]["tool_use_prompt_tokens_details"] == [ - detail.model_dump(exclude_none=True) for detail in usage_metadata.tool_use_prompt_tokens_details - ] - - @pytest.mark.vcr @pytest.mark.asyncio async def test_adk_captures_metrics(memory_logger): - """Test that token usage metrics are captured from LLM responses.""" + """setup_adk() alone yields one usage-bearing Google GenAI ``llm`` span under a task ``llm_call``.""" assert not memory_logger.pop() agent = Agent( @@ -859,28 +809,16 @@ async def test_adk_captures_metrics(memory_logger): spans = memory_logger.pop() - # Google GenAI owns the provider call: exactly one usage-bearing ``llm`` span - # per Gemini request, so trace totals count its usage once. - (llm_span,) = _assert_provider_owns_llm_spans(spans) - assert llm_span["metadata"]["model"] == ADK_MODEL - assert llm_span["metadata"]["provider"] == "google" - - assert_metrics_are_valid(llm_span["metrics"]) - assert ( - llm_span["metrics"]["tokens"] - == llm_span["metrics"]["prompt_tokens"] + llm_span["metrics"]["completion_tokens"] - ) + # setup_adk() enables Google GenAI, which owns the one usage-bearing ``llm`` + # span per Gemini request, so trace totals count its usage once. + _assert_provider_owns_llm_spans(spans) - # ADK's model-call span keeps its readable request/response and call type, - # but as a task parent of the provider span. + # ADK's model-call span keeps its request/response, call type, and time to + # first token, as a task parent of the provider span. (adk_model_span,) = _adk_model_call_spans(spans) assert adk_model_span["span_attributes"]["type"] == "task" assert adk_model_span["span_attributes"]["name"] == "llm_call [direct_response]" - assert adk_model_span["context"]["span_origin"]["instrumentation"]["name"] == "adk-auto" - assert adk_model_span["metadata"]["model"] == ADK_MODEL - assert adk_model_span["metadata"]["provider"] == "google" assert "usage_metadata" in adk_model_span["output"] - assert "time_to_first_token" in adk_model_span["metrics"] assert 0 < adk_model_span["metrics"]["time_to_first_token"] < 10 @@ -1036,9 +974,7 @@ async def test_adk_structured_output_schema(memory_logger, agent_kwargs, prompt, model_call_spans_with_schema = [ span for span in _adk_model_call_spans(spans) - if "input" in span - and "config" in span["input"] - and span["input"]["config"].get(schema_field) is not None + if "input" in span and "config" in span["input"] and span["input"]["config"].get(schema_field) is not None ] assert len(model_call_spans_with_schema) > 0, f"Should have at least one LLM call with {schema_field}"