From ba84aa595829c10a7c1060086162d245067c4321 Mon Sep 17 00:00:00 2001 From: Kexin Chen <22360275@zju.edu.cn> Date: Fri, 2 Oct 2026 19:38:38 +0800 Subject: [PATCH 1/3] fix(anthropic): preserve MCP tool call context in spans --- .../integrations/anthropic/test_anthropic.py | 166 +++++++++++++++++- .../integrations/anthropic/tracing.py | 42 +++-- 2 files changed, 191 insertions(+), 17 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 03117e938..314182693 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -21,7 +21,7 @@ from braintrust.integrations.test_utils import verify_autoinstrument_script from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter PROJECT_NAME = "test-anthropic-app" @@ -346,11 +346,20 @@ def test_anthropic_beta_messages_create_captures_compaction_metadata(memory_logg @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) -def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory_logger): +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_async,vcr_cassette_name", + [ + (False, "test_anthropic_beta_messages_create_preserves_inline_mcp_blocks"), + (True, "test_anthropic_beta_messages_create_preserves_inline_mcp_blocks"), + ], + ids=["sync", "async"], +) +async def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory_logger, is_async, vcr_cassette_name): if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": pytest.skip("Inline MCP tool definitions require the latest Anthropic API") - client = wrap_anthropic(_get_client()) + client = wrap_anthropic(_get_async_client() if is_async else _get_client()) response = client.beta.messages.create( model=LATEST_MODEL, max_tokens=512, @@ -373,8 +382,11 @@ def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory_logge tools=[{"type": "mcp_toolset", "mcp_server_name": "braintrust-test"}], betas=["mcp-client-2026-09-15"], ) + if is_async: + response = await response - span = find_span_by_name(memory_logger.pop(), "anthropic.messages.create") + spans = memory_logger.pop() + span = find_span_by_name(spans, "anthropic.messages.create") assert "error" not in span assert span["metadata"]["tools"] == [{"type": "mcp_toolset", "mcp_server_name": "braintrust-test"}] @@ -398,6 +410,152 @@ def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory_logge assert tool_result["tool_use_id"] == response_tool_result.tool_use_id assert tool_result["content"][0]["text"] == response_tool_result.content[0].text + tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) + assert len(tool_spans) == 1 + tool_span = tool_spans[0] + assert tool_span["span_attributes"]["name"] == response_tool_use.name + assert tool_span["input"] == response_tool_use.input + assert tool_span["output"] == [block.model_dump() for block in response_tool_result.content] + assert tool_span["metadata"] == { + "tool_use_id": response_tool_use.id, + "tool_call_type": "mcp_tool_use", + "tool_result_type": "mcp_tool_result", + } + assert tool_span["span_parents"] == [span["span_id"]] + assert tool_span["root_span_id"] == span["root_span_id"] + + +@pytest.mark.parametrize("results_first", [False, True], ids=["calls-first", "results-first"]) +def test_anthropic_mcp_tool_spans_pair_by_id(memory_logger, results_first): + # Supplement the recorded provider response with deterministic ordering and + # incomplete-pair cases that cannot be requested reliably from a live model. + calls = [ + {"type": "mcp_tool_use", "id": "mcp-1", "name": "lookup", "input": {"key": "first"}, "server_name": "catalog"}, + { + "type": "mcp_tool_use", + "id": "mcp-2", + "name": "lookup", + "input": {"key": "second"}, + "server_name": "catalog", + }, + {"type": "mcp_tool_use", "id": "mcp-pending", "name": "pending", "input": {}, "server_name": "catalog"}, + ] + results = [ + {"type": "mcp_tool_result", "tool_use_id": "mcp-2", "content": [{"type": "text", "text": "second value"}]}, + {"type": "mcp_tool_result", "tool_use_id": "mcp-1", "content": [{"type": "text", "text": "first value"}]}, + { + "type": "mcp_tool_result", + "tool_use_id": "mcp-orphan", + "content": [{"type": "text", "text": "orphan value"}], + }, + ] + content = (results + calls if results_first else calls + results) + [ + {"type": "tool_use", "id": "client-tool", "name": "local_lookup", "input": {}}, + {"type": "tool_result", "tool_use_id": "client-tool", "content": "client result"}, + ] + before = json.dumps(content) + with logger.start_span(name="MCP pairs", type="llm") as parent: + _log_message_to_span(SimpleNamespace(content=content), parent) + + spans = memory_logger.pop() + llm_span = find_span_by_name(spans, "MCP pairs") + tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) + assert len(tool_spans) == 4 + by_id = {span["metadata"]["tool_use_id"]: span for span in tool_spans} + assert set(by_id) == {"mcp-1", "mcp-2", "mcp-pending", "mcp-orphan"} + for call, result in zip(calls[:2], reversed(results[:2])): + span = by_id[call["id"]] + assert span["span_attributes"]["name"] == call["name"] + assert span["input"] == call["input"] + assert span["output"] == result["content"] + assert span["metadata"]["tool_call_type"] == "mcp_tool_use" + assert span["metadata"]["tool_result_type"] == "mcp_tool_result" + assert by_id["mcp-pending"]["span_attributes"]["name"] == "pending" + assert by_id["mcp-pending"]["input"] == {} + assert by_id["mcp-pending"].get("output") is None + assert by_id["mcp-orphan"]["span_attributes"]["name"] == "mcp" + assert by_id["mcp-orphan"].get("input") is None + assert by_id["mcp-orphan"]["output"] == results[2]["content"] + for span in tool_spans: + assert span["span_parents"] == [llm_span["span_id"]] + assert span["root_span_id"] == llm_span["root_span_id"] + assert llm_span["output"]["content"] == content + assert json.dumps(content) == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_anthropic_mcp_stream_accumulates_tool_input(memory_logger, is_async): + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": + pytest.skip("MCP stream events require the latest Anthropic SDK") + + # Supplemental typed-event coverage: no MCP SSE cassette is available. + # Split input across deltas to exercise the real SDK stream accumulator. + call = { + "type": "mcp_tool_use", + "id": "mcp-1", + "name": "lookup", + "input": {"key": "record"}, + "server_name": "catalog", + } + result = { + "type": "mcp_tool_result", + "tool_use_id": "mcp-1", + "is_error": False, + "content": [{"type": "text", "text": "found"}], + } + wire_events = [ + { + "type": "message_start", + "message": { + "id": "msg_mcp", + "type": "message", + "role": "assistant", + "model": LATEST_MODEL, + "content": [], + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": dict(call, input={})}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"key":'}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": ' "record"}'}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": result}, + {"type": "content_block_stop", "index": 1}, + {"type": "message_stop"}, + ] + adapter = TypeAdapter(anthropic.types.beta.BetaRawMessageStreamEvent) + events = [adapter.validate_python(event) for event in wire_events] + before = [event.model_dump() for event in events] + + async def async_events(): + for event in events: + yield event + + with logger.start_span(name="MCP stream", type="llm") as parent: + source = async_events() if is_async else iter(events) + stream = TracedMessageStream(source, parent, time.time()) + observed = [event async for event in stream] if is_async else list(stream) + stream._log_final_message() + assert observed == events + assert [event.model_dump() for event in events] == before + spans = memory_logger.pop() + parent = find_span_by_name(spans, "MCP stream") + tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) + assert len(tool_spans) == 1 + child = tool_spans[0] + assert child["span_attributes"]["name"] == call["name"] + assert child["input"] == call["input"] + assert child["output"][0]["text"] == "found" + assert child["metadata"]["tool_use_id"] == call["id"] + assert child["metadata"]["tool_call_type"] == "mcp_tool_use" + assert child["span_parents"] == [parent["span_id"]] + assert parent["output"]["content"][0]["input"] == call["input"] + @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) def test_anthropic_messages_create_captures_refusal_stop_details(memory_logger): diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 01a5552cc..c04f90d39 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -42,20 +42,24 @@ def accumulate_event(event=None, current_snapshot=None, **kwargs): return current_snapshot -def _accumulate_event_accepts_json_bufs() -> bool: - """Return whether ``accumulate_event`` takes the ``json_bufs`` keyword. +try: + from anthropic.lib.streaming._beta_messages import accumulate_event as accumulate_beta_event + from anthropic.types.beta import BetaRawMessageStartEvent +except ImportError: + accumulate_beta_event = accumulate_event + BetaRawMessageStartEvent = None - anthropic>=1.5.0 buffers partial tool-input JSON per content block in a - caller-owned ``json_bufs`` dict and requires the keyword; older releases - reject it. - """ + +def _accumulate_event_parameters(accumulator) -> set[str]: + """Discover buffer/header keywords added in newer Anthropic SDK versions.""" try: - return "json_bufs" in inspect.signature(accumulate_event).parameters + return set(inspect.signature(accumulator).parameters) except (TypeError, ValueError): - return False + return set() -_ACCUMULATE_EVENT_ACCEPTS_JSON_BUFS = _accumulate_event_accepts_json_bufs() +_ACCUMULATE_EVENT_PARAMETERS = _accumulate_event_parameters(accumulate_event) +_BETA_ACCUMULATE_EVENT_PARAMETERS = _accumulate_event_parameters(accumulate_beta_event) # Anthropic model parameters that we want to track as span metadata. @@ -720,6 +724,8 @@ def __init__(self, msg_stream, span, request_start_time: float): self.__msg_stream = msg_stream # Per-stream partial JSON buffers for anthropic>=1.5.0 accumulate_event. self.__json_bufs: dict[int, bytes] = {} + self.__accumulate_event = accumulate_event + self.__accumulate_parameters = _ACCUMULATE_EVENT_PARAMETERS self.__span = span self.__metrics = {} self.__snapshot = None @@ -778,8 +784,18 @@ def __process_message(self, m): if self.__time_to_first_token is None: self.__time_to_first_token = time.time() - self.__request_start_time - accumulate_kwargs = {"json_bufs": self.__json_bufs} if _ACCUMULATE_EVENT_ACCEPTS_JSON_BUFS else {} - self.__snapshot = accumulate_event(event=m, current_snapshot=self.__snapshot, **accumulate_kwargs) + if BetaRawMessageStartEvent is not None and isinstance(m, BetaRawMessageStartEvent): + # The regular accumulator does not recognize beta MCP input deltas. + self.__accumulate_event = accumulate_beta_event + self.__accumulate_parameters = _BETA_ACCUMULATE_EVENT_PARAMETERS + + accumulate_kwargs = {} + if "json_bufs" in self.__accumulate_parameters: + accumulate_kwargs["json_bufs"] = self.__json_bufs + if "request_headers" in self.__accumulate_parameters: + request = getattr(self.__msg_stream, "request", None) + accumulate_kwargs["request_headers"] = getattr(request, "headers", None) + self.__snapshot = self.__accumulate_event(event=m, current_snapshot=self.__snapshot, **accumulate_kwargs) if m.type == "message_delta": # Anthropic <0.122.0 drops output_tokens_details when accumulating @@ -1336,7 +1352,7 @@ def _start_span(name, kwargs): return start_span(name=name, type="llm", metadata=metadata, input=_input) -_SERVER_TOOL_USE_TYPE = "server_tool_use" +_SERVER_TOOL_USE_TYPES = ("server_tool_use", "mcp_tool_use") def _is_server_tool_result_type(item_type: Any) -> bool: @@ -1469,7 +1485,7 @@ def _log_server_tool_spans(content: Any, parent_span) -> None: continue item_type = item.get("type") - if item_type == _SERVER_TOOL_USE_TYPE: + if item_type in _SERVER_TOOL_USE_TYPES: call_id = item.get("id") if isinstance(call_id, str): calls_by_id[call_id] = item From 1309d02bcd1b35a8c303fe1f1fc29486ba15c888 Mon Sep 17 00:00:00 2001 From: Kexin Chen <22360275@zju.edu.cn> Date: Mon, 5 Oct 2026 00:46:57 +0800 Subject: [PATCH 2/3] test(anthropic): use recorded MCP response for pairing regression --- .../integrations/anthropic/test_anthropic.py | 82 ++++--------------- 1 file changed, 18 insertions(+), 64 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 314182693..704ca8c13 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -355,7 +355,9 @@ def test_anthropic_beta_messages_create_captures_compaction_metadata(memory_logg ], ids=["sync", "async"], ) -async def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory_logger, is_async, vcr_cassette_name): +async def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks( + memory_logger, is_async, vcr_cassette, vcr_cassette_name +): if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": pytest.skip("Inline MCP tool definitions require the latest Anthropic API") @@ -410,79 +412,31 @@ async def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks(memory assert tool_result["tool_use_id"] == response_tool_result.tool_use_id assert tool_result["content"][0]["text"] == response_tool_result.content[0].text + # Derive the child-span expectations from the recorded provider response, + # independently of the tracing output. + recorded_content = json.loads(vcr_cassette.responses[0]["body"]["string"])["content"] + recorded_call = next(block for block in recorded_content if block["type"] == "mcp_tool_use") + recorded_result = next( + block + for block in recorded_content + if block["type"] == "mcp_tool_result" and block["tool_use_id"] == recorded_call["id"] + ) tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) assert len(tool_spans) == 1 tool_span = tool_spans[0] - assert tool_span["span_attributes"]["name"] == response_tool_use.name - assert tool_span["input"] == response_tool_use.input + assert tool_span["span_attributes"]["name"] == recorded_call["name"] + assert tool_span["input"] == recorded_call["input"] assert tool_span["output"] == [block.model_dump() for block in response_tool_result.content] + assert tool_span["output"][0]["text"] == recorded_result["content"][0]["text"] assert tool_span["metadata"] == { - "tool_use_id": response_tool_use.id, - "tool_call_type": "mcp_tool_use", - "tool_result_type": "mcp_tool_result", + "tool_use_id": recorded_call["id"], + "tool_call_type": recorded_call["type"], + "tool_result_type": recorded_result["type"], } assert tool_span["span_parents"] == [span["span_id"]] assert tool_span["root_span_id"] == span["root_span_id"] -@pytest.mark.parametrize("results_first", [False, True], ids=["calls-first", "results-first"]) -def test_anthropic_mcp_tool_spans_pair_by_id(memory_logger, results_first): - # Supplement the recorded provider response with deterministic ordering and - # incomplete-pair cases that cannot be requested reliably from a live model. - calls = [ - {"type": "mcp_tool_use", "id": "mcp-1", "name": "lookup", "input": {"key": "first"}, "server_name": "catalog"}, - { - "type": "mcp_tool_use", - "id": "mcp-2", - "name": "lookup", - "input": {"key": "second"}, - "server_name": "catalog", - }, - {"type": "mcp_tool_use", "id": "mcp-pending", "name": "pending", "input": {}, "server_name": "catalog"}, - ] - results = [ - {"type": "mcp_tool_result", "tool_use_id": "mcp-2", "content": [{"type": "text", "text": "second value"}]}, - {"type": "mcp_tool_result", "tool_use_id": "mcp-1", "content": [{"type": "text", "text": "first value"}]}, - { - "type": "mcp_tool_result", - "tool_use_id": "mcp-orphan", - "content": [{"type": "text", "text": "orphan value"}], - }, - ] - content = (results + calls if results_first else calls + results) + [ - {"type": "tool_use", "id": "client-tool", "name": "local_lookup", "input": {}}, - {"type": "tool_result", "tool_use_id": "client-tool", "content": "client result"}, - ] - before = json.dumps(content) - with logger.start_span(name="MCP pairs", type="llm") as parent: - _log_message_to_span(SimpleNamespace(content=content), parent) - - spans = memory_logger.pop() - llm_span = find_span_by_name(spans, "MCP pairs") - tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) - assert len(tool_spans) == 4 - by_id = {span["metadata"]["tool_use_id"]: span for span in tool_spans} - assert set(by_id) == {"mcp-1", "mcp-2", "mcp-pending", "mcp-orphan"} - for call, result in zip(calls[:2], reversed(results[:2])): - span = by_id[call["id"]] - assert span["span_attributes"]["name"] == call["name"] - assert span["input"] == call["input"] - assert span["output"] == result["content"] - assert span["metadata"]["tool_call_type"] == "mcp_tool_use" - assert span["metadata"]["tool_result_type"] == "mcp_tool_result" - assert by_id["mcp-pending"]["span_attributes"]["name"] == "pending" - assert by_id["mcp-pending"]["input"] == {} - assert by_id["mcp-pending"].get("output") is None - assert by_id["mcp-orphan"]["span_attributes"]["name"] == "mcp" - assert by_id["mcp-orphan"].get("input") is None - assert by_id["mcp-orphan"]["output"] == results[2]["content"] - for span in tool_spans: - assert span["span_parents"] == [llm_span["span_id"]] - assert span["root_span_id"] == llm_span["root_span_id"] - assert llm_span["output"]["content"] == content - assert json.dumps(content) == before - - @pytest.mark.asyncio @pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) async def test_anthropic_mcp_stream_accumulates_tool_input(memory_logger, is_async): From bed164c9f328113c744fd6ea26502b81429587a5 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 10:49:53 -0400 Subject: [PATCH 3/3] test(anthropic): cover MCP streaming with VCR --- ...pic_mcp_stream_accumulates_tool_input.yaml | 1098 +++++++++++++++++ .../integrations/anthropic/test_anthropic.py | 132 +- 2 files changed, 1168 insertions(+), 62 deletions(-) create mode 100644 py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_mcp_stream_accumulates_tool_input.yaml diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_mcp_stream_accumulates_tool_input.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_mcp_stream_accumulates_tool_input.yaml new file mode 100644 index 000000000..9bcdfa61e --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_mcp_stream_accumulates_tool_input.yaml @@ -0,0 +1,1098 @@ +interactions: +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"Use ask_wiki_question + for repository braintrustdata/braintrust-sdk-python. Ask this exact question: + Give a detailed explanation of the SDK''s primary implementation language, the + package and source directory where it lives, and how a contributor would run + its focused tests. Include enough detail to fully answer each part, and then + briefly summarize the result."}],"model":"claude-haiku-4-5-20251001","mcp_servers":[{"type":"url","name":"braintrust-test","url":"https://mcp.deepwiki.com/mcp"}],"tools":[{"type":"mcp_toolset","mcp_server_name":"braintrust-test"}],"stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-beta: + - mcp-client-2026-09-15 + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '634' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 1.8.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-helper-method: + - stream + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.8.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-stream-helper: + - beta.messages + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages?beta=true + response: + body: + string: 'event: message_start + + data: {"type":"message_start","message":{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjGog73Qnh8HtBuW9oKn","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":918,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":30,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null} } + + + event: content_block_start + + data: {"type":"content_block_start","index":0,"content_block":{"type":"mcp_tool_listing","mcp_server_name":"braintrust-test","tools":[{"name":"ask_wiki_question","description":"Ask + any question about a GitHub repository''s codebase and get an AI-powered answer\ngrounded + in its DeepWiki.","input_schema":{"properties":{"repoName":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"}],"description":"GitHub + repository or list of repositories (max 10) in owner/repo format."},"question":{"description":"The + question to ask about the repository.","type":"string"}},"required":["repoName","question"],"type":"object"}},{"name":"read_wiki_contents","description":"View + documentation about a GitHub repository.","input_schema":{"properties":{"repoName":{"description":"GitHub + repository in owner/repo format (e.g. \"facebook/react\").","type":"string"}},"required":["repoName"],"type":"object"}},{"name":"read_wiki_structure","description":"Get + a list of documentation topics for a GitHub repository.","input_schema":{"properties":{"repoName":{"description":"GitHub + repository in owner/repo format (e.g. \"facebook/react\").","type":"string"}},"required":["repoName"],"type":"object"}}]} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":0 } + + + event: content_block_start + + data: {"type":"content_block_start","index":1,"content_block":{"type":"mcp_tool_use","id":"mcptoolu_01RUkrJ2vptxWjW6MkXdfWnn","name":"ask_wiki_question","input":{},"server_name":"braintrust-test"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"repoN"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ame\": + \""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"braintr"} + } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ustdata"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"/braint"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"rust-sd"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"k-pytho"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"n\""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":", + \"ques"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"tion\": + "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"Give + a"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + detail"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ed + expl"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"anation"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + of the"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + SDK''s "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"primary"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + implem"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"entatio"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"n + langu"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"age, + th"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"e + packa"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ge + and "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"source + "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"directo"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ry + wher"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"e + it li"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ves, + an"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"d + how a"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + contri"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"butor + w"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ould + ru"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"n + its f"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ocused + "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"tests. + "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"Include"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + enough"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + detail"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + to ful"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ly + answ"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"er + each"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" + part, "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"and + the"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"n + brief"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ly + summ"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"arize + t"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"he + resu"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"lt.\"}"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":1 } + + + event: content_block_start + + data: {"type":"content_block_start","index":2,"content_block":{"type":"mcp_tool_result","tool_use_id":"mcptoolu_01RUkrJ2vptxWjW6MkXdfWnn","is_error":false,"content":[{"type":"text","text":"The + Braintrust SDK''s primary implementation language is Python. The Python package + is located in the `py/` directory, with its source code residing in `py/src/braintrust/` . + Contributors can run focused tests using `nox` sessions, which are defined + in `py/noxfile.py` .\n\n## Implementation Language and Directory Structure\n\nThe + Braintrust SDK is implemented in Python . The main Python package is found + within the `py/` directory of the repository . The importable source code + for the `braintrust` package is specifically located at `py/src/braintrust/` + .\n\n## Running Focused Tests\n\nTests for the SDK are managed using `nox` + . The `noxfile.py` in the `py/` directory defines various test sessions .\n\nTo + run all `nox` sessions, you can navigate to the `py/` directory and execute + `make test` . This is equivalent to running `nox -x` .\n\nTo run a specific + test session, you can use the `nox -s ` command . For example:\n* `nox + -s test_core` runs all tests that do not require optional third-party dependencies .\n* `nox + -s \"test_openai(latest)\"` runs tests specifically for the OpenAI wrapper + using the latest version .\n* `nox -s \"test_anthropic(latest)\"` runs tests + for the Anthropic wrapper using the latest version .\n\nYou can list all available + `nox` sessions by running `nox -l` .\n\nMany integration tests utilize `pytest-vcr` + to replay recorded HTTP responses . To disable VCR and make live HTTP calls, + you can append `--disable-vcr` to your `nox` command, for example: `nox -s + \"test_openai(latest)\" -- --disable-vcr` .\n\n## Summary\n\nThe Braintrust + SDK is a Python-based library with its source code located in `py/src/braintrust/` + . Contributors can run focused tests using `nox` sessions defined in `py/noxfile.py` + . Specific sessions can be targeted with `nox -s `, and VCR + can be disabled for live API calls using `--disable-vcr` .\n\n## Notes\n\nThe + `Makefile` in the root directory and the `py/Makefile` provide convenient + `make` targets that wrap `nox` commands for common testing and development + workflows. For instance, `make test` from the `py/` directory executes `nox + -x` .\n\nWiki pages you might want to explore:\n- [Overview (braintrustdata/braintrust-sdk-python)](/wiki/braintrustdata/braintrust-sdk-python#1)\n- + [Development Guide (braintrustdata/braintrust-sdk-python)](/wiki/braintrustdata/braintrust-sdk-python#9)\n\nView + this search on DeepWiki: https://deepwiki.com/search/give-a-detailed-explanation-of_e7d70861-6c6b-4f8c-89dd-9955ed2f82a2\n"}]} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":2 } + + + event: content_block_start + + data: {"type":"content_block_start","index":3,"content_block":{"type":"text","text":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"##"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + Summary\n\nThe"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + Braintrust SDK"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + for"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + Python is implemented entirely"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + in **"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"Python**"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":". + The package"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + source"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + code is located in"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + the `py/"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"src/braint"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"rust/` + directory"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + within"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + the repository. "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n\nContributors + can run"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + focused tests using the"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + **"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"nox`**"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + testing framework, which"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + is configured"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + in `"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"py/nox"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"file.py`."} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + They"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + have"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + several"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + options:"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n\n1"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":". + **Run all"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + tests**: Navigate"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + to the `py"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"/` + directory and"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + execute `make test"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + ("} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"equivalent + to `n"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"ox + -x`)"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n2. + **"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"Run + specific test sessions"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"**: + Use `n"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"ox + -s "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"` + commands"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":","} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + such"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + as:"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n - + `"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"nox + -s"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + test_core`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + for"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + core"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + tests"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + without"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + optional dependencies\n "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"- + `nox"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + -s \"test"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"_openai("} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"latest)\"` + for"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + OpenAI wrapper"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + tests"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n - + `n"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"ox + -s \""} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"test_anthropic"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"(latest)\"`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + for Anthropic"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + wrapper tests"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n3. + **List"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + available sessions**: Run"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + `nox -"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"l`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"\n4. + **"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"Make"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + live API"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + calls**: Append"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + `--disable-"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"vcr` + to"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + bypass"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + recorded"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + HTTP"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + responses ("} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"the"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + tests"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + use `"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"pytest-vcr"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + by"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + default)\n\nThe"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + repository"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + also"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + provides"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + convenient"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + `"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"Makefile`"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + targets in"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + both"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + the root and"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + `py/`"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + directories that wrap `"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"nox` + commands"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + for stream"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":"lined + development"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":3,"delta":{"type":"text_delta","text":" + workflows."} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":3 } + + + event: message_delta + + data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":2711,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":427,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0}} + } + + + event: message_stop + + data: {"type":"message_stop" } + + + ' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T14:47:39Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T14:47:39Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T14:47:39Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T14:47:39Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a45d38fdabba39f7-YYZ + connection: + - keep-alive + content-length: + - '26368' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - text/event-stream; charset=utf-8 + date: + - Mon, 05 Oct 2026 14:47:39 GMT + request-id: + - req_011CfjGofCiEvLDpW2xDmmcx + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-4ea0f09d06f3a84e8e9ecc60dc0caebd-b63c0fea707c374f-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 704ca8c13..802792680 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -21,7 +21,7 @@ from braintrust.integrations.test_utils import verify_autoinstrument_script from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import find_span_by_name, find_spans_by_type, init_test_logger -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel PROJECT_NAME = "test-anthropic-app" @@ -437,78 +437,86 @@ async def test_anthropic_beta_messages_create_preserves_inline_mcp_blocks( assert tool_span["root_span_id"] == span["root_span_id"] +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) @pytest.mark.asyncio -@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) -async def test_anthropic_mcp_stream_accumulates_tool_input(memory_logger, is_async): +@pytest.mark.parametrize( + "is_async,vcr_cassette_name", + [ + (False, "test_anthropic_mcp_stream_accumulates_tool_input"), + (True, "test_anthropic_mcp_stream_accumulates_tool_input"), + ], + ids=["sync", "async"], +) +async def test_anthropic_mcp_stream_accumulates_tool_input(memory_logger, is_async, vcr_cassette, vcr_cassette_name): if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": pytest.skip("MCP stream events require the latest Anthropic SDK") - # Supplemental typed-event coverage: no MCP SSE cassette is available. - # Split input across deltas to exercise the real SDK stream accumulator. - call = { - "type": "mcp_tool_use", - "id": "mcp-1", - "name": "lookup", - "input": {"key": "record"}, - "server_name": "catalog", - } - result = { - "type": "mcp_tool_result", - "tool_use_id": "mcp-1", - "is_error": False, - "content": [{"type": "text", "text": "found"}], + client = wrap_anthropic(_get_async_client() if is_async else _get_client()) + params = { + "model": LATEST_MODEL, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": ( + "Use ask_wiki_question for repository braintrustdata/braintrust-sdk-python. " + "Ask this exact question: Give a detailed explanation of the SDK's primary " + "implementation language, the package and source directory where it lives, " + "and how a contributor would run its focused tests. Include enough detail " + "to fully answer each part, and then briefly summarize the result." + ), + } + ], + "mcp_servers": [ + { + "type": "url", + "name": "braintrust-test", + "url": "https://mcp.deepwiki.com/mcp", + } + ], + "tools": [{"type": "mcp_toolset", "mcp_server_name": "braintrust-test"}], + "betas": ["mcp-client-2026-09-15"], } - wire_events = [ - { - "type": "message_start", - "message": { - "id": "msg_mcp", - "type": "message", - "role": "assistant", - "model": LATEST_MODEL, - "content": [], - "usage": {"input_tokens": 1, "output_tokens": 1}, - }, - }, - {"type": "content_block_start", "index": 0, "content_block": dict(call, input={})}, - {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"key":'}}, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": ' "record"}'}, - }, - {"type": "content_block_stop", "index": 0}, - {"type": "content_block_start", "index": 1, "content_block": result}, - {"type": "content_block_stop", "index": 1}, - {"type": "message_stop"}, - ] - adapter = TypeAdapter(anthropic.types.beta.BetaRawMessageStreamEvent) - events = [adapter.validate_python(event) for event in wire_events] - before = [event.model_dump() for event in events] - - async def async_events(): - for event in events: - yield event - - with logger.start_span(name="MCP stream", type="llm") as parent: - source = async_events() if is_async else iter(events) - stream = TracedMessageStream(source, parent, time.time()) - observed = [event async for event in stream] if is_async else list(stream) - stream._log_final_message() - assert observed == events - assert [event.model_dump() for event in events] == before + + if is_async: + async with client.beta.messages.stream(**params) as stream: + events = [event async for event in stream] + message = await stream.get_final_message() + else: + with client.beta.messages.stream(**params) as stream: + events = list(stream) + message = stream.get_final_message() + + # Confirm the recording contains split input deltas; that is the provider + # behavior the beta accumulator must reassemble. + body = vcr_cassette.responses[0]["body"]["string"] + if isinstance(body, bytes): + body = body.decode() + input_json_deltas = [] + for line in body.splitlines(): + if not line.startswith("data: "): + continue + event = json.loads(line[6:]) + if event.get("type") == "content_block_delta" and event.get("delta", {}).get("type") == "input_json_delta": + input_json_deltas.append(event) + assert len(input_json_deltas) > 1 + + call = next(block for block in message.content if block.type == "mcp_tool_use") + result = next(block for block in message.content if block.type == "mcp_tool_result") + assert any(event.type == "content_block_delta" for event in events) spans = memory_logger.pop() - parent = find_span_by_name(spans, "MCP stream") + parent = find_span_by_name(spans, "anthropic.messages.stream") tool_spans = find_spans_by_type(spans, SpanTypeAttribute.TOOL) assert len(tool_spans) == 1 child = tool_spans[0] - assert child["span_attributes"]["name"] == call["name"] - assert child["input"] == call["input"] - assert child["output"][0]["text"] == "found" - assert child["metadata"]["tool_use_id"] == call["id"] + assert child["span_attributes"]["name"] == call.name + assert child["input"] == call.input + assert child["output"] == [block.model_dump() for block in result.content] + assert child["metadata"]["tool_use_id"] == call.id assert child["metadata"]["tool_call_type"] == "mcp_tool_use" assert child["span_parents"] == [parent["span_id"]] - assert parent["output"]["content"][0]["input"] == call["input"] + parent_call = next(block for block in parent["output"]["content"] if block["type"] == "mcp_tool_use") + assert parent_call["input"] == call.input @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"])