From 0a9cd9e5118fb94423d8a27f32682bcf1c35e32d Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 16:04:24 -0400 Subject: [PATCH] feat(claude-agent-sdk): record verbatim prompt mode --- .../test_query_helper_verbatim_prompts.json | 625 ++++++++++++++++++ .../claude_agent_sdk/test_claude_agent_sdk.py | 21 +- .../integrations/claude_agent_sdk/tracing.py | 10 + 3 files changed, 650 insertions(+), 6 deletions(-) create mode 100644 py/src/braintrust/integrations/claude_agent_sdk/cassettes/latest/test_query_helper_verbatim_prompts.json diff --git a/py/src/braintrust/integrations/claude_agent_sdk/cassettes/latest/test_query_helper_verbatim_prompts.json b/py/src/braintrust/integrations/claude_agent_sdk/cassettes/latest/test_query_helper_verbatim_prompts.json new file mode 100644 index 000000000..4bc38e8d2 --- /dev/null +++ b/py/src/braintrust/integrations/claude_agent_sdk/cassettes/latest/test_query_helper_verbatim_prompts.json @@ -0,0 +1,625 @@ +{ + "cassette_name": "test_query_helper_verbatim_prompts", + "events": [ + { + "op": "write", + "payload": { + "kind": "json", + "value": { + "request": { + "hooks": { + "UserPromptSubmit": [ + { + "hookCallbackIds": [ + "hook_0" + ], + "matcher": null + } + ] + }, + "subtype": "initialize" + }, + "request_id": "req_1_e4d8c8eb", + "type": "control_request" + } + } + }, + { + "op": "read", + "payload": { + "hook_event": "SessionStart", + "hook_id": "15ddf4f7-0fe9-4ea9-8df7-5aed8549b4b8", + "hook_name": "SessionStart:startup", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "subtype": "hook_started", + "type": "system", + "uuid": "002a16c6-49c3-4576-9f1b-ed221ba7ae47" + } + }, + { + "op": "read", + "payload": { + "hook_event": "SessionStart", + "hook_id": "d7499c16-e002-4688-b6a4-3aeb2407bda0", + "hook_name": "SessionStart:startup", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "subtype": "hook_started", + "type": "system", + "uuid": "d6a04569-7532-41a8-9382-b9279f9bceb9" + } + }, + { + "op": "read", + "payload": { + "exit_code": 0, + "hook_event": "SessionStart", + "hook_id": "15ddf4f7-0fe9-4ea9-8df7-5aed8549b4b8", + "hook_name": "SessionStart:startup", + "outcome": "success", + "output": "", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "stderr": "", + "stdout": "", + "subtype": "hook_response", + "type": "system", + "uuid": "1ddfcd84-77a9-46b6-9191-b361555a6468" + } + }, + { + "op": "read", + "payload": { + "exit_code": 0, + "hook_event": "SessionStart", + "hook_id": "d7499c16-e002-4688-b6a4-3aeb2407bda0", + "hook_name": "SessionStart:startup", + "outcome": "success", + "output": "", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "stderr": "", + "stdout": "", + "subtype": "hook_response", + "type": "system", + "uuid": "e0ea3471-551a-4ad8-9a58-8d24dd4d74dd" + } + }, + { + "op": "read", + "payload": { + "response": { + "pending_permission_requests": [], + "pending_user_dialog_requests": [], + "request_id": "req_1_e4d8c8eb", + "response": { + "account": { + "apiKeySource": "ANTHROPIC_API_KEY", + "apiProvider": "firstParty", + "tokenSource": "claude.ai" + }, + "agents": [], + "available_output_styles": [], + "commands": [], + "models": [] + }, + "subtype": "success" + }, + "type": "control_response" + } + }, + { + "op": "write", + "payload": { + "kind": "json", + "value": { + "client_composed": true, + "message": { + "content": "Say hello in one short sentence.", + "role": "user" + }, + "parent_tool_use_id": null, + "session_id": "", + "type": "user" + } + } + }, + { + "op": "read", + "payload": { + "request": { + "callback_id": "hook_0", + "input": { + "cwd": "", + "hook_event_name": "UserPromptSubmit", + "permission_mode": "bypassPermissions", + "prompt": "Say hello in one short sentence.", + "prompt_id": "1aa84217-82f2-489a-b452-5c6930156004", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "transcript_path": "" + }, + "subtype": "hook_callback", + "tool_use_id": "9348a6d6-ba5d-4482-8610-34364aaccfbf" + }, + "request_id": "30e74d73-9dd5-4015-9d52-df1c28c70c2c", + "type": "control_request" + } + }, + { + "op": "write", + "payload": { + "kind": "json", + "value": { + "response": { + "request_id": "30e74d73-9dd5-4015-9d52-df1c28c70c2c", + "response": { + "hookSpecificOutput": { + "additionalContext": "Remember the answer should stay concise.", + "hookEventName": "UserPromptSubmit" + } + }, + "subtype": "success" + }, + "type": "control_response" + } + } + }, + { + "op": "read", + "payload": { + "agents": [ + "claude", + "Explore", + "general-purpose", + "Plan", + "statusline-setup" + ], + "analytics_disabled": false, + "apiKeySource": "ANTHROPIC_API_KEY", + "capabilities": [ + "interrupt_receipt_v1", + "interrupt_cancel_queued_v1", + "msg_lifecycle_v1", + "mcp_read_resource_v1", + "mcp_tool_ui_meta_v1" + ], + "claude_code_version": "2.1.283", + "cwd": "", + "fast_mode_disabled_reason": "sdk_opt_in_required", + "fast_mode_state": "off", + "mcp_servers": [ + { + "name": "linear-server", + "source": "local", + "status": "pending" + } + ], + "memory_paths": { + "auto": "" + }, + "messaging_socket_path": "", + "model": "claude-haiku-4-5-20251001", + "output_style": "default", + "per_turn_effort_active": false, + "permissionMode": "bypassPermissions", + "plugins": [ + { + "name": "rust-analyzer-lsp", + "path": "", + "source": "rust-analyzer-lsp@claude-plugins-official", + "version": "1.0.0" + }, + { + "name": "trace-claude-code", + "path": "", + "source": "trace-claude-code@braintrust-claude-plugin", + "version": "3.0.1" + }, + { + "name": "superset", + "path": "", + "source": "superset@skills-dir", + "version": "0.3.0" + }, + { + "name": "agents-md", + "path": "builtin", + "source": "agents-md@builtin" + }, + { + "name": "telemetry", + "path": "builtin", + "source": "telemetry@builtin" + } + ], + "product_feedback_disabled": false, + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "skills": [ + "braintrust", + "create-braintrust-demo", + "commit-message", + "sdk-benchmarking", + "sdk-ci-triage", + "sdk-dependency-updates", + "sdk-integrations", + "sdk-vcr-workflows", + "superset:10x", + "superset:automate", + "superset:browser", + "superset:computer", + "superset:contribute", + "superset:doctor", + "superset:feedback", + "superset:integrations", + "superset:orchestrate", + "superset:page", + "superset:plugins", + "superset:setup", + "superset:standup", + "design", + "design-sync", + "dataviz", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "doctor", + "loop", + "claude-api", + "run", + "run-skill-generator", + "anthropic-skills:braintrust-graph-styles", + "anthropic-skills:customer-usage-enterprise-dashie", + "anthropic-skills:docx", + "anthropic-skills:ghost-note", + "anthropic-skills:import-memory", + "anthropic-skills:morning", + "anthropic-skills:opportunity-deal-review", + "anthropic-skills:pdf", + "anthropic-skills:pipeline-metric-definitions", + "anthropic-skills:pptx", + "anthropic-skills:setup", + "anthropic-skills:skill-creator", + "anthropic-skills:xlsx" + ], + "slash_commands": [ + "braintrust", + "create-braintrust-demo", + "commit-message", + "sdk-benchmarking", + "sdk-ci-triage", + "sdk-dependency-updates", + "sdk-integrations", + "sdk-vcr-workflows", + "superset:superset-automation", + "superset:10x", + "superset:automate", + "superset:browser", + "superset:computer", + "superset:contribute", + "superset:doctor", + "superset:feedback", + "superset:integrations", + "superset:orchestrate", + "superset:page", + "superset:plugins", + "superset:setup", + "superset:standup", + "design", + "design-sync", + "dataviz", + "update-config", + "verify", + "debug", + "code-review", + "simplify", + "batch", + "fewer-permission-prompts", + "doctor", + "loop", + "claude-api", + "run", + "run-skill-generator", + "advisor", + "agents", + "auto-mode-setup", + "autocompact", + "clear", + "color", + "compact", + "config", + "output-style", + "context", + "effort", + "fast", + "focus", + "heapdump", + "init", + "mcp", + "import", + "model", + "__remote-workflow", + "workflow-launch-exec", + "reload-plugins", + "reload-skills", + "rename", + "security-review", + "usage", + "insights", + "recap", + "skill-doctor", + "goal", + "design-consent", + "design-revoke", + "list-agents", + "team-onboarding", + "anthropic-skills:braintrust-graph-styles", + "anthropic-skills:customer-usage-enterprise-dashie", + "anthropic-skills:discovery-call-prep", + "anthropic-skills:docx", + "anthropic-skills:ghost-note", + "anthropic-skills:import-memory", + "anthropic-skills:meeting-to-linear", + "anthropic-skills:morning", + "anthropic-skills:opportunity-deal-review", + "anthropic-skills:pdf", + "anthropic-skills:pipeline-metric-definitions", + "anthropic-skills:pipeline-review", + "anthropic-skills:pptx", + "anthropic-skills:setup", + "anthropic-skills:skill-creator", + "anthropic-skills:weekly-update", + "anthropic-skills:xlsx" + ], + "subtype": "init", + "terminal_slash_commands": [ + "doctor", + "color", + "focus", + "reload-plugins" + ], + "tools": [ + "Task", + "Bash", + "CronCreate", + "CronDelete", + "CronList", + "DesignSync", + "Edit", + "EnterWorktree", + "ExitWorktree", + "ListAgents", + "LSP", + "Monitor", + "NotebookEdit", + "PushNotification", + "Read", + "ReportFindings", + "ScheduleWakeup", + "SendMessage", + "Skill", + "TaskCreate", + "TaskGet", + "TaskList", + "TaskStop", + "TaskUpdate", + "ToolSearch", + "WebFetch", + "WebSearch", + "Write" + ], + "type": "system", + "uuid": "1456f20f-b051-4acc-bfa1-53872e3b175a", + "view_mode": "default" + } + }, + { + "op": "read", + "payload": { + "estimated_tokens": 50, + "estimated_tokens_delta": 50, + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "subtype": "thinking_tokens", + "type": "system", + "uuid": "9bf94cd6-32c8-4ec6-a540-c81c2f16b83b" + } + }, + { + "op": "read", + "payload": { + "estimated_tokens": 177, + "estimated_tokens_delta": 127, + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "subtype": "thinking_tokens", + "type": "system", + "uuid": "5a1f3a62-e9e3-49b5-b6db-76216e829c87" + } + }, + { + "op": "read", + "payload": { + "message": { + "container": null, + "content": [ + { + "signature": "", + "thinking": "", + "type": "thinking" + } + ], + "context_management": null, + "diagnostics": null, + "id": "msg_011CfWS41QXwtZ3dLZy2K5cs", + "input_transformations": [], + "model": "claude-haiku-4-5-20251001", + "role": "assistant", + "stop_details": null, + "stop_reason": null, + "stop_sequence": null, + "type": "message", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 8598 + }, + "cache_creation_input_tokens": 8598, + "cache_read_input_tokens": 9925, + "inference_geo": "not_available", + "input_tokens": 10, + "output_tokens": 6, + "service_tier": "standard" + } + }, + "parent_tool_use_id": null, + "request_id": "req_011CfWS415wivHGXAfWWHPWv", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "timestamp": "2026-09-28T20:01:15.025Z", + "type": "assistant", + "uuid": "09a86328-fe91-4cd1-bf6f-5359620da28b" + } + }, + { + "op": "read", + "payload": { + "message": { + "container": null, + "content": [ + { + "text": "Hello! I'm ready to help you with the Braintrust SDK Python repository.", + "type": "text" + } + ], + "context_management": null, + "diagnostics": null, + "id": "msg_011CfWS41QXwtZ3dLZy2K5cs", + "input_transformations": [], + "model": "claude-haiku-4-5-20251001", + "role": "assistant", + "stop_details": null, + "stop_reason": null, + "stop_sequence": null, + "type": "message", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 8598 + }, + "cache_creation_input_tokens": 8598, + "cache_read_input_tokens": 9925, + "inference_geo": "not_available", + "input_tokens": 10, + "output_tokens": 6, + "service_tier": "standard" + } + }, + "parent_tool_use_id": null, + "request_id": "req_011CfWS415wivHGXAfWWHPWv", + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "timestamp": "2026-09-28T20:01:15.180Z", + "type": "assistant", + "uuid": "2d0ca34a-f082-4181-a2e1-6ca9a81280ef" + } + }, + { + "op": "read", + "payload": { + "api_error_status": null, + "duration_api_ms": 6146, + "duration_ms": 5145, + "fast_mode_disabled_reason": "sdk_opt_in_required", + "fast_mode_state": "off", + "first_content_frame_ms": 3927, + "is_error": false, + "modelUsage": { + "claude-haiku-4-5-20251001": { + "cacheCreationInputTokens": 8598, + "cacheReadInputTokens": 9925, + "canonicalModel": "claude-haiku-4-5", + "contextWindow": 200000, + "costBasis": "list", + "costUSD": 0.013339, + "inputTokens": 909, + "maxOutputTokens": 32000, + "outputTokens": 138, + "provider": "firstParty", + "thinkingTokens": 104, + "webSearchRequests": 0 + } + }, + "num_turns": 1, + "permission_denials": [], + "queued_turn_count": 0, + "result": "Hello! I'm ready to help you with the Braintrust SDK Python repository.", + "result_index": 0, + "session_id": "af9c70f6-57ce-4f75-953f-065e51c58571", + "stop_reason": "end_turn", + "subagent_stats": { + "by_type": {}, + "completed": 0, + "failed": 0, + "killed": { + "parent": 0, + "system": 0, + "user": 0 + }, + "max_depth": 0, + "refused": { + "budget": 0, + "concurrency_limit": 0, + "depth_limit": 0 + }, + "requested": { + "background": 0, + "foreground": 0, + "unset": 0 + }, + "spawned": 0, + "spawned_by_subagents": 0, + "started_in_background": 0 + }, + "subtype": "success", + "terminal_reason": "completed", + "time_to_request_ms": 185, + "total_cost_usd": 0.013339, + "ttft_ms": 4928, + "ttft_stream_ms": 3927, + "type": "result", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 8598 + }, + "cache_creation_input_tokens": 8598, + "cache_read_input_tokens": 9925, + "inference_geo": "not_available", + "input_tokens": 10, + "iterations": [ + { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 8598 + }, + "cache_creation_input_tokens": 8598, + "cache_read_input_tokens": 9925, + "input_tokens": 10, + "output_tokens": 128, + "type": "message" + } + ], + "output_tokens": 128, + "output_tokens_details": { + "thinking_tokens": 104 + }, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + }, + "uuid": "877c0ce7-e6b7-44d3-bdc1-6c2a0d9ff712" + } + } + ], + "sdk_version": "0.2.160" +} diff --git a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py index 2be9e56c0..c996292be 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py @@ -419,15 +419,22 @@ async def test_query_helper_keeps_options_untouched_and_logs_aggregate_task_usag pytest.skip("The 0.1.10 query() transport lifecycle is incompatible with the client cassette") assert not memory_logger.pop() prompt = "Say hello in one short sentence." + supports_verbatim_prompts = _sdk_version_at_least("0.2.158") hooks = {"UserPromptSubmit": [claude_agent_sdk.HookMatcher(hooks=[_concise_user_prompt_hook])]} - options = claude_agent_sdk.ClaudeAgentOptions( - model=TEST_MODEL, - permission_mode="bypassPermissions", - hooks=hooks, - ) + option_values: dict[str, Any] = { + "model": TEST_MODEL, + "permission_mode": "bypassPermissions", + "hooks": hooks, + } + if supports_verbatim_prompts: + option_values["verbatim_prompts"] = True + options = claude_agent_sdk.ClaudeAgentOptions(**option_values) + cassette_name = "test_user_prompt_submit_hook_creates_function_span" + if supports_verbatim_prompts: + cassette_name = "test_query_helper_verbatim_prompts" transport = make_cassette_transport( - cassette_name="test_user_prompt_submit_hook_creates_function_span", + cassette_name=cassette_name, prompt="", options=options, ) @@ -447,6 +454,8 @@ async def test_query_helper_keeps_options_untouched_and_logs_aggregate_task_usag spans = memory_logger.pop() task_span = find_span_by_name(find_spans_by_type(spans, SpanTypeAttribute.TASK), "Claude Agent") + if supports_verbatim_prompts: + assert task_span["metadata"]["verbatim_prompts"] is True llm_spans = find_spans_by_type(spans, SpanTypeAttribute.LLM) assert len(llm_spans) == 1 assert not { diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index 086088643..7b91eea86 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -1094,11 +1094,14 @@ def __init__( query_start_time: float | None = None, captured_messages: list[dict[str, Any]] | None = None, include_partial_messages: bool = False, + verbatim_prompts: bool = False, ) -> None: + metadata = {"verbatim_prompts": True} if verbatim_prompts else None self._root_span = start_span( name=CLAUDE_AGENT_TASK_SPAN_NAME, span_attributes={"type": SpanTypeAttribute.TASK}, input=prompt or None, + metadata=metadata, start_time=query_start_time, ) self._context_tracker = ContextTracker( @@ -1274,6 +1277,10 @@ def _include_partial_messages(options: Any) -> bool: return getattr(options, "include_partial_messages", False) is True +def _verbatim_prompts(options: Any) -> bool: + return getattr(options, "verbatim_prompts", False) is True + + async def _stream_messages_with_tracing( generator: AsyncIterable[Any], *, @@ -1321,6 +1328,7 @@ async def wrapped_query(*args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]: query_start_time=query_start_time, captured_messages=captured_messages, include_partial_messages=_include_partial_messages(options), + verbatim_prompts=_verbatim_prompts(options), ) generator = _bind_request_tracker_to_query(original_query(*args, **kwargs), request_tracker) @@ -1341,6 +1349,7 @@ class WrappedClaudeSDKClient(Wrapper): def __init__(self, *args: Any, **kwargs: Any): options = args[0] if args else kwargs.get("options") self.__include_partial_messages = _include_partial_messages(options) + self.__verbatim_prompts = _verbatim_prompts(options) client = original_client_class(*args, **kwargs) super().__init__(client) self.__client = client @@ -1402,6 +1411,7 @@ def __start_request_tracker(self) -> RequestTracker: query_start_time=self.__query_start_time, captured_messages=self.__captured_messages, include_partial_messages=self.__include_partial_messages, + verbatim_prompts=self.__verbatim_prompts, ) query = getattr(self.__client, "_query", None) _install_query_message_tracing(query)