diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py index 2b601dc4f..e10a9f558 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openrouter.py @@ -29,4 +29,26 @@ assert span["metadata"]["model"] == "gpt-4o-mini" assert "4" in span["output"][0]["message"]["content"] +with autoinstrument_test_context( + "test_wrap_openrouter_beta_responses_send", integration="openrouter" +) as memory_logger: + client = openrouter.OpenRouter(api_key=os.environ.get("OPENROUTER_API_KEY")) + response = client.beta.responses.send( + model="openai/gpt-4o-mini", + input="Say one short sentence about observability.", + max_output_tokens=64, + temperature=0, + ) + assert response.output + + spans = memory_logger.pop() + assert len(spans) == 1, f"Expected 1 beta Responses span, got {len(spans)}" + span = spans[0] + assert span["span_attributes"]["name"] == "openrouter.beta.responses.send" + assert span["input"] == "Say one short sentence about observability." + assert span["metadata"]["model"] == "gpt-4o-mini" + assert span["metadata"]["provider"] == "openai" + assert span["output"][0]["type"] == "message" + assert span["metrics"]["tokens"] > 0 + print("SUCCESS") diff --git a/py/src/braintrust/integrations/openrouter/cassettes/0.6.0/test_wrap_openrouter_responses_send.yaml b/py/src/braintrust/integrations/openrouter/cassettes/0.6.0/test_wrap_openrouter_beta_responses_send.yaml similarity index 100% rename from py/src/braintrust/integrations/openrouter/cassettes/0.6.0/test_wrap_openrouter_responses_send.yaml rename to py/src/braintrust/integrations/openrouter/cassettes/0.6.0/test_wrap_openrouter_beta_responses_send.yaml diff --git a/py/src/braintrust/integrations/openrouter/cassettes/latest/test_wrap_openrouter_beta_responses_send.yaml b/py/src/braintrust/integrations/openrouter/cassettes/latest/test_wrap_openrouter_beta_responses_send.yaml new file mode 100644 index 000000000..552c0ddcf --- /dev/null +++ b/py/src/braintrust/integrations/openrouter/cassettes/latest/test_wrap_openrouter_beta_responses_send.yaml @@ -0,0 +1,57 @@ +interactions: +- request: + body: '{"input":"Say one short sentence about observability.","max_output_tokens":64,"model":"openai/gpt-4o-mini","service_tier":"auto","store":false,"stream":false,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '176' + content-type: + - application/json + host: + - openrouter.ai + user-agent: + - speakeasy-sdk/python 1.3.22 2.914.0 1.0.0 openrouter + method: POST + uri: https://openrouter.ai/api/v1/responses + response: + body: + string: "\n \n{\"id\":\"gen-1791211520-PtJQA4FwfoqkSl8ZIW4k\",\"object\":\"response\",\"created_at\":1791211520,\"model\":\"openai/gpt-4o-mini\",\"status\":\"completed\",\"completed_at\":1791211521,\"output\":[{\"id\":\"msg_tmp_wbxaa4cv5zq\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Observability + enables organizations to understand and monitor the internal state of complex + systems through external outputs.\",\"annotations\":[],\"logprobs\":[]}]}],\"error\":null,\"incomplete_details\":null,\"tools\":[],\"tool_choice\":\"auto\",\"parallel_tool_calls\":true,\"max_output_tokens\":64,\"temperature\":0,\"top_p\":1,\"presence_penalty\":0,\"frequency_penalty\":0,\"top_logprobs\":0,\"max_tool_calls\":null,\"metadata\":{},\"background\":false,\"previous_response_id\":null,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"store\":false,\"instructions\":null,\"text\":{\"format\":{\"type\":\"text\"}},\"reasoning\":null,\"safety_identifier\":null,\"prompt_cache_key\":null,\"prompt_cache_options\":null,\"usage\":{\"input_tokens\":15,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":18,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":33,\"cost\":0.00001305,\"is_byok\":false,\"cost_details\":{\"upstream_inference_cost\":0.00001305,\"upstream_inference_input_cost\":0.00000225,\"upstream_inference_output_cost\":0.0000108}}}" + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Generation-Id,X-Provider-Name,request-id,cf-ray + cf-ray: + - a45d35a368790c69-YYZ + connection: + - keep-alive + content-length: + - '1303' + content-type: + - application/json + date: + - Mon, 05 Oct 2026 14:45:20 GMT + permissions-policy: + - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com" + "https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com") + referrer-policy: + - no-referrer, strict-origin-when-cross-origin + server: + - cloudflare + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-generation-id: + - gen-1791211520-PtJQA4FwfoqkSl8ZIW4k + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openrouter/integration.py b/py/src/braintrust/integrations/openrouter/integration.py index 2ec64107c..f1d9b0b01 100644 --- a/py/src/braintrust/integrations/openrouter/integration.py +++ b/py/src/braintrust/integrations/openrouter/integration.py @@ -2,7 +2,7 @@ from braintrust.integrations.base import BaseIntegration -from .patchers import ChatPatcher, EmbeddingsPatcher, ResponsesPatcher +from .patchers import BetaResponsesPatcher, ChatPatcher, EmbeddingsPatcher, ResponsesPatcher class OpenRouterIntegration(BaseIntegration): @@ -15,4 +15,5 @@ class OpenRouterIntegration(BaseIntegration): ChatPatcher, EmbeddingsPatcher, ResponsesPatcher, + BetaResponsesPatcher, ) diff --git a/py/src/braintrust/integrations/openrouter/patchers.py b/py/src/braintrust/integrations/openrouter/patchers.py index 73e9ed622..41335aca7 100644 --- a/py/src/braintrust/integrations/openrouter/patchers.py +++ b/py/src/braintrust/integrations/openrouter/patchers.py @@ -1,5 +1,7 @@ """OpenRouter patchers.""" +from functools import partial + from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher from .tracing import ( @@ -51,19 +53,38 @@ class EmbeddingsPatcher(CompositeFunctionWrapperPatcher): class ResponsesSendPatcher(FunctionWrapperPatcher): - name = "openrouter.beta.responses.send" + name = "openrouter.responses.send" target_module = "openrouter.responses" target_path = "Responses.send" wrapper = _responses_send_wrapper class ResponsesSendAsyncPatcher(FunctionWrapperPatcher): - name = "openrouter.beta.responses.send_async" + name = "openrouter.responses.send_async" target_module = "openrouter.responses" target_path = "Responses.send_async" wrapper = _responses_send_async_wrapper class ResponsesPatcher(CompositeFunctionWrapperPatcher): - name = "openrouter.beta.responses" + name = "openrouter.responses" sub_patchers = (ResponsesSendPatcher, ResponsesSendAsyncPatcher) + + +class BetaResponsesSendPatcher(FunctionWrapperPatcher): + name = "openrouter.beta.responses.send" + target_module = "openrouter.beta_responses" + target_path = "BetaResponses.send" + wrapper = partial(_responses_send_wrapper, span_name="openrouter.beta.responses.send") + + +class BetaResponsesSendAsyncPatcher(FunctionWrapperPatcher): + name = "openrouter.beta.responses.send_async" + target_module = "openrouter.beta_responses" + target_path = "BetaResponses.send_async" + wrapper = partial(_responses_send_async_wrapper, span_name="openrouter.beta.responses.send") + + +class BetaResponsesPatcher(CompositeFunctionWrapperPatcher): + name = "openrouter.beta.responses" + sub_patchers = (BetaResponsesSendPatcher, BetaResponsesSendAsyncPatcher) diff --git a/py/src/braintrust/integrations/openrouter/test_openrouter.py b/py/src/braintrust/integrations/openrouter/test_openrouter.py index 54f1f88ea..d3bb6ad61 100644 --- a/py/src/braintrust/integrations/openrouter/test_openrouter.py +++ b/py/src/braintrust/integrations/openrouter/test_openrouter.py @@ -153,7 +153,7 @@ def test_wrap_openrouter_embeddings_generate(memory_logger): @pytest.mark.vcr -def test_wrap_openrouter_responses_send(memory_logger): +def test_wrap_openrouter_beta_responses_send(memory_logger): assert not memory_logger.pop() client = wrap_openrouter(_get_client()) @@ -185,6 +185,34 @@ def test_wrap_openrouter_responses_send(memory_logger): assert_metrics_are_valid(span["metrics"], start, end) +@pytest.mark.vcr +def test_wrap_openrouter_responses_send(memory_logger): + client = _get_client() + if not hasattr(client, "responses"): + pytest.skip("GA Responses is unavailable in this OpenRouter version") + + wrap_openrouter(client) + response = client.responses.send( + model=CHAT_MODEL, + input="Say one short sentence about observability.", + max_output_tokens=64, + temperature=0, + ) + assert response.output + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == "openrouter.responses.send" + assert span["span_attributes"]["type"] == "llm" + assert span["input"] == "Say one short sentence about observability." + assert span["metadata"]["model"] == "gpt-4o-mini" + assert span["metadata"]["provider"] == "openai" + assert span["output"][0]["type"] == "message" + assert span["metrics"]["tokens"] > 0 + assert span["context"]["span_origin"]["instrumentation"]["name"] == "openrouter-auto" + + @pytest.mark.vcr @pytest.mark.asyncio async def test_wrap_openrouter_responses_send_async_stream(memory_logger): diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index 404ea9a76..2d78100b4 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -1,5 +1,6 @@ """OpenRouter-specific tracing helpers.""" +import importlib.util import logging import time from collections.abc import AsyncIterator, Iterator @@ -625,9 +626,18 @@ async def _embeddings_generate_async_wrapper(wrapped, instance, args, kwargs): return result -def _responses_send_wrapper(wrapped, instance, args, kwargs): +def _responses_span_name(span_name: str | None) -> str: + if span_name is not None: + return span_name + # In older releases, openrouter.responses.Responses backs client.beta.responses. + if importlib.util.find_spec("openrouter.beta_responses") is None: + return "openrouter.beta.responses.send" + return "openrouter.responses.send" + + +def _responses_send_wrapper(wrapped, instance, args, kwargs, *, span_name: str | None = None): request_metadata = _build_request_metadata(kwargs, _RESPONSES_REQUEST_KEYS) - span = _start_span("openrouter.beta.responses.send", kwargs.get("input"), request_metadata) + span = _start_span(_responses_span_name(span_name), kwargs.get("input"), request_metadata) start_time = time.time() try: @@ -650,9 +660,9 @@ def _responses_send_wrapper(wrapped, instance, args, kwargs): return result -async def _responses_send_async_wrapper(wrapped, instance, args, kwargs): +async def _responses_send_async_wrapper(wrapped, instance, args, kwargs, *, span_name: str | None = None): request_metadata = _build_request_metadata(kwargs, _RESPONSES_REQUEST_KEYS) - span = _start_span("openrouter.beta.responses.send", kwargs.get("input"), request_metadata) + span = _start_span(_responses_span_name(span_name), kwargs.get("input"), request_metadata) start_time = time.time() try: @@ -677,7 +687,7 @@ async def _responses_send_async_wrapper(wrapped, instance, args, kwargs): def wrap_openrouter(client: Any) -> Any: """Wrap a single OpenRouter client instance for tracing.""" - from .patchers import ChatPatcher, EmbeddingsPatcher, ResponsesPatcher + from .patchers import BetaResponsesPatcher, ChatPatcher, EmbeddingsPatcher, ResponsesPatcher chat = getattr(client, "chat", None) if chat is not None: @@ -687,9 +697,13 @@ def wrap_openrouter(client: Any) -> Any: if embeddings is not None: EmbeddingsPatcher.wrap_target(embeddings) + responses = getattr(client, "responses", None) + if responses is not None: + ResponsesPatcher.wrap_target(responses) + beta = getattr(client, "beta", None) responses = getattr(beta, "responses", None) if beta is not None else None if responses is not None: - ResponsesPatcher.wrap_target(responses) + BetaResponsesPatcher.wrap_target(responses) return client