From d371e6c4ca51332ae14576906afc6ffd90d4c53d Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 15:57:54 -0400 Subject: [PATCH] fix(openai): normalize iterable tools metadata --- ...se_generator_tools_are_logged_as_list.yaml | 105 ++++++++++++++++++ .../integrations/openai/test_openai.py | 33 ++++++ .../braintrust/integrations/openai/tracing.py | 5 +- 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_parse_generator_tools_are_logged_as_list.yaml diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_parse_generator_tools_are_logged_as_list.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_parse_generator_tools_are_logged_as_list.yaml new file mode 100644 index 000000000..7880c0699 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_chat_parse_generator_tools_are_logged_as_list.yaml @@ -0,0 +1,105 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Say 24."}],"model":"gpt-4o-mini","stream":false,"tool_choice":"none","tools":[{"type":"function","function":{"name":"lookup","description":"Look + up a value","strict":true,"parameters":{"type":"object","properties":{},"additionalProperties":false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '289' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.19.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-helper-method: + - chat.completions.parse + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.19.2 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-ETBZIMxddyW4ZTokevpShKzW0M4Sx\",\n \"object\": + \"chat.completion\",\n \"created\": 1790624688,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"24.\",\n \"refusal\": null,\n + \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 40,\n \"completion_tokens\": + 2,\n \"total_tokens\": 42,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_2fed836403\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4253ea89a38abac-YYZ + connection: + - keep-alive + content-length: + - '809' + content-type: + - application/json + date: + - Mon, 28 Sep 2026 19:44:48 GMT + openai-processing-ms: + - '533' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999997' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e90cfbea17b84a2db5b46db54e5412a4 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index d54b48d36..993834b43 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -1581,6 +1581,39 @@ async def test_openai_async_parallel_requests(memory_logger): assert_metrics_are_valid(span["metrics"]) +@pytest.mark.vcr +@pytest.mark.skipif( + Version(openai.__version__) < Version("3.19.1"), reason="iterable tools are supported by OpenAI 3.19.1+" +) +def test_openai_chat_parse_generator_tools_are_logged_as_list(memory_logger): + assert not memory_logger.pop() + + tools = [ + { + "type": "function", + "function": { + "name": "lookup", + "description": "Look up a value", + "strict": True, + "parameters": {"type": "object", "properties": {}, "additionalProperties": False}, + }, + } + ] + client = wrap_openai(openai.OpenAI()) + + response = client.chat.completions.parse( + model=TEST_MODEL, + messages=[{"role": "user", "content": "Say 24."}], + tools=(tool for tool in tools), + tool_choice="none", + ) + + assert response.choices[0].message.content + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["metadata"]["tools"] == tools + + @pytest.mark.vcr def test_openai_not_given_filtering(memory_logger): """Test that NOT_GIVEN values are filtered out of logged inputs but API call still works.""" diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 5b2ed9692..7bdca5d10 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -6,7 +6,7 @@ import inspect import json import time -from collections.abc import Callable +from collections.abc import Callable, Iterable, Mapping from typing import Any from braintrust.integrations.utils import ( @@ -170,6 +170,9 @@ def _filter_metadata(params: dict[str, Any], allowlist: tuple[str, ...]) -> dict value = params[key] if _is_not_given(value): continue + if key == "tools" and not isinstance(value, (list, str, bytes, Mapping)) and isinstance(value, Iterable): + value = list(value) + params[key] = value if key in ("response_format", "text_format"): value = _serialize_response_format(value) metadata[key] = value