From 375482807a80ee7eb625ddbfafa8c3e3b4233276 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 11:29:50 -0400 Subject: [PATCH] fix(litellm): replace retired text completion model Route text completion metrics coverage through Anthropic and refresh both LiteLLM version cassette sets. Fixes #865 --- ...itellm_text_completion_metrics[async].yaml | 153 +++++++---------- ...litellm_text_completion_metrics[sync].yaml | 161 ++++++++---------- ...itellm_text_completion_metrics[async].yaml | 133 ++++++--------- ...litellm_text_completion_metrics[sync].yaml | 123 ++++++------- .../integrations/litellm/test_litellm.py | 4 +- 5 files changed, 239 insertions(+), 335 deletions(-) diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml index 77b55f938..2620d2db8 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml @@ -1,114 +1,87 @@ interactions: - request: - body: '{"model":"gpt-3.5-turbo-instruct","prompt":"What''s 12 + 12?"}' + body: '{"model":"claude-haiku-4-5-20251001","messages":[{"role":"user","content":[{"type":"text","text":"What''s + 12 + 12?"}]}],"max_tokens":4096}' headers: - Accept: + accept: - application/json - Accept-Encoding: + accept-encoding: - gzip, deflate - Connection: + anthropic-version: + - '2023-06-01' + connection: - keep-alive - Content-Length: - - '61' - Content-Type: + content-length: + - '137' + content-type: - application/json - Host: - - api.openai.com - User-Agent: - - AsyncOpenAI/Python 1.99.9 - X-Stainless-Arch: - - arm64 - X-Stainless-Async: - - async:asyncio - X-Stainless-Lang: - - python - X-Stainless-OS: - - MacOS - X-Stainless-Package-Version: - - 1.99.9 - X-Stainless-Raw-Response: - - 'true' - X-Stainless-Runtime: - - CPython - X-Stainless-Runtime-Version: - - 3.12.12 - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' + host: + - api.anthropic.com + user-agent: + - litellm/1.74.0 method: POST - uri: https://api.openai.com/v1/completions + uri: https://api.anthropic.com/v1/messages response: body: - string: "{\n \"id\": \"cmpl-DdJ9HtCsgRTbqBUitaYeCb6F03nAP\",\n \"object\": - \"text_completion\",\n \"created\": 1778260771,\n \"model\": \"gpt-3.5-turbo-instruct:20230824-v2\",\n - \ \"choices\": [\n {\n \"text\": \"\\n12 + 12 = 24\",\n \"index\": - 0,\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n - \ ],\n \"usage\": {\n \"prompt_tokens\": 8,\n \"completion_tokens\": - 8,\n \"total_tokens\": 16\n }\n}\n" + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjKty5VdNab2MB7YKbV5","type":"message","role":"assistant","content":[{"type":"text","text":"12 + + 12 = 24"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":16,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":13,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: - Access-Control-Allow-Origin: - - '*' - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - 9f8a213fcfa034bd-SJC - Cache-Control: - - no-cache, must-revalidate + CF-RAY: + - a45d74605ac9df5a-YYZ Connection: - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' Content-Type: - application/json Date: - - Fri, 08 May 2026 17:19:32 GMT + - Mon, 05 Oct 2026 15:28:10 GMT Server: - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload Transfer-Encoding: - chunked - Via: - - envoy-router-6bb47b9fb6-dlvwm - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 + Vary: + - Accept-Encoding + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T15:28:10Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC content-length: - - '383' - openai-model: - - gpt-3.5-turbo-instruct:20230824-v2 - openai-organization: - - braintrust-data - openai-processing-ms: - - '183' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - set-cookie: - - __cf_bm=J7BwJkU4m3tydxu1TkSveYALVcfIWtQLIywG_Z3ocQk-1778260771.8019595-1.0.1.1-58SjNkFWP1Gv_WcjPlzrr_En2YMwTJG.KPMIrIwd3zcJe9Di0qBgjsHtErhXq_sKTthex4_ynaF99wvwmCV3gaDS.LnE5NkagenL9n2DEQW3BCP13HK5FbW4LoU1u1sF; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 08 May 2026 - 17:49:32 GMT - x-engine-geography: - - US - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '3500' - x-ratelimit-limit-tokens: - - '90000' - x-ratelimit-remaining-requests: - - '3498' - x-ratelimit-remaining-tokens: - - '89995' - x-ratelimit-reset-requests: - - 17ms - x-ratelimit-reset-tokens: - - 2ms - x-request-id: - - req_9df32c1fe39a4e36b70eb5a753424340 + - '505' + request-id: + - req_011CfjKtxqcB34mjyVcGfyb1 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-6a95ef5c791ed7efd537c7ce7db18b8e-7296b20215cb2017-01 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml index 16d2c69e6..76c15d23e 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml @@ -1,114 +1,87 @@ interactions: - request: - body: '{"model":"gpt-3.5-turbo-instruct","prompt":"What''s 12 + 12?"}' + body: '{"model": "claude-haiku-4-5-20251001", "messages": [{"role": "user", "content": + [{"type": "text", "text": "What''s 12 + 12?"}]}], "max_tokens": 4096}' headers: - Accept: + accept: - application/json - Accept-Encoding: + accept-encoding: - gzip, deflate - Connection: + anthropic-version: + - '2023-06-01' + connection: - keep-alive - Content-Length: - - '61' - Content-Type: + content-length: + - '148' + content-type: - application/json - Host: - - api.openai.com - User-Agent: - - OpenAI/Python 1.99.9 - X-Stainless-Arch: - - arm64 - X-Stainless-Async: - - 'false' - X-Stainless-Lang: - - python - X-Stainless-OS: - - MacOS - X-Stainless-Package-Version: - - 1.99.9 - X-Stainless-Raw-Response: - - 'true' - X-Stainless-Runtime: - - CPython - X-Stainless-Runtime-Version: - - 3.12.12 - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' + host: + - api.anthropic.com + user-agent: + - litellm/1.74.0 method: POST - uri: https://api.openai.com/v1/completions + uri: https://api.anthropic.com/v1/messages response: body: - string: "{\n \"id\": \"cmpl-DdJ9HMEtJxIjFHx4qKe8JuNurI74b\",\n \"object\": - \"text_completion\",\n \"created\": 1778260771,\n \"model\": \"gpt-3.5-turbo-instruct:20230824-v2\",\n - \ \"choices\": [\n {\n \"text\": \"\\n\\n24\",\n \"index\": 0,\n - \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n - \ \"usage\": {\n \"prompt_tokens\": 8,\n \"completion_tokens\": 2,\n - \ \"total_tokens\": 10\n }\n}\n" + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjKtvXCemPAig21nU8po","type":"message","role":"assistant","content":[{"type":"text","text":"12 + + 12 = 24"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":16,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":13,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: - Access-Control-Allow-Origin: - - '*' - CF-Cache-Status: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T15:28:09Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T15:28:10Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: - DYNAMIC - CF-Ray: - - 9f8a213c1ccdad84-SJC - Cache-Control: - - no-cache, must-revalidate - Connection: + cf-ray: + - a45d745ca878051b-YYZ + connection: - keep-alive - Content-Type: + content-length: + - '505' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: - application/json - Date: - - Fri, 08 May 2026 17:19:31 GMT - Server: + date: + - Mon, 05 Oct 2026 15:28:10 GMT + request-id: + - req_011CfjKtvJ4PfUB5PJAtX7N4 + server: - cloudflare - Strict-Transport-Security: + strict-transport-security: - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: + traceresponse: + - 00-cc4703a5bd5ccd8c3a8cc52a270f3ff8-fbbf197d206af7a0-01 + transfer-encoding: - chunked - Via: - - envoy-router-854cb8774b-62tzj - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - content-length: - - '375' - openai-model: - - gpt-3.5-turbo-instruct:20230824-v2 - openai-organization: - - braintrust-data - openai-processing-ms: - - '159' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - set-cookie: - - __cf_bm=hwp6vll96JcVvTXRJKXVk62ZHOsK9JSVnfvv3Gj4Y74-1778260771.2184882-1.0.1.1-ERioTFSLV4jh0BseuCItsa0NWz_5hfwCiXAXQWmT7lSeqQnX6SpX0FzP0_eZAmE6kaot4SfUFoYAz3k1xMSgI.GXg0XSPNo.tevB4IaKhF56bzPVcfdlUJI6GR0Gmxj5; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 08 May 2026 - 17:49:31 GMT - x-engine-geography: - - US - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '3500' - x-ratelimit-limit-tokens: - - '90000' - x-ratelimit-remaining-requests: - - '3498' - x-ratelimit-remaining-tokens: - - '89995' - x-ratelimit-reset-requests: - - 17ms - x-ratelimit-reset-tokens: - - 2ms - x-request-id: - - req_41d504ad51ab45fa9ddfd578c6fc2ad1 + vary: + - Accept-Encoding + x-robots-tag: + - none status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml index 1944e645e..72e7b66c9 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml @@ -1,108 +1,87 @@ interactions: - request: - body: '{"model":"gpt-3.5-turbo-instruct","prompt":"What''s 12 + 12?"}' + body: '{"model":"claude-haiku-4-5-20251001","messages":[{"role":"user","content":[{"type":"text","text":"What''s + 12 + 12?"}]}],"max_tokens":64000}' headers: accept: - application/json accept-encoding: - gzip, deflate + anthropic-version: + - '2023-06-01' connection: - keep-alive content-length: - - '61' + - '138' content-type: - application/json host: - - api.openai.com + - api.anthropic.com user-agent: - - AsyncOpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 + - litellm/1.104.0 method: POST - uri: https://api.openai.com/v1/completions + uri: https://api.anthropic.com/v1/messages response: body: - string: "{\n \"id\": \"cmpl-EQw2EwFLi5niMiAzudM4bX55ItWI7\",\n \"object\": - \"text_completion\",\n \"created\": 1790088322,\n \"model\": \"gpt-3.5-turbo-instruct:20230824-v2\",\n - \ \"choices\": [\n {\n \"text\": \"\\n\\n12 + 12 = 24\",\n \"index\": - 0,\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n - \ ],\n \"usage\": {\n \"prompt_tokens\": 8,\n \"completion_tokens\": - 8,\n \"total_tokens\": 16\n }\n}\n" + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjKsuqW4bjbVgK2QHveQ","type":"message","role":"assistant","content":[{"type":"text","text":"12 + + 12 = 24"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":16,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":13,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: - Access-Control-Allow-Origin: - - '*' - Access-Control-Expose-Headers: - - X-Request-ID - - CF-Ray - - CF-Ray - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - a3f217cdc8234f2e-EWR - Cache-Control: - - no-cache, must-revalidate + CF-RAY: + - a45d7406ec16ab9a-YYZ Connection: - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' Content-Type: - application/json Date: - - Tue, 22 Sep 2026 14:45:23 GMT + - Mon, 05 Oct 2026 15:27:56 GMT Server: - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload Transfer-Encoding: - chunked - Via: - - envoy-router-canary-779bfb455c-h4fgh - X-Content-Type-Options: - - nosniff - alt-svc: - - h3=":443"; ma=86400 + Vary: + - Accept-Encoding + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T15:27:56Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T15:27:56Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T15:27:56Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T15:27:56Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC content-length: - - '385' - openai-model: - - gpt-3.5-turbo-instruct:20230824-v2 - openai-processing-ms: - - '536' - openai-version: - - '2020-10-01' - x-engine-geography: - - GB - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '3500' - x-ratelimit-limit-tokens: - - '90000' - x-ratelimit-remaining-requests: - - '3498' - x-ratelimit-remaining-tokens: - - '89995' - x-ratelimit-reset-requests: - - 17ms - x-ratelimit-reset-tokens: - - 2ms - x-request-id: - - req_5cd2bdfd92084ba0afaa39da574c3435 + - '505' + request-id: + - req_011CfjKsubrsvDN7Ui69QuNY + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-0d89190d983c5538b75b282095830ced-8fb7095697b6c7ef-01 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml index 134b846d9..4803df72d 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml @@ -1,108 +1,87 @@ interactions: - request: - body: '{"model":"gpt-3.5-turbo-instruct","prompt":"What''s 12 + 12?"}' + body: '{"model": "claude-haiku-4-5-20251001", "messages": [{"role": "user", "content": + [{"type": "text", "text": "What''s 12 + 12?"}]}], "max_tokens": 64000}' headers: accept: - application/json accept-encoding: - gzip, deflate + anthropic-version: + - '2023-06-01' connection: - keep-alive content-length: - - '61' + - '149' content-type: - application/json host: - - api.openai.com + - api.anthropic.com user-agent: - - OpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 + - litellm/1.104.0 method: POST - uri: https://api.openai.com/v1/completions + uri: https://api.anthropic.com/v1/messages response: body: - string: "{\n \"id\": \"cmpl-EQw2DoWwq9UzMMva9tMIEM8pWgsM7\",\n \"object\": - \"text_completion\",\n \"created\": 1790088321,\n \"model\": \"gpt-3.5-turbo-instruct:20230824-v2\",\n - \ \"choices\": [\n {\n \"text\": \"\\n\\n24\",\n \"index\": 0,\n - \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n - \ \"usage\": {\n \"prompt_tokens\": 8,\n \"completion_tokens\": 2,\n - \ \"total_tokens\": 10\n }\n}\n" + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjKssFDhZUXmFRLRGU9g","type":"message","role":"assistant","content":[{"type":"text","text":"12 + + 12 = 24"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":16,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":13,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: - access-control-allow-origin: - - '*' - access-control-expose-headers: - - X-Request-ID - - CF-Ray - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - cache-control: - - no-cache, must-revalidate + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T15:27:55Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T15:27:55Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T15:27:55Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T15:27:55Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS cf-cache-status: - DYNAMIC cf-ray: - - a3f217c15a8dc356-EWR + - a45d7401584f713b-YYZ connection: - keep-alive content-length: - - '375' + - '505' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' content-type: - application/json date: - - Tue, 22 Sep 2026 14:45:21 GMT - openai-model: - - gpt-3.5-turbo-instruct:20230824-v2 - openai-processing-ms: - - '893' - openai-version: - - '2020-10-01' + - Mon, 05 Oct 2026 15:27:55 GMT + request-id: + - req_011CfjKsqtce3aScJFtzBDuQ server: - cloudflare strict-transport-security: - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-fbae6e27bb3bfab6d4b16c29c4acdf78-6b4815d879d330e0-01 transfer-encoding: - chunked - via: - - envoy-router-57c56456f9-pm9mb - x-content-type-options: - - nosniff - x-engine-geography: - - AU - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '3500' - x-ratelimit-limit-tokens: - - '90000' - x-ratelimit-remaining-requests: - - '3498' - x-ratelimit-remaining-tokens: - - '89995' - x-ratelimit-reset-requests: - - 17ms - x-ratelimit-reset-tokens: - - 2ms - x-request-id: - - req_edba99bc411e44738524a73bf0721627 + vary: + - Accept-Encoding + x-robots-tag: + - none status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 472900f2c..e66e7bd43 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -18,7 +18,7 @@ TEST_ORG_ID = "test-org-litellm-py-tracing" PROJECT_NAME = "test-project-litellm-py-tracing" TEST_MODEL = "gpt-4o-mini" # cheapest model for tests -TEST_TEXT_MODEL = "gpt-3.5-turbo-instruct" +TEST_TEXT_MODEL = "anthropic/claude-haiku-4-5-20251001" TEST_PROMPT = "What's 12 + 12?" TEST_SYSTEM_PROMPT = "You are a helpful assistant that only responds with numbers." TEST_CACHE_MODEL = "anthropic/claude-haiku-4-5-20251001" @@ -228,7 +228,7 @@ def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["model"] == TEST_TEXT_MODEL - assert span["metadata"]["provider"] == "openai" + assert span["metadata"]["provider"] == "anthropic" assert TEST_PROMPT in str(span["input"]) assert "text" in span["output"][0]