fix(litellm): use underlying provider for metadata, drop eager serialization

Align the LiteLLM integration with .agents/skills/sdk-integrations/SKILL.md.
Audit against the SKILL surfaced four issues:

- **`metadata.provider` was the framework name.** SKILL §Completion-style
  explicitly says LiteLLM must set `metadata.provider` to the underlying
  provider (openai / cohere / openrouter / ...), not "litellm". Fixed by
  routing every span through `_resolve_provider(model)` which calls
  `litellm.get_llm_provider(model)`; falls back to `"litellm"` only if the
  lookup fails. `@lru_cache(maxsize=256)` on the resolver so the hot path
  doesn't re-dispatch through litellm's routing table per call.
- **Excess serialization.** Every non-streaming wrapper called
  `_try_to_dict(response)` on the full pydantic response then indexed
  into it — but `bt_json` already serializes at log time. Replaced with
  direct attribute reads via a shared `_get_field(obj, key, default)`
  helper that works on both dicts and SDK objects (openai / anthropic /
  cohere had all been open-coding this). Streaming paths left intact.
- **Dead code.** `_is_litellm_patched` was only used by crewai until #587
  moved crewai to the "wrapper never owns tokens" pattern; no remaining
  callers anywhere. Removed the helper and its two subprocess tests.
- **Excessive comments.** Trimmed the huge `wrap_litellm` /
  `_is_litellm_patched` docstrings and the "wrapt wrapper for litellm.X"
  one-liner on every wrapper.

Also lifted the rerank output cap (`results[:100]`) into
`integrations/utils.RERANK_OUTPUT_MAX_RESULTS` and documented the
"rerank exception to no-silent-caps" rule in the SKILL — cohere carries
the same magic number and can adopt the shared constant in a follow-up.

## Test plan

- [x] Updated existing VCR-backed tests to assert the underlying
      provider (`openai`, `cohere`, `openrouter`) instead of `"litellm"`.
      No cassette re-recording (HTTP requests unchanged; span-shape
      assertions only).
- [x] Added positive `metadata["provider"] == "openrouter"` assertion
      to the existing OpenRouter cassette test as coverage for the new
      resolver.
- [x] Dropped the two `_is_litellm_patched` subprocess tests alongside
      the helper.
- [x] **No new mocks/fakes, no cassette re-recording.**
- [x] `ruff check` + `ruff format --check` on touched files — clean.
- [ ] Reviewer to run: `cd py && nox -s "test_litellm(latest)"` and
      `cd py && nox -s "test_litellm(1.74.0)"` (author's box has a
      broken Python 3.14 install).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: starfolkbot

.agents/skills/sdk-integrations/SKILL.md

@@ -165,6 +165,7 @@ The integration that directly owns the model/provider API response owns token ac
 - Do not over-serialize. Braintrust serializes at send/log time. Integration tracing only needs readable Python dicts/lists/scalars and materialized attachments.
 - Keep wrapper bodies thin: prepare traced input, open span, call provider, normalize result, log.
 - Do not wrap `span.log(...)` / `span.set_attributes(...)` in broad try/except. Braintrust span methods are boundary-safe.
+- **Rerank exception to "no silent caps":** rerank result lists can be huge and are noisy in the span UI; cap the `output` list at a fixed max (e.g. 100 entries) via a named constant. The cap is compact-span hygiene, not coverage bounding — the model already ranked everything.
 
 ### Span origin
 

py/src/braintrust/integrations/litellm/__init__.py

@@ -1,74 +1,18 @@
 """Braintrust LiteLLM integration."""
 
-import importlib
-
 from .integration import LiteLLMIntegration
-from .patchers import LiteLLMAcompletionPatcher, LiteLLMCompletionPatcher, wrap_litellm
+from .patchers import wrap_litellm
 
 
 def patch_litellm() -> bool:
-    """Patch LiteLLM to add Braintrust tracing.
-
-    This wraps litellm.completion, litellm.acompletion, litellm.responses,
-    litellm.aresponses, litellm.image_generation, litellm.aimage_generation,
-    litellm.embedding, litellm.aembedding, litellm.moderation,
-    litellm.speech, litellm.aspeech, litellm.transcription,
-    litellm.atranscription, litellm.rerank, and litellm.arerank to automatically
-    create Braintrust spans with detailed token metrics, timing, and costs.
-
-    Returns:
-        True if LiteLLM was patched (or already patched), False if LiteLLM is not installed.
+    """Patch LiteLLM top-level callables to emit Braintrust spans.
 
-    Example:
-        ```python
-        import braintrust
-        braintrust.integrations.litellm.patch_litellm()
-
-        import litellm
-        from braintrust import init_logger
-
-        logger = init_logger(project="my-project")
-        response = litellm.completion(
-            model="gpt-4o-mini",
-            messages=[{"role": "user", "content": "Hello"}]
-        )
-        ```
+    Returns ``True`` if LiteLLM was patched (or already patched), ``False``
+    if LiteLLM is not installed.
     """
     return LiteLLMIntegration.setup()
 
 
-def _is_litellm_patched() -> bool:
-    """Return ``True`` when the LiteLLM completion entry points are patched.
-
-    Internal helper used by other Braintrust integrations (e.g. CrewAI,
-    which delegates its LLM calls to LiteLLM) to decide whether to emit
-    token metrics on their own LLM spans.  When LiteLLM is already
-    patched, the Braintrust ``Completion`` span it produces is the leaf
-    span and must own token accounting; non-leaf wrappers should emit
-    timing-only metrics to avoid double-counting during trace-tree
-    rollup.
-
-    Not part of the public API — cross-integration callers import this
-    directly by its underscore-prefixed name.
-
-    Returns ``False`` if ``litellm`` is not importable or neither of the
-    ``completion`` / ``acompletion`` entry points has been patched.
-    """
-    try:
-        litellm = importlib.import_module("litellm")
-    except ImportError:
-        return False
-
-    completion = getattr(litellm, "completion", None)
-    acompletion = getattr(litellm, "acompletion", None)
-    return bool(
-        LiteLLMCompletionPatcher.has_patch_marker(completion)
-        or LiteLLMCompletionPatcher.has_patch_marker(litellm)
-        or LiteLLMAcompletionPatcher.has_patch_marker(acompletion)
-        or LiteLLMAcompletionPatcher.has_patch_marker(litellm)
-    )
-
-
 __all__ = [
     "LiteLLMIntegration",
     "patch_litellm",

py/src/braintrust/integrations/litellm/patchers.py

@@ -171,37 +171,9 @@ class LiteLLMArerankPatcher(FunctionWrapperPatcher):
 
 
 def wrap_litellm(litellm: Any) -> Any:
-    """Wrap a LiteLLM module to add Braintrust tracing.
+    """Wrap a LiteLLM module-like object in-place with Braintrust tracing.
 
-    Unlike :func:`patch_litellm`, which patches the globally-imported ``litellm``
-    module, this function instruments a specific module object (or any object
-    that exposes the same top-level callables such as ``completion``,
-    ``acompletion``, ``responses``, ``aresponses``, ``image_generation``,
-    ``text_completion``, ``atext_completion``, ``aimage_generation``,
-    ``embedding``, ``aembedding``, ``moderation``, ``amoderation``, ``speech``,
-    ``aspeech``, ``transcription``, ``atranscription``, ``rerank``, and
-    ``arerank``). Each patcher is applied idempotently — calling
-    ``wrap_litellm`` twice on the same object is safe.
-
-    Args:
-        litellm: The ``litellm`` module or a module-like object that exposes
-            the standard LiteLLM top-level functions.
-
-    Returns:
-        The same *litellm* object, with tracing wrappers applied in-place.
-
-    Example::
-
-        import litellm
-        from braintrust.integrations.litellm import wrap_litellm
-
-        wrap_litellm(litellm)
-
-        # All subsequent calls are automatically traced.
-        response = litellm.completion(
-            model="gpt-4o-mini",
-            messages=[{"role": "user", "content": "Hello"}],
-        )
+    Idempotent; safe to call twice on the same object.
     """
     for patcher in _ALL_LITELLM_PATCHERS:
         patcher.wrap_target(litellm)

py/src/braintrust/integrations/litellm/test_litellm.py

@@ -8,7 +8,6 @@
 from braintrust.integrations.litellm import patch_litellm
 from braintrust.integrations.test_utils import (
     assert_metrics_are_valid,
-    run_in_subprocess,
     verify_autoinstrument_script,
 )
 from braintrust.test_helpers import assert_dict_matches, init_test_logger
@@ -72,7 +71,7 @@ def test_litellm_completion_metrics(memory_logger) -> None:
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
 
 
@@ -96,7 +95,7 @@ async def test_litellm_acompletion_metrics(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
 
 
@@ -119,7 +118,7 @@ def test_litellm_text_completion_metrics(memory_logger) -> None:
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_TEXT_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
     assert "text" in span["output"][0]
 
@@ -144,7 +143,7 @@ async def test_litellm_atext_completion_metrics(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_TEXT_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
     assert "text" in span["output"][0]
 
@@ -186,7 +185,7 @@ def test_litellm_completion_streaming_sync(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
     assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower()
 
@@ -220,7 +219,7 @@ async def test_litellm_acompletion_streaming_async(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
 
 
@@ -250,7 +249,7 @@ def test_litellm_responses_metrics(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
 
 
@@ -281,7 +280,7 @@ async def test_litellm_aresponses_metrics(memory_logger):
     metrics = span["metrics"]
     assert_metrics_are_valid(metrics, start, end)
     assert span["metadata"]["model"] == TEST_MODEL
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert TEST_PROMPT in str(span["input"])
 
 
@@ -303,7 +302,7 @@ def test_litellm_embeddings(memory_logger):
     span = spans[0]
     assert span
     assert span["metadata"]["model"] == "text-embedding-ada-002"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert "This is a test" in str(span["input"])
 
 
@@ -323,7 +322,7 @@ async def test_litellm_aembedding(memory_logger):
     span = spans[0]
     assert span
     assert span["metadata"]["model"] == "text-embedding-ada-002"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert "This is a test" in str(span["input"])
 
 
@@ -344,7 +343,7 @@ def test_litellm_moderation(memory_logger):
     span = spans[0]
     assert span
     assert span["metadata"]["model"] == "omni-moderation-latest"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert "This is a test message" in str(span["input"])
 
 
@@ -362,7 +361,7 @@ async def test_litellm_amoderation(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["metadata"]["model"] == "omni-moderation-latest"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert "This is a test message" in str(span["input"])
 
 
@@ -386,7 +385,7 @@ def test_litellm_image_generation(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["metadata"]["model"] == "gpt-image-1-mini"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert span["input"] == prompt
     assert span["output"]["images_count"] == 1
     assert span["metrics"]["duration"] >= 0
@@ -413,7 +412,7 @@ async def test_litellm_aimage_generation(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["metadata"]["model"] == "gpt-image-1-mini"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert span["input"] == prompt
     assert span["output"]["images_count"] == 1
     assert span["metrics"]["duration"] >= 0
@@ -457,7 +456,7 @@ def test_litellm_transcription(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["metadata"]["model"] == "whisper-1"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert isinstance(span["input"]["file"], Attachment)
     assert span["input"]["file"].reference["filename"] == "test_audio.wav"
     assert span["input"]["file"].reference["content_type"] in ("audio/x-wav", "audio/wav")  # OS-dependent
@@ -479,7 +478,7 @@ async def test_litellm_atranscription(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["metadata"]["model"] == "whisper-1"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert isinstance(span["input"]["file"], Attachment)
     assert span["input"]["file"].reference["filename"] == "test_audio.wav"
     assert span["input"]["file"].reference["content_type"] in ("audio/x-wav", "audio/wav")  # OS-dependent
@@ -506,7 +505,7 @@ def test_litellm_speech(memory_logger):
     assert span["metadata"]["model"] == "tts-1"
     assert span["metadata"]["voice"] == "alloy"
     assert span["metadata"]["response_format"] == "mp3"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert span["input"] == "Hello, this is a test."
     _assert_speech_output_attachment(span)
 
@@ -532,7 +531,7 @@ async def test_litellm_aspeech(memory_logger):
     assert span["metadata"]["model"] == "tts-1"
     assert span["metadata"]["voice"] == "alloy"
     assert span["metadata"]["response_format"] == "mp3"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "openai"
     assert span["input"] == "Hello, this is a test."
     _assert_speech_output_attachment(span)
 
@@ -636,7 +635,7 @@ async def test_litellm_async_parallel_requests(memory_logger):
     # Verify each span has proper data
     for i, span in enumerate(spans):
         assert span["metadata"]["model"] == TEST_MODEL
-        assert span["metadata"]["provider"] == "litellm"
+        assert span["metadata"]["provider"] == "openai"
         assert prompts[i] in str(span["input"])
         assert_metrics_are_valid(span["metrics"])
 
@@ -687,7 +686,7 @@ def test_litellm_tool_calls(memory_logger):
             "span_attributes": {"type": "llm", "name": "Completion"},
             "metadata": {
                 "model": TEST_MODEL,
-                "provider": "litellm",
+                "provider": "openai",
                 "tools": lambda tools_list: (
                     len(tools_list) == 1
                     and any(tool.get("function", {}).get("name") == "get_weather" for tool in tools_list)
@@ -790,45 +789,6 @@ def test_patch_litellm_responses():
     verify_autoinstrument_script("test_patch_litellm_responses.py")
 
 
-def test_is_litellm_patched_internal_helper():
-    """Internal ``_is_litellm_patched()`` flips True after setup and honors manual wrap.
-
-    Run in a subprocess so that marker attributes set by other tests in this
-    module do not leak into the assertion.
-    """
-    result = run_in_subprocess(
-        """
-        from braintrust.integrations.litellm import _is_litellm_patched, patch_litellm, wrap_litellm
-        assert _is_litellm_patched() is False, 'expected unpatched at startup'
-        assert patch_litellm() is True
-        assert _is_litellm_patched() is True, 'expected True after patch_litellm()'
-        # idempotent wrap on the already-patched module stays True.
-        import litellm
-        wrap_litellm(litellm)
-        assert _is_litellm_patched() is True
-        print("SUCCESS")
-        """
-    )
-    assert result.returncode == 0, result.stderr
-    assert "SUCCESS" in result.stdout
-
-
-def test_is_litellm_patched_returns_false_without_litellm_installed():
-    """``_is_litellm_patched()`` must not raise when ``litellm`` is absent."""
-    result = run_in_subprocess(
-        """
-        import sys
-        # Simulate a missing litellm install.
-        sys.modules['litellm'] = None
-        from braintrust.integrations.litellm import _is_litellm_patched
-        assert _is_litellm_patched() is False
-        print("SUCCESS")
-        """
-    )
-    assert result.returncode == 0, result.stderr
-    assert "SUCCESS" in result.stdout
-
-
 def test_patch_litellm_aresponses():
     """Test that patch_litellm() patches aresponses (subprocess to avoid global state pollution)."""
     verify_autoinstrument_script("test_patch_litellm_aresponses.py")
@@ -901,10 +861,10 @@ def test_litellm_openrouter_no_booleans_in_metrics(memory_logger):
     spans = memory_logger.pop()
     assert len(spans) == 1
     metrics = spans[0]["metrics"]
-    metadata_keys = set(spans[0]["metadata"])
-    assert "api_key" not in metadata_keys
+    metadata = spans[0]["metadata"]
+    assert "api_key" not in metadata
+    assert metadata["provider"] == "openrouter"
 
-    # No boolean values should be in metrics
     for key, value in metrics.items():
         assert not isinstance(value, bool)
     assert "is_byok" not in metrics
@@ -934,7 +894,7 @@ def test_litellm_rerank(memory_logger):
     span = spans[0]
     assert span["span_attributes"]["name"] == "Rerank"
     assert span["span_attributes"]["type"] == "llm"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "cohere"
     assert span["metadata"]["model"] == RERANK_MODEL
     assert span["metadata"]["top_n"] == 2
     assert span["metadata"]["document_count"] == 3
@@ -979,7 +939,7 @@ async def test_litellm_arerank(memory_logger):
     assert len(spans) == 1
     span = spans[0]
     assert span["span_attributes"]["name"] == "Rerank"
-    assert span["metadata"]["provider"] == "litellm"
+    assert span["metadata"]["provider"] == "cohere"
     assert span["metadata"]["model"] == RERANK_MODEL
     assert span["metadata"]["top_n"] == 2
     assert span["metadata"]["document_count"] == 3