feat: carry judge reasoning in the judge track payload and telemetry

AGENTS.md

@@ -238,6 +238,7 @@ Payload attached to every LaunchDarkly tracking event.
 | `graphKey` | `str?` | Present when the event was produced inside an agent graph. |
 | `toolKey` | `str?` | Present when the event is for a tool call. |
 | `judgeConfigKey` | `str?` | Present when the event is from a judge execution. |
+| `judgeReasoning` | `str?` | The judge's explanation of its score. Opt-in via `LD_CAPTURE_JUDGE_REASONING=true` and absent otherwise, because reasoning may quote the conversation it graded. Truncated at 4000 characters. |
 
 #### `NativeTool`
 

TELEMETRY-CONTRACT.md

@@ -226,17 +226,31 @@ SDK writes a `gen_ai.evaluation.result` span event on that `invoke_agent` span:
 |---|---|
 | `gen_ai.evaluation.name` | judge config key |
 | `gen_ai.evaluation.score.value` | numeric score, only when the judge returned a finite number |
+| `gen_ai.evaluation.explanation` | judge reasoning, only when opted in and non-empty. Truncated at 4000 characters |
 
 The same keys are mirrored as span attributes, so section 2 lists them too.
 `gen_ai.evaluation.score.label` is not invented.
-The existing `track(evaluationMetricKey)` call is unchanged and still feeds AI Config Monitoring —
-a judge that returns a non-numeric score emits no evaluation event but still tracks the metric.
-
-`gen_ai.evaluation.explanation` is deliberately **not** emitted. The judge's reasoning is
-model-generated prose about the user's conversation — content, under section 7 — and content
-attributes require `captureContent` / `capture_content`, a handler-factory option this layer does
-not receive. The reasoning is still returned to the caller in `judgeResults` / `judge_results`;
-only the telemetry copy is withheld. Exporting it needs its own opt-in.
+The existing `track(evaluationMetricKey)` call still feeds AI Config Monitoring — a judge that
+returns a non-numeric score emits no evaluation event but still tracks the metric.
+
+`gen_ai.evaluation.explanation` is **opt-in and off by default**. Set
+`LD_CAPTURE_JUDGE_REASONING` to `true`, `1`, `on` or `yes` to export it; anything else, including
+an unset variable, withholds reasoning from both the span and the track payload while keeping the
+score.
+
+> **Warning — reasoning can contain sensitive data.** A judge sees the full input and output it is
+> grading, and its reasoning is free-form model prose that may quote them verbatim. Enabling this
+> can therefore send PII, secrets, or other regulated content to LaunchDarkly. Nothing in the SDK
+> redacts it: there is no PII or token detection on the reasoning text, only a 4000-character clip.
+> Enable it only where the evaluated conversations are known not to carry sensitive data, or where
+> your own judge instructions keep reasoning abstract.
+
+The switch is separate from `capture_content` (section 7) deliberately: gating reasoning there
+would force a caller who wants this one field to also ship every request and response.
+
+When enabled, the same reasoning rides the evaluation metric event as `TrackData.judgeReasoning`,
+on both the inline path and the deferred `run_judge` path, so it reaches LaunchDarkly through the
+evaluation pipeline rather than only through the collector.
 
 ---
 

packages/client/src/launchdarkly_ai_server/judges.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import random
 from collections.abc import Callable
 from math import isfinite
@@ -25,8 +26,11 @@
     parse_json_with_possible_fences,
     to_ld_context,
     to_usage_dict,
+    truncate_judge_reasoning,
 )
 
+_REASONING_ENABLED_VALUES = frozenset({"1", "true", "on", "yes"})
+
 
 def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool:
     """Returns True when the handler covers the given provider or is a wildcard."""
@@ -38,6 +42,38 @@ def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool:
 
 logger = logging.getLogger(__name__)
 
+
+def judge_reasoning_enabled() -> bool:
+    """Whether judge reasoning leaves the process. Opt-in via ``LD_CAPTURE_JUDGE_REASONING``.
+
+    Reasoning is model prose about the evaluated conversation and may quote it, so it stays in
+    the process until a deployment asks for it.
+    """
+    value = os.environ.get("LD_CAPTURE_JUDGE_REASONING", "").strip().lower()
+    return value in _REASONING_ENABLED_VALUES
+
+
+def build_judge_track_data(
+    base_track_data: TrackData,
+    judge_config_key: str,
+    reasoning: str | None,
+) -> TrackData:
+    """Builds the payload for a judge's evaluation metric event, carrying ``judgeReasoning``
+    alongside the score.
+    """
+    track_data: TrackData = {**base_track_data, "judgeConfigKey": judge_config_key}
+    if reasoning and judge_reasoning_enabled():
+        track_data["judgeReasoning"] = truncate_judge_reasoning(reasoning)
+    return track_data
+
+
+def judge_explanation(reasoning: str | None) -> str | None:
+    """The reasoning to put on telemetry, or ``None`` when suppressed."""
+    if not reasoning or not judge_reasoning_enabled():
+        return None
+    return truncate_judge_reasoning(reasoning)
+
+
 _FORMATTING_INSTRUCTIONS = "\n".join(
     [
         "Your response MUST be in valid JSON format with the following structure:",
@@ -201,10 +237,7 @@ async def run_judges(
                 )
                 numeric_score = _numeric_score(score)
                 if numeric_score is not None:
-                    record_evaluation(
-                        numeric_score,
-                        reasoning if judge_handler.capture_content else None,
-                    )
+                    record_evaluation(numeric_score, judge_explanation(reasoning))
 
                 evaluation_metric_key = (
                     judge_ai_config.get("evaluationMetricKey")
@@ -218,7 +251,7 @@ async def run_judges(
                     client.track(
                         evaluation_metric_key,
                         to_ld_context(client, user_context),
-                        {**base_track_data, "judgeConfigKey": judge_key},
+                        build_judge_track_data(base_track_data, judge_key, reasoning),
                         score,
                     )
 
@@ -432,19 +465,16 @@ def _matches(h: ProviderHandler) -> bool:
         reasoning = parsed.get("reasoning", "")
         numeric_score = _numeric_score(score)
         if numeric_score is not None:
-            record_evaluation(
-                numeric_score,
-                reasoning if judge_handler.capture_content else None,
-            )
+            record_evaluation(numeric_score, judge_explanation(reasoning))
         raw_usage = result["usage"]
 
         usage = to_usage_dict(raw_usage)
 
-        merged_track_data: TrackData = {
-            **task.parent_track_data,
-            **result["track_data"],
-            "judgeConfigKey": task.config_key,
-        }
+        merged_track_data: TrackData = build_judge_track_data(
+            {**task.parent_track_data, **result["track_data"]},
+            task.config_key,
+            reasoning,
+        )
 
         return JudgeRunResult(
             score=score, response=reasoning, usage=usage, track_data=merged_track_data

packages/client/src/launchdarkly_ai_server/utils.py

@@ -682,6 +682,16 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None:
     span.add_event("feature_flag", feature_flag_attrs)
 
 
+JUDGE_REASONING_MAX_LENGTH = 4000
+
+
+def truncate_judge_reasoning(reasoning: str) -> str:
+    """Clips reasoning to :data:`JUDGE_REASONING_MAX_LENGTH`, marking that it was clipped."""
+    if len(reasoning) <= JUDGE_REASONING_MAX_LENGTH:
+        return reasoning
+    return reasoning[:JUDGE_REASONING_MAX_LENGTH] + "…"
+
+
 def set_openllmetry_prompt(span: Any, messages: list[dict[str, str]]) -> None:
     """Set OpenLLMetry-style indexed prompt attributes on a span.
 

packages/client/tests/test_judges.py

@@ -3,13 +3,22 @@
 Reference: TESTING.md §3.14
 """
 
+import json
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 import launchdarkly_ai_server.lifecycle as lifecycle_module
-from launchdarkly_ai_server import JudgeResult, ProviderHandler, run_judges
+from launchdarkly_ai_server import (
+    JudgeResult,
+    JudgeTask,
+    ProviderHandler,
+    run_judge,
+    run_judges,
+)
+from launchdarkly_ai_server.judges import judge_explanation
+from launchdarkly_ai_server.utils import JUDGE_REASONING_MAX_LENGTH
 
 CONTEXT = {"kind": "user", "key": "u1"}
 
@@ -387,6 +396,160 @@ async def test_returns_empty_dict_when_judges_array_is_empty(
         assert result == {}
 
 
+class TestJudgeReasoning:
+    """Reasoning leaves the process on the metric event and on the evaluation event.
+
+    Reference: TELEMETRY-CONTRACT.md §4a
+    """
+
+    @staticmethod
+    def _judge_variation() -> dict[str, Any]:
+        return {
+            "model": {"name": "gpt-4"},
+            "provider": {"name": "TestProvider"},
+            "instructions": "judge",
+            "evaluationMetricKey": "quality",
+            "_ldMeta": {
+                "enabled": True,
+                "variationKey": "j1",
+                "version": 1,
+                "mode": "messages",
+            },
+        }
+
+    @staticmethod
+    def _parent_config() -> dict[str, Any]:
+        return {
+            "model": {"name": "gpt-4"},
+            "provider": {"name": "TestProvider"},
+            "instructions": "hi",
+            "judgeConfiguration": {"judges": [{"key": "judge-1", "samplingRate": 1.0}]},
+        }
+
+    @pytest.fixture(autouse=True)
+    def _opt_in(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("LD_CAPTURE_JUDGE_REASONING", "true")
+
+    async def _run(
+        self, client: MagicMock, reasoning: str = "clear and correct"
+    ) -> None:
+        async def fn(
+            config, user_input, tool_handlers, variables, history=None
+        ) -> dict:  # type: ignore[override]
+            return {
+                "output": json.dumps({"score": 0.9, "reasoning": reasoning}),
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            }
+
+        handler = ProviderHandler(fn=fn, provides_for=("TestProvider", "messages"))  # type: ignore[arg-type]
+        client.variation = AsyncMock(return_value=self._judge_variation())
+
+        import random
+
+        with patch.object(random, "random", return_value=0.0):
+            await run_judges(
+                config=self._parent_config(),
+                user_context=CONTEXT,
+                handler=handler,
+                user_input="q",
+                llm_response="response",
+                base_track_data={"runId": "run-1", "configKey": "parent"},
+            )
+
+    async def test_metric_event_carries_reasoning(
+        self, mock_ld_client: MagicMock
+    ) -> None:
+        await self._run(mock_ld_client)
+
+        metric_key, _context, track_data, score = mock_ld_client.track.call_args[0]
+        assert metric_key == "quality"
+        assert score == 0.9
+        assert track_data["judgeReasoning"] == "clear and correct"
+        assert track_data["judgeConfigKey"] == "judge-1"
+        assert track_data["runId"] == "run-1"
+
+    async def test_reasoning_is_withheld_by_default_without_losing_the_score(
+        self, mock_ld_client: MagicMock, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.delenv("LD_CAPTURE_JUDGE_REASONING")
+        await self._run(mock_ld_client)
+
+        _metric_key, _context, track_data, score = mock_ld_client.track.call_args[0]
+        assert "judgeReasoning" not in track_data
+        assert score == 0.9
+
+    async def test_empty_reasoning_is_omitted(self, mock_ld_client: MagicMock) -> None:
+        await self._run(mock_ld_client, reasoning="")
+
+        _metric_key, _context, track_data, _score = mock_ld_client.track.call_args[0]
+        assert "judgeReasoning" not in track_data
+
+    async def test_long_reasoning_is_truncated(self, mock_ld_client: MagicMock) -> None:
+        await self._run(
+            mock_ld_client, reasoning="a" * (JUDGE_REASONING_MAX_LENGTH + 50)
+        )
+
+        _metric_key, _context, track_data, _score = mock_ld_client.track.call_args[0]
+        assert len(track_data["judgeReasoning"]) == JUDGE_REASONING_MAX_LENGTH + 1
+        assert track_data["judgeReasoning"].endswith("…")
+
+    def test_explanation_follows_the_opt_in_not_capture_content(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        assert judge_explanation("clear and correct") == "clear and correct"
+        assert judge_explanation("") is None
+
+        monkeypatch.delenv("LD_CAPTURE_JUDGE_REASONING")
+        assert judge_explanation("clear and correct") is None
+
+
+class TestRunJudgeTrackData:
+    """The deferred path must carry reasoning too, so a background worker's track call
+    is not a downgrade from the inline one.
+    """
+
+    @staticmethod
+    def _task() -> JudgeTask:
+        return JudgeTask(
+            config_key="judge-1",
+            judge_config={
+                "model": {"name": "gpt-4"},
+                "provider": {"name": "TestProvider"},
+                "instructions": "judge",
+            },
+            judge_meta={"enabled": True, "variationKey": "j1", "version": 1},
+            actual_output="response",
+            user_context=CONTEXT,
+            judge_provider="TestProvider",
+            judge_mode="messages",
+            collapse_messages=False,
+            parent_track_data={"runId": "run-1", "configKey": "parent"},
+            evaluation_metric_key="quality",
+        )
+
+    async def test_track_data_carries_reasoning(
+        self, mock_ld_client: MagicMock, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("LD_CAPTURE_JUDGE_REASONING", "true")
+
+        async def fn(
+            config, user_input, tool_handlers, variables, history=None
+        ) -> dict:  # type: ignore[override]
+            return {
+                "output": '{"score": 0.4, "reasoning": "missed the question"}',
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            }
+
+        handler = ProviderHandler(fn=fn, provides_for=("TestProvider", "messages"))  # type: ignore[arg-type]
+
+        result = await run_judge(self._task(), [handler])
+
+        assert result is not None
+        assert result.response == "missed the question"
+        assert result.track_data["judgeReasoning"] == "missed the question"
+        assert result.track_data["judgeConfigKey"] == "judge-1"
+
+
 class TestScoreGuard:
     """`float(score)` used to sit ahead of the evaluation-metric track, so a junk score killed it."""
 

tests/test_cross_handler_parity.py

@@ -327,8 +327,8 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough(
     "gen_ai.agent.name",
     "gen_ai.conversation.id",
     # Judge evaluation event + mirrored span attributes on the judge invoke_agent span.
-    # `explanation` is gated on the judge handler's capture_content, like every other content
-    # attribute. See TELEMETRY-CONTRACT.md 4a.
+    # `explanation` carries the judge's reasoning and is opt-in via
+    # LD_CAPTURE_JUDGE_REASONING. See TELEMETRY-CONTRACT.md 4a.
     "gen_ai.evaluation.result",
     "gen_ai.evaluation.name",
     "gen_ai.evaluation.score.value",