feat(AIC-3534): add a TypeSafe Jev judge handler (#149)

## Summary
- Add `launchdarkly-ai-typesafe`, a judge handler that sends a
variation's `classifiers` to Jev in one call.
- Each label becomes its own `judge_results` entry, tracked on that
label's `eventKey`, with the selected label as the reason.
- A score is the expected rubric index divided by one less than the
number of levels, then clamped to 0–1.

https://launchdarkly.atlassian.net/browse/AIC-3534

## Test plan
- [x] `pytest packages/typesafe/tests
packages/client/tests/test_judges.py
packages/client/tests/test_conversation.py
packages/client/tests/test_judge_scoring.py
packages/client/tests/test_schema.py`
- [ ] Run a TypeSafe judge variation and confirm one result per label,
with the label value in `response`


Made with [Cursor](https://cursor.com)

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: andrewklatzke

.github/workflows/release-please.yml

@@ -35,6 +35,7 @@ on:
           - packages/langchain-agents
           - packages/langchain-messages
           - packages/opencode-agents
+          - packages/typesafe
           - packages/vercel-agents
           - packages/vercel-messages
       dry_run:
@@ -73,6 +74,8 @@ jobs:
       package-langchain-messages-tag-name: ${{ steps.release.outputs['packages/langchain-messages--tag_name'] }}
       package-opencode-agents-released: ${{ steps.release.outputs['packages/opencode-agents--release_created'] }}
       package-opencode-agents-tag-name: ${{ steps.release.outputs['packages/opencode-agents--tag_name'] }}
+      package-typesafe-released: ${{ steps.release.outputs['packages/typesafe--release_created'] }}
+      package-typesafe-tag-name: ${{ steps.release.outputs['packages/typesafe--tag_name'] }}
       package-vercel-agents-released: ${{ steps.release.outputs['packages/vercel-agents--release_created'] }}
       package-vercel-agents-tag-name: ${{ steps.release.outputs['packages/vercel-agents--tag_name'] }}
       package-vercel-messages-released: ${{ steps.release.outputs['packages/vercel-messages--release_created'] }}
@@ -424,6 +427,37 @@ jobs:
           password: ${{ env.PYPI_AUTH_TOKEN }}
           packages-dir: packages/opencode-agents/dist/
 
+  release-typesafe:
+    runs-on: ubuntu-latest
+    needs: ['release-please']
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    if: ${{ needs.release-please.outputs.package-typesafe-released == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/ci
+        with:
+          workspace_path: packages/typesafe
+      - uses: ./.github/actions/build
+        with:
+          workspace_path: packages/typesafe
+      - name: Attest build provenance
+        uses: actions/attest@v4
+        with:
+          subject-path: 'packages/typesafe/dist/*'
+      - uses: launchdarkly/gh-actions/actions/release-secrets@release-secrets-v1.2.0
+        name: 'Get PyPI token'
+        with:
+          aws_assume_role: ${{ vars.AWS_ROLE_ARN }}
+          ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN'
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/typesafe/dist/
+
   release-vercel-agents:
     runs-on: ubuntu-latest
     needs: ['release-please']

.release-please-manifest.json

@@ -11,5 +11,6 @@
   "packages/vercel-messages": "0.1.0",
   "packages/vercel-agents": "0.1.0",
   "packages/litellm-messages": "0.1.0",
-  "packages/litellm-agents": "0.1.0"
+  "packages/litellm-agents": "0.1.0",
+  "packages/typesafe": "0.1.0"
 }

.sdk_metadata.json

@@ -90,6 +90,17 @@
         "aiCompletionConfig": { "introduced": "0.1" }
       }
     },
+    "python-ai-typesafe": {
+      "name": "TypeSafe Jev AI SDK for Python",
+      "type": "ai-provider",
+      "path": "packages/typesafe",
+      "languages": ["Python"],
+      "aiSdkNames": ["launchdarkly-ai-typesafe"],
+      "releases": { "tag-prefix": "launchdarkly-ai-typesafe-" },
+      "features": {
+        "aiJudgeConfig": { "introduced": "0.1" }
+      }
+    },
     "python-ai-langchain-agents": {
       "name": "LangChain Agents AI SDK for Python",
       "type": "ai-provider",

README.md

@@ -89,6 +89,7 @@ Tier 0 — Core Client           (launchdarkly-ai-server)
 | [`launchdarkly-ai-litellm-agents`](packages/litellm-agents/README.md) | `*` (any) | `agent` | OpenAI Agents SDK with LiteLLM models |
 | [`launchdarkly-ai-vercel-messages`](packages/vercel-messages/README.md) | `*` (any) | `messages` | Vercel AI SDK with AI Gateway routing |
 | [`launchdarkly-ai-vercel-agents`](packages/vercel-agents/README.md) | `*` (any) | `agent` | Native Vercel `ai.Agent` loop and graph adapter |
+| [`launchdarkly-ai-typesafe`](packages/typesafe/README.md) | TypeSafe | `messages` | TypeSafe Jev judge handler |
 
 ## Quick Start
 

examples/register.py

@@ -22,6 +22,7 @@
 from launchdarkly_ai_openai_agents import create_openai_agent_handler
 from launchdarkly_ai_openai_messages import create_openai_messages_handler
 from launchdarkly_ai_server import global_registry
+from launchdarkly_ai_typesafe import create_typesafe_handler
 
 global_registry.register(
     handlers=[
@@ -31,6 +32,7 @@
         create_claude_messages_handler(),
         create_langchain_messages_handler(),
         create_langchain_agents_handler(),
+        create_typesafe_handler(),
     ],
     tools={
         # LD documentation agent tools

packages/client/README.md

@@ -348,6 +348,28 @@ async def main():
 asyncio.run(main())
 ```
 
+#### Deferred judges
+
+With `skip_judges=True`, `invoke()` returns `judge_tasks` instead of running them. Run each with `run_judge(task, handlers)` on your own thread or process, then track the result. A judge that scores several labels in one call (a TypeSafe judge, for example) returns `metrics`: track each entry under its own `eventKey`, and do not also track `evaluation_metric_key`.
+
+```python
+from ldclient import Context
+from launchdarkly_ai_server import get_client, run_judge
+
+result = await run_judge(task, handlers)
+context = Context.from_dict(task.user_context)
+if result and result.metrics:
+    for metric in result.metrics:
+        get_client().track(
+            metric["eventKey"],
+            context,
+            {**result.track_data, "judgeConfigKey": metric["judgeConfigKey"]},
+            metric["score"],
+        )
+elif result and task.evaluation_metric_key:
+    get_client().track(task.evaluation_metric_key, context, result.track_data, result.score)
+```
+
 ### `graph(key, **options)`
 
 Runs a multi-agent workflow defined in a LaunchDarkly agent graph flag. The SDK uses a **model-driven router**: it starts at the root node, presents outgoing edges as handoff choices to the model, and follows whichever edge the model selects. The loop terminates when the model produces a final answer, a leaf is reached, a cycle is detected, or the step cap is hit.

packages/client/src/launchdarkly_ai_server/conversation.py

@@ -36,6 +36,7 @@ class _JudgeEvalCapture:
     released: bool = False
     span: Any = None
     pending_end: Callable[[], None] | None = None
+    mirrored: bool = False
 
 
 # Every tracer this SDK creates is named "@launchdarkly/ai-<package>". The processor is registered
@@ -82,7 +83,12 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None:
 
 
 def _record_evaluation(
-    span: Any, name: str, score: float, explanation: str | None = None
+    span: Any,
+    name: str,
+    score: float,
+    explanation: str | None = None,
+    *,
+    mirror_attributes: bool = True,
 ) -> None:
     """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes.
 
@@ -99,6 +105,10 @@ def _record_evaluation(
     if explanation:
         attrs["gen_ai.evaluation.explanation"] = explanation
     span.add_event("gen_ai.evaluation.result", attrs)
+    # A classifier records one event per label. Span attributes hold a single name and
+    # score, so only the first evaluation is mirrored. Later events keep their own attributes.
+    if not mirror_attributes:
+        return
     for key, value in attrs.items():
         span.set_attribute(key, value)
 
@@ -237,9 +247,18 @@ async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]:
     """
     capture = _JudgeEvalCapture(name=name)
 
-    def record(score: float, explanation: str | None = None) -> None:
+    def record(
+        score: float, explanation: str | None = None, name: str | None = None
+    ) -> None:
         if capture.span is not None:
-            _record_evaluation(capture.span, capture.name, score, explanation)
+            _record_evaluation(
+                capture.span,
+                name or capture.name,
+                score,
+                explanation,
+                mirror_attributes=not capture.mirrored,
+            )
+            capture.mirrored = True
 
     token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture))
     try:

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -17,6 +17,7 @@
     build_message_history,
     numeric_score,
     parse_judge_response,
+    typesafe_judge_entries,
 )
 from ..lifecycle import extract_variation
 from ..trajectory import (
@@ -968,10 +969,17 @@ async def _run_ld_judge_for_result(
                 "invalid_judge_output",
                 "judge handler result must be a mapping",
             )
+        judge_output = result.get("output", result.get("response"))
         try:
-            raw_score, reason = parse_judge_response(
-                result.get("output", result.get("response"))
-            )
+            entries = typesafe_judge_entries(judge_output)
+            if entries is not None:
+                # A criterion event is one score. The remaining labels are
+                # available on the online judge_results path. The reason is the
+                # question key: ingest drops a COMPLETE judge event with an
+                # empty reason, and a Jev label has no prose verdict.
+                raw_score, reason = entries[0]["score"], entries[0]["key"]
+            else:
+                raw_score, reason = parse_judge_response(judge_output)
         except ValueError as error:
             return self._criterion_error_result(
                 base, started_clock, "invalid_judge_output", str(error)

packages/client/src/launchdarkly_ai_server/judge_scoring.py

@@ -65,6 +65,55 @@ def numeric_score(score: Any) -> float | None:
     return value if isfinite(value) else None
 
 
+def typesafe_judge_entries(raw: Any) -> list[dict[str, Any]] | None:
+    """Parse a TypeSafe handler payload into question key, score, event key, and reason.
+
+    Returns ``None`` when ``raw`` is not a TypeSafe multi-result payload, so a
+    classic ``{"score", "reasoning"}`` judge stays on :func:`parse_judge_response`.
+    Raises ``ValueError`` when the payload claims to be TypeSafe but is malformed.
+    """
+    parsed: Any
+    if isinstance(raw, str):
+        parsed = parse_json_with_possible_fences(raw)
+    elif isinstance(raw, Mapping):
+        parsed = raw
+    else:
+        parsed = None
+    if not isinstance(parsed, Mapping) or parsed.get("kind") != "typesafe":
+        return None
+    results = parsed.get("results")
+    if not isinstance(results, list) or not results:
+        raise ValueError("TypeSafe judge output is missing results")
+    entries: list[dict[str, Any]] = []
+    for item in results:
+        if (
+            not isinstance(item, Mapping)
+            or not isinstance(item.get("key"), str)
+            or not item["key"]
+        ):
+            raise ValueError("TypeSafe judge result is missing a key")
+        score = numeric_score(item.get("score"))
+        if score is None:
+            raise ValueError(
+                f"TypeSafe judge result {item['key']!r} is missing a finite score"
+            )
+        event_key = item.get("eventKey")
+        if not isinstance(event_key, str) or not event_key:
+            raise ValueError(
+                f"TypeSafe judge result {item['key']!r} is missing an eventKey"
+            )
+        reason = item.get("reason")
+        entries.append(
+            {
+                "key": item["key"],
+                "score": score,
+                "eventKey": event_key,
+                "reason": reason if isinstance(reason, str) else "",
+            }
+        )
+    return entries
+
+
 def parse_judge_response(raw: Any) -> tuple[Any, str]:
     """Parse a judge model response into ``(score, reasoning)``.