Merge pull request #1852 from gooddata/jt/eval-trace-labels

feat(gooddata-eval): label gen-ai traces through W3C baggage

Author: tychtjan

packages/gooddata-eval/AGENTS.md

@@ -156,6 +156,7 @@ Run this before guessing a `--model` string.
 | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
 | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
+| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
 A gitignored `.env` at the repo root is the normal place for these; load it with

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -13,12 +13,15 @@
 protocol, not on this class.
 """
 
+import functools
 import json
 import logging
 import os
+import re
 import time
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, TypeVar
+from urllib.parse import quote
 
 import httpx
 
@@ -135,6 +138,45 @@ def _float_env(name: str, default: float) -> float:
 # 4-turn conversation run to 4x the budget, which is not what a user would sit through.
 _ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0)
 
+# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as
+# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*`
+# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing
+# generations, retries and the title trace included -- carries them from creation. Langfuse v4
+# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be
+# a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx
+# refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest.
+_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
+_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
+
+
+@functools.cache
+def _warn_skipped_trace_label(key: str) -> None:
+    """Warn once per key per process: every ChatClient re-reads the labels."""
+    _log.warning("%s: skipping label with key %r, keys must match [A-Za-z0-9_-]+", _TRACE_LABELS_ENV, key)
+
+
+def _trace_baggage() -> dict[str, str]:
+    """The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none.
+
+    Each label becomes trace metadata, and ``model_version`` -- the combo the report groups
+    on -- also becomes the observation ``version``, the dimension Langfuse groups cost by.
+    Values are percent-encoded: a space or comma would otherwise end the baggage entry.
+    """
+    labels: dict[str, str] = {}
+    for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","):
+        key, sep, value = pair.partition("=")
+        key = key.strip()
+        if not sep or not key:
+            continue
+        if _TRACE_LABEL_KEY.fullmatch(key):
+            labels[key] = value.strip()
+        else:
+            _warn_skipped_trace_label(key)
+    entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()]
+    if labels.get("model_version"):
+        entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}")
+    return {"baggage": ",".join(entries)} if entries else {}
+
 
 def set_default_turn_timeout(seconds: float | None) -> None:
     """Set the per-turn budget every ChatClient built afterwards inherits.
@@ -470,7 +512,7 @@ def __init__(
         the ``enableAiContextSetup`` feature flag is on for the organization.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
-        self._auth = {"Authorization": f"Bearer {token}"}
+        self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
         # 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check
         # fires between events, so a turn that goes silent needs the transport to give up too.
         budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s

packages/gooddata-eval/tests/test_sse_client.py

@@ -1,5 +1,8 @@
 # (C) 2026 GoodData Corporation
 import json
+import logging
+from collections.abc import Callable
+from urllib.parse import unquote
 
 import httpx
 import pytest
@@ -1053,3 +1056,77 @@ def handler(request):
     )
     client.ask(item)
     assert captured["body"]["userContext"] == _ATTACHMENT
+
+
+def _baggage_of(requests: list[httpx.Request]) -> list[dict[str, str] | None]:
+    """Each request's `baggage` header as {key: decoded value}, or None when absent."""
+    out: list[dict[str, str] | None] = []
+    for r in requests:
+        raw = r.headers.get("baggage")
+        out.append(None if raw is None else {k: unquote(v) for k, v in (e.split("=", 1) for e in raw.split(","))})
+    return out
+
+
+def _record_requests(requests: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]:
+    def handler(request: httpx.Request) -> httpx.Response:
+        requests.append(request)
+        if request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "c1"})
+        return httpx.Response(200, content=_OK_SSE)
+
+    return handler
+
+
+def test_trace_labels_ride_every_request_as_langfuse_baggage(monkeypatch: pytest.MonkeyPatch) -> None:
+    """gen-ai's Langfuse span processor copies `langfuse_*` baggage onto every span it starts, so
+    the labels reach the root generation and its cost-bearing children at creation."""
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "model_version=gpt-5.2 openai,github_run_id=42")
+    requests: list[httpx.Request] = []
+    client = _client_with_handler(_record_requests(requests))
+
+    client.send_message(client.create_conversation(), "q")
+
+    expected = {
+        "langfuse_version": "gpt-5.2 openai",
+        "langfuse_metadata_model_version": "gpt-5.2 openai",
+        "langfuse_metadata_github_run_id": "42",
+    }
+    assert _baggage_of(requests) == [expected, expected]
+    # Encoded, not raw: a space or comma in a value would otherwise split or end the entry.
+    assert " " not in requests[0].headers["baggage"]
+
+
+def test_trace_labels_without_a_model_version_set_no_langfuse_version(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42")
+    requests: list[httpx.Request] = []
+    _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}]
+
+
+@pytest.mark.parametrize("key", ["modèle", "run id", "c++"])
+def test_trace_label_with_a_non_token_key_is_skipped_with_one_warning(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, key: str
+) -> None:
+    """httpx refuses a non-ASCII header, gen-ai's OTel propagator drops a key with a space, and
+    it reads `+` as a space; each would otherwise fail the run or mislabel the trace."""
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", f"github_run_id=42,{key}=1")
+    requests: list[httpx.Request] = []
+
+    for _ in range(2):
+        _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] * 2
+    assert [r.levelno for r in caplog.records if repr(key) in r.getMessage()] == [logging.WARNING]
+
+
+@pytest.mark.parametrize("raw", [None, "", "no-equals-sign", " , "])
+def test_no_trace_labels_send_no_baggage(monkeypatch: pytest.MonkeyPatch, raw: str | None) -> None:
+    if raw is None:
+        monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False)
+    else:
+        monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", raw)
+    requests: list[httpx.Request] = []
+    _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [None]