Merge branch 'master' into feat/agentic-best-run-latency

#1849 added two lines to the visualization assertion message (Executed,
Stated Value Matches) at the same place this branch moved the raise behind
the shared outcome tail. Both land: the new lines close the message, then
the tail's keyword arguments follow.

1565 passed, ruff clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/AGENTS.md

@@ -156,6 +156,7 @@ Run this before guessing a `--model` string.
 | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
 | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
+| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
 A gitignored `.env` at the repo root is the normal place for these; load it with

packages/gooddata-eval/README.md

@@ -574,6 +574,55 @@ The `expected_output` rubric:
 Each criterion is scored independently by the LLM judge, so `quality_score`
 is the fraction of satisfied criteria.
 
+### Items asked in a dashboard context (`user_context`)
+
+An item can be asked the way a user asks from an open dashboard or an attached widget.
+Its `user_context` is sent verbatim as `userContext` on every chat message of the item,
+follow-up and clarification messages included — the server treats a message without one
+as a cleared context. Requires the `enableAiContextSetup` feature flag on the target
+organization — without it the server ignores the dashboard view. Applies to chat items
+only; `dashboard_summary` items carry their scope in `summary_input` instead. No
+`userContext` key is sent at all when neither the item nor, for `agentic_conversation`,
+the current turn sets one.
+
+```json
+{
+  "id": "ctx-001",
+  "dataset_name": "my_dataset_ctx",
+  "test_kind": "general_question",
+  "question": "Which dashboard am I looking at, and what does it cover?",
+  "user_context": {
+    "view": {
+      "dashboard": {
+        "id": "sales_overview",
+        "title": "Sales Overview",
+        "widgets": [
+          {"widgetType": "insight", "widgetId": "w1", "title": "Revenue by Month", "visualizationId": "revenue_by_month"}
+        ]
+      }
+    }
+  },
+  "expected_output": "Names the Sales Overview dashboard and summarizes its charts."
+}
+```
+
+The schema belongs to the AI chat API (`UserContext`: `view.dashboard`,
+`referencedObjects`, `activeObject`), so it is not validated here beyond being an object.
+In a Langfuse dataset, put it in the item `metadata` (or in the `input` object) under
+`user_context`. An object in `input` wins and `metadata` is then not read; a missing or
+`null` value in `input` falls back to `metadata`. The value used must be an object or `null`;
+anything else fails the dataset load.
+
+`agentic_conversation` items take the item's `user_context` as the context of the first
+turn. A turn in `expected_output.turns` can set its own `user_context`, which applies
+from that turn on until another turn changes it, as the attached context does in the UI:
+
+| Turn | Context sent |
+|---|---|
+| no `user_context` key | the current one, unchanged |
+| `"user_context": {...}` | this one, from this turn on |
+| `"user_context": null` | none, from this turn on |
+
 ## Supported test kinds
 
 | test_kind | What the agent must produce | Extra required |

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -19,10 +19,12 @@
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
+from gooddata_eval.core.agentic.report_skill import evaluate_agentic_report_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -44,6 +46,7 @@ class _LfKw(TypedDict, total=False):
         "agentic_metric_skill",
         "agentic_alert_skill",
         "agentic_dashboard_skill",
+        "agentic_report_skill",
         "agentic_search",
         "agentic_general_question",
         "agentic_guardrail",
@@ -94,7 +97,8 @@ class _LfKw(TypedDict, total=False):
 #
 # agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
 # any chart it authors in conversation state and writes neither until a user saves from the UI, so
-# it is a candidate for the allowlist once the dataset has runs behind it.
+# it is a candidate for the allowlist once the dataset has runs behind it. agentic_report_skill is
+# absent for the same reason: the report draft stays in conversation state until a user saves it.
 WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
 
 
@@ -176,9 +180,11 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
+            requires_execution=requires_execution_of(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
@@ -191,6 +197,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_dashboard_skill":
@@ -203,6 +210,20 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
+            **lf_kw,
+        )
+    elif kind == "agentic_report_skill":
+        return evaluate_agentic_report_skill(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo,
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
@@ -215,6 +236,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_search":
@@ -230,6 +252,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_general_question":
@@ -255,6 +278,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_kda_skill":
@@ -267,6 +291,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_what_if":
@@ -278,6 +303,7 @@ def _dispatch_agentic(
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_anomaly_detection":
@@ -290,6 +316,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -300,6 +327,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             fixture=ConversationFixture.model_validate(fixture_data),
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     else:

packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py

@@ -53,6 +53,14 @@
     evaluate_agentic_metric_skill,
     run_agentic_metric_skill,
 )
+from gooddata_eval.core.agentic.report_skill import (
+    AgenticReportSummary,
+    ReportEvaluation,
+    ReportRunResult,
+    ReportSkillAssertionError,
+    evaluate_agentic_report_skill,
+    run_agentic_report_skill,
+)
 from gooddata_eval.core.agentic.search_tool import (
     AgenticSearchSummary,
     SearchResult,
@@ -75,6 +83,7 @@
     "AgenticGuardrailSummary",
     "AgenticKdaSummary",
     "AgenticMetricSummary",
+    "AgenticReportSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
     "AlertEvaluation",
@@ -95,6 +104,9 @@
     "KdaSkillAssertionError",
     "MetricRunResult",
     "MetricSkillAssertionError",
+    "ReportEvaluation",
+    "ReportRunResult",
+    "ReportSkillAssertionError",
     "RunResult",
     "SearchResult",
     "SearchToolAssertionError",
@@ -108,6 +120,7 @@
     "evaluate_agentic_guardrail",
     "evaluate_agentic_kda_skill",
     "evaluate_agentic_metric_skill",
+    "evaluate_agentic_report_skill",
     "evaluate_agentic_search_tool",
     "evaluate_agentic_visualization",
     "run_agentic_alert_skill",
@@ -117,6 +130,7 @@
     "run_agentic_guardrail",
     "run_agentic_kda_skill",
     "run_agentic_metric_skill",
+    "run_agentic_report_skill",
     "run_agentic_search_tool",
     "run_agentic_visualization",
 ]

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -663,12 +663,18 @@ def run_agentic_alert_skill(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     sdk = GoodDataSdk.create(host, token)
 
@@ -864,6 +870,7 @@ def evaluate_agentic_alert_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
@@ -887,6 +894,7 @@ def evaluate_agentic_alert_skill(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py

@@ -354,6 +354,7 @@ def run_agentic_anomaly_detection(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticAnomalySummary:
     """Run the anomaly-detection agentic evaluation K times and return a summary.
 
@@ -366,7 +367,12 @@ def run_agentic_anomaly_detection(
         raise ValueError(f"k must be >= 1, got {k}")
     run_results: list[AnomalyRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     def _run_once(conv_id: str) -> AnomalyRunResult:
@@ -545,6 +551,7 @@ def evaluate_agentic_anomaly_detection(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
 ) -> AgenticEvalOutcome:
     """Run anomaly-detection evaluation, log to Langfuse, and raise on failure."""
     langfuse, window_start = open_trace_window(langfuse)
@@ -559,6 +566,7 @@ def evaluate_agentic_anomaly_detection(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id: