chore(gooddata-eval): reconcile the five remaining open eval PRs

Rebuilt on master now that #1789 (loop exit_reason) and #1801 (anomaly
detection) have landed. Carries #1797, #1798, #1816, #1831, #1839.

Three reconciliations the individual PRs cannot make on their own:

- Dispatch registration in cli/agentic_runner.py is additive across the
  evaluator PRs; each pair conflicts and each resolution is the union.
- #1789's exit_reason/turns_used are now on master in the same item-detail
  dicts #1816 moves behind a per-run builder. Both land: the per-run fields
  go into _run_detail, and max_iterations stays at the item level because it
  is the same for every run.
- #1816's structural guard requires every multi-run evaluator to call
  build_failed_runs. dashboard_summary, forecasting and anomaly_detection
  postdate it and had no attachment point, so each grew one. anomaly
  detection is now ON MASTER without it, so this gap stops being a merge
  artefact the day #1816 lands.

Also supplies summary_input to #1816's failed-runs report test, which
otherwise fails a dashboard-summary item on a missing fixture field before
its evaluator is reached.

1524 passed, 1 skipped. ruff clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py

@@ -31,6 +31,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -611,6 +612,13 @@ def _write_scores(ctx: RunTraceContext) -> None:
     best = summary.best
     ev = best.evaluation
     detail = _detail(best)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.evaluation.strict_pass,
+        detail=_detail,
+    )
     runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
@@ -630,6 +638,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = len(summary.run_results)
+        exc.failed_runs = failed_runs
         raise exc
 
     return AgenticEvalOutcome(
@@ -639,4 +648,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py

@@ -30,6 +30,7 @@
 import httpx
 from gooddata_sdk import GoodDataSdk
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._trace_linker import (
     RunIdentity,
     RunTraceContext,
@@ -406,8 +407,14 @@ def run_agentic_dashboard_summary(
     )
 
 
-def _detail(summary: AgenticDashboardSummarySummary) -> dict:
-    best = summary.best
+def _run_detail(run: DashboardSummaryRunResult) -> dict:
+    """The diagnostic fields for ONE run, shared by the best run and every failing one.
+
+    Extracted so a failing run is described by exactly the same keys as the winning run --
+    for this kind that means the criteria breakdown and how much of the dashboard the run
+    managed to execute, which is what tells a bad summary apart from a half-loaded one.
+    """
+    best = run
     detail = dict(best.evaluation.detail)
     # How much of the dashboard the summary actually covered. A summary graded against a
     # rubric that mentions a widget which never executed fails for a reason that has
@@ -420,6 +427,10 @@ def _detail(summary: AgenticDashboardSummarySummary) -> dict:
     return detail
 
 
+def _detail(summary: AgenticDashboardSummarySummary) -> dict:
+    return _run_detail(summary.best)
+
+
 class DashboardSummaryAssertionError(AgenticAssertionError):
     """Raised when an agentic dashboard-summary evaluation fails."""
 
@@ -517,6 +528,9 @@ def _write_scores(ctx: RunTraceContext) -> None:
     for run in summary.run_results:
         timings = timings + run.timings
     runs_passed = sum(1 for r in summary.run_results if r.passed)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.passed, detail=_run_detail)
 
     if not summary.pass_at_k:
         error = DashboardSummaryAssertionError(
@@ -531,6 +545,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         error.timings = timings
         error.runs_passed = runs_passed
         error.runs_effective = len(summary.run_results)
+        error.failed_runs = failed_runs
         raise error
 
     return AgenticEvalOutcome(
@@ -541,4 +556,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         timings=timings,
         runs_passed=runs_passed,
         runs_effective=len(summary.run_results),
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py

@@ -22,6 +22,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -588,6 +589,13 @@ def _write_scores(ctx: RunTraceContext) -> None:
     best = summary.best
     ev = best.evaluation
     detail = _detail(best)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.evaluation.strict_pass,
+        detail=_detail,
+    )
     runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
@@ -608,6 +616,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = len(summary.run_results)
+        exc.failed_runs = failed_runs
         raise exc
 
     return AgenticEvalOutcome(
@@ -617,4 +626,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/tests/test_agentic_runner.py

@@ -849,7 +849,17 @@ def test_failed_runs_reaches_the_report_for_every_kind(kind, expected_output, ta
     field empty on every item (confirmed live: 135 agentic_guardrail results, all with
     `failed_runs: []`, including 16 items that passed 1 of 3 runs). A kind whose evaluator
     stops attaching them fails here rather than quietly reporting nothing."""
-    item = DatasetItem(id="q1", dataset_name="ds", test_kind=kind, question="q", expected_output=expected_output)
+    item = DatasetItem(
+        id="q1",
+        dataset_name="ds",
+        test_kind=kind,
+        question="q",
+        expected_output=expected_output,
+        # Dispatch rejects a dashboard-summary item without it before the evaluator is ever
+        # called, so the item would fail for a missing fixture field rather than for the
+        # thing under test.
+        summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None,
+    )
     canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, failed_runs=[_A_FAILED_RUN])
     with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned):
         report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01")