Skip to content

Commit 1054889

Browse files
Jan Brandejsclaude
andcommitted
Merge upstream master into jb/eval-report-latency-percentiles
Master moved the report evaluator to document_skill.py (#1862), which the branch's per-run latency changes could not follow on their own. - core/agentic/report_skill.py: take master's deprecation shim and re-apply the branch's run_latency_s / run_latencies_s additions to document_skill.py, where the evaluator now lives. - tests/test_agentic_document_skill.py (renamed from test_agentic_report_skill.py): take master's file and re-apply the four run_latencies_s assertions under the new names. - tests/test_agentic_runner.py: both sides appended tests at the end of the file; keep both. - core/interaction.py: master renamed the agentic_report_skill kind to agentic_document_skill, so the multi-turn set follows. The guard test that requires every agentic kind to be classified caught this; the README list is updated to match. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2 parents 60dad1e + f8e960a commit 1054889

21 files changed

Lines changed: 2957 additions & 1145 deletions

‎packages/gooddata-eval/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ their eval kind, and states latency for each:
317317
`agentic_general_question`, `agentic_guardrail` and `agentic_search`.
318318
- **Multi-turn** — kinds that drive a simulated user or fixture turns: `agentic_visualization` (and
319319
`vis_agentic`), `agentic_metric_skill`, `agentic_alert_skill`, `agentic_dashboard_skill`,
320-
`agentic_report_skill`, `agentic_kda_skill`, `agentic_anomaly_detection`, `agentic_what_if` and
320+
`agentic_document_skill`, `agentic_kda_skill`, `agentic_anomaly_detection`, `agentic_what_if` and
321321
`agentic_conversation`.
322322

323323
```json

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@
1515
from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
1616
from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
1717
from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
18+
from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill
1819
from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
1920
from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
2021
from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
2122
from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
22-
from gooddata_eval.core.agentic.report_skill import evaluate_agentic_report_skill
2323
from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
2424
from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
2525
from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
@@ -39,6 +39,7 @@ class _LfKw(TypedDict, total=False):
3939
model_version_override: str | None
4040
reasoning_effort: ReasoningEffort | None
4141
submit_trace_link: SubmitTraceLink
42+
run_metadata_extra: dict[str, Any] | None
4243

4344

4445
AGENTIC_TEST_KINDS = frozenset(
@@ -48,7 +49,7 @@ class _LfKw(TypedDict, total=False):
4849
"agentic_metric_skill",
4950
"agentic_alert_skill",
5051
"agentic_dashboard_skill",
51-
"agentic_report_skill",
52+
"agentic_document_skill",
5253
"agentic_search",
5354
"agentic_general_question",
5455
"agentic_guardrail",
@@ -99,8 +100,8 @@ class _LfKw(TypedDict, total=False):
99100
#
100101
# agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
101102
# any chart it authors in conversation state and writes neither until a user saves from the UI, so
102-
# it is a candidate for the allowlist once the dataset has runs behind it. agentic_report_skill is
103-
# absent for the same reason: the report draft stays in conversation state until a user saves it.
103+
# it is a candidate for the allowlist once the dataset has runs behind it. agentic_document_skill is
104+
# absent for the same reason: the document draft stays in conversation state until a user saves it.
104105
WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
105106

106107

@@ -153,11 +154,13 @@ def _dispatch_agentic(
153154
agent_id: str | None = None,
154155
submit_trace_link: SubmitTraceLink = run_trace_link_inline,
155156
gate: EvalGate = DEFAULT_GATE,
157+
run_metadata_extra: dict[str, Any] | None = None,
156158
) -> AgenticEvalOutcome:
157159
"""Call the appropriate evaluate_agentic_* function for the item's test_kind.
158160
159161
`gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K
160162
to gate over; the CLI refuses --gate power for a dataset containing one.
163+
`run_metadata_extra` reaches every kind unchanged.
161164
162165
Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
163166
conversation_id, response_id, detail) on success and attaches the same four attributes
@@ -173,6 +176,7 @@ def _dispatch_agentic(
173176
"model_version_override": model_version_override,
174177
"reasoning_effort": reasoning_effort,
175178
"submit_trace_link": submit_trace_link,
179+
"run_metadata_extra": run_metadata_extra,
176180
}
177181

178182
if kind in ("vis_agentic", "agentic_visualization"):
@@ -215,8 +219,8 @@ def _dispatch_agentic(
215219
user_context=item.user_context,
216220
**lf_kw,
217221
)
218-
elif kind == "agentic_report_skill":
219-
return evaluate_agentic_report_skill(
222+
elif kind == "agentic_document_skill":
223+
return evaluate_agentic_document_skill(
220224
host=host,
221225
token=token,
222226
workspace_id=workspace_id,
@@ -413,6 +417,7 @@ def run_agentic_items(
413417
agent_id: str | None = None,
414418
concurrency: int = 1,
415419
gate: EvalGate = DEFAULT_GATE,
420+
tags: tuple[str, ...] = (),
416421
) -> EvalReport:
417422
"""Run agentic items through evaluate_agentic_* and return an EvalReport.
418423
@@ -421,9 +426,14 @@ def run_agentic_items(
421426
from the parallel ones -- a metric being created and dropped mid-run would otherwise be
422427
visible to a catalog-reading item running alongside it.
423428
429+
``tags`` are added to every Langfuse trace gd-eval writes for these items, after the
430+
automatic ones. They reach Langfuse whenever trace linking runs, with or without
431+
``use_langfuse``.
432+
424433
Results are collected in dataset order regardless of completion order.
425434
"""
426435
langfuse = make_langfuse_client() if use_langfuse else None
436+
run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None
427437

428438
report = EvalReport(model=model_version, gate=normalize_gate(gate))
429439
total = len(items)
@@ -469,6 +479,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
469479
agent_id,
470480
submit_trace_link=linker.submit,
471481
gate=gate,
482+
run_metadata_extra=run_metadata_extra,
472483
)
473484
if isinstance(outcome, AgenticEvalOutcome):
474485
reasoning_steps = outcome.reasoning_steps

‎packages/gooddata-eval/src/gooddata_eval/cli/main.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,17 @@ def _build_parser() -> argparse.ArgumentParser:
183183
action="store_true",
184184
help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
185185
)
186+
run.add_argument(
187+
"--tag",
188+
dest="tags",
189+
action="append",
190+
default=[],
191+
type=_parse_tag,
192+
metavar="TAG",
193+
help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. "
194+
"Repeat for several tags. Applied wherever traces are written: single-turn kinds only "
195+
"with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.",
196+
)
186197
run.add_argument(
187198
"--agent-id",
188199
dest="agent_id",
@@ -377,6 +388,14 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
377388
return None, val
378389

379390

391+
def _parse_tag(val: str) -> str:
392+
"""Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag."""
393+
tag = val.strip()
394+
if not tag:
395+
raise argparse.ArgumentTypeError("tag must not be blank")
396+
return tag
397+
398+
380399
def _make_progress_callbacks(console: Console):
381400
"""Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
382401
@@ -549,6 +568,7 @@ def _run(config: RunConfig) -> int:
549568
model_id=resolved.model_id,
550569
provider_type=resolved.provider_type,
551570
reasoning_effort=config.reasoning_effort,
571+
tags=config.tags,
552572
)
553573

554574
def on_langfuse_item_done(
@@ -574,6 +594,7 @@ def on_langfuse_item_done(
574594
model_version=resolved.model_id,
575595
reasoning_effort=config.reasoning_effort,
576596
use_langfuse=config.log_to_langfuse,
597+
tags=config.tags,
577598
run_ts=run_ts,
578599
on_item_start=on_item_start,
579600
on_item_done=on_item_done,
@@ -728,6 +749,7 @@ def main(argv: list[str] | None = None) -> int:
728749
agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
729750
turn_timeout_s=args.turn_timeout,
730751
item_timeout_s=args.item_timeout,
752+
tags=tuple(dict.fromkeys(args.tags)),
731753
)
732754
return _run(config)
733755
except (

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@
2424
evaluate_agentic_dashboard_skill,
2525
run_agentic_dashboard_skill,
2626
)
27+
from gooddata_eval.core.agentic.document_skill import (
28+
AgenticDocumentSummary,
29+
DocumentEvaluation,
30+
DocumentRunResult,
31+
DocumentSkillAssertionError,
32+
evaluate_agentic_document_skill,
33+
run_agentic_document_skill,
34+
)
2735
from gooddata_eval.core.agentic.general_question import (
2836
AgenticGeneralQuestionSummary,
2937
GeneralQuestionAssertionError,
@@ -53,14 +61,6 @@
5361
evaluate_agentic_metric_skill,
5462
run_agentic_metric_skill,
5563
)
56-
from gooddata_eval.core.agentic.report_skill import (
57-
AgenticReportSummary,
58-
ReportEvaluation,
59-
ReportRunResult,
60-
ReportSkillAssertionError,
61-
evaluate_agentic_report_skill,
62-
run_agentic_report_skill,
63-
)
6464
from gooddata_eval.core.agentic.search_tool import (
6565
AgenticSearchSummary,
6666
SearchResult,
@@ -76,13 +76,22 @@
7676
run_agentic_visualization,
7777
)
7878

79+
# Deprecated names of the document skill, kept for existing callers.
80+
AgenticReportSummary = AgenticDocumentSummary
81+
ReportEvaluation = DocumentEvaluation
82+
ReportRunResult = DocumentRunResult
83+
ReportSkillAssertionError = DocumentSkillAssertionError
84+
evaluate_agentic_report_skill = evaluate_agentic_document_skill
85+
run_agentic_report_skill = run_agentic_document_skill
86+
7987
__all__ = [
8088
"AgenticAlertSummary",
8189
"AgenticDashboardSummary",
8290
"AgenticGeneralQuestionSummary",
8391
"AgenticGuardrailSummary",
8492
"AgenticKdaSummary",
8593
"AgenticMetricSummary",
94+
"AgenticDocumentSummary",
8695
"AgenticReportSummary",
8796
"AgenticSearchSummary",
8897
"AgenticRunSummary",
@@ -95,6 +104,9 @@
95104
"DashboardEvaluation",
96105
"DashboardRunResult",
97106
"DashboardSkillAssertionError",
107+
"DocumentEvaluation",
108+
"DocumentRunResult",
109+
"DocumentSkillAssertionError",
98110
"GeneralQuestionAssertionError",
99111
"GeneralQuestionResult",
100112
"GuardrailAssertionError",
@@ -116,6 +128,7 @@
116128
"evaluate_agentic_alert_skill",
117129
"evaluate_agentic_conversation",
118130
"evaluate_agentic_dashboard_skill",
131+
"evaluate_agentic_document_skill",
119132
"evaluate_agentic_general_question",
120133
"evaluate_agentic_guardrail",
121134
"evaluate_agentic_kda_skill",
@@ -126,6 +139,7 @@
126139
"run_agentic_alert_skill",
127140
"run_agentic_conversation",
128141
"run_agentic_dashboard_skill",
142+
"run_agentic_document_skill",
129143
"run_agentic_general_question",
130144
"run_agentic_guardrail",
131145
"run_agentic_kda_skill",

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
ExperimentRun,
2222
ScoreTarget,
2323
build_experiment_root_span,
24+
trace_tags,
2425
)
2526

2627
# Part of this module's public surface: external callers import both names from here.
@@ -302,7 +303,10 @@ def _experiment_root_span(
302303
"""gd-eval's own root span for one (dataset item, run) -- the whole experiment item."""
303304
start, end = _span_window(trace, window)
304305
session_id = conversation_id or getattr(trace, "session_id", None)
305-
tags = tuple(tag for tag in ("gd-eval", run_metadata.get("testing_framework")) if tag)
306+
custom_tags = run_metadata.get("tags") or ()
307+
if isinstance(custom_tags, str):
308+
custom_tags = (custom_tags,)
309+
tags = trace_tags("gd-eval", run_metadata.get("testing_framework"), *custom_tags)
306310
# One predicate for both the name and the input, so a falsy question cannot name the span
307311
# after the item while the input still says "question".
308312
has_input = item_input is not None
@@ -514,7 +518,8 @@ def build_run_context(
514518
model_version_override: Explicit model-version tag; when set it wins over the
515519
workspace lookup.
516520
run_metadata_extra: Optional extra key/values added to the dataset-run metadata
517-
(e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
521+
(e.g. a testing-framework tag or a CI run id for scoping). A `tags` list is
522+
also added to the trace tags, after the automatic ones. Default None keeps
518523
behavior unchanged. The SDK-derived model_version is applied last and cannot
519524
be overwritten by this dict.
520525
reasoning_effort: Effort the run requested, stamped into both the run name and

0 commit comments

Comments
 (0)