Skip to content

Commit 13f1555

Browse files
Merge pull request #1856 from gooddata/snapshot-master-ab84a23b-to-rel/dev
[bot] Merge master/ab84a23b into rel/dev
2 parents 43f654a + ab84a23 commit 13f1555

3 files changed

Lines changed: 65 additions & 2 deletions

File tree

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ def _dispatch_agentic(
302302
question=item.question,
303303
expected_output=eo if isinstance(eo, dict) else {},
304304
k=k,
305+
gate=gate,
305306
agent_id=agent_id,
306307
user_context=item.user_context,
307308
**lf_kw,

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@
2525
from dataclasses import dataclass, field
2626
from typing import Any
2727

28+
from gooddata_eval.core.agentic._gate import (
29+
DEFAULT_GATE,
30+
EvalGate,
31+
gate_failure_note,
32+
gate_passed,
33+
log_gate_scores,
34+
stamp_gate_metadata,
35+
)
2836
from gooddata_eval.core.agentic._trace_linker import (
2937
RunIdentity,
3038
RunTraceContext,
@@ -469,6 +477,7 @@ def evaluate_agentic_what_if(
469477
k: int = _DEFAULT_K,
470478
max_iterations: int = _DEFAULT_MAX_ITERATIONS,
471479
initial_conversation_id: str | None = None,
480+
gate: EvalGate = DEFAULT_GATE,
472481
agent_id: str | None = None,
473482
langfuse: object | None = None,
474483
dataset_item_id: str = "",
@@ -501,6 +510,8 @@ def evaluate_agentic_what_if(
501510
window_end = utc_now()
502511

503512
def _write_scores(ctx: RunTraceContext) -> None:
513+
stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate)
514+
504515
for run_idx, run in enumerate(summary.run_results):
505516
pt = ctx.trace(run.conversation_id)
506517
ev = run.evaluation
@@ -528,6 +539,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
528539
with ctx.observe(pt, run_idx) as tid:
529540
for score_name, value in strict_checks.items():
530541
ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
542+
log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
531543
ctx.quality(
532544
tid,
533545
strict_checks=strict_checks,
@@ -572,9 +584,10 @@ def _write_scores(ctx: RunTraceContext) -> None:
572584
detail = _detail(best)
573585
runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
574586

575-
if not summary.pass_at_k:
587+
if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
588+
gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results))
576589
message = (
577-
f"What-if assertion failed. strict_pass={ev.strict_pass} "
590+
f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} "
578591
f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, "
579592
f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, "
580593
f"maql_correct={ev.maql_correct}, scenario_count_correct={ev.scenario_count_correct}, "

‎packages/gooddata-eval/tests/test_agentic_what_if.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,8 @@ def __init__(self):
362362
self.scores: dict[str, float] = {}
363363
# Not named `quality`: the method below would overwrite itself on first call.
364364
self.quality_call: dict = {}
365+
# stamp_gate_metadata writes the gate and K here, the way every other kind does.
366+
self.run_metadata: dict = {}
365367

366368
def trace(self, _conversation_id):
367369
return None
@@ -437,3 +439,50 @@ def test_cost_is_reported_even_when_the_tool_was_never_reached():
437439
ev.triggered hid that and understated what the item cost."""
438440
ctx = _scored(_EXPECTED, calls=[])
439441
assert "cost_usd" in ctx.quality_call
442+
443+
444+
# ── the gate decides the verdict, not pass@K alone ──────────────────────────
445+
#
446+
# run_agentic_what_if computed pass^K and the evaluator never read it, so `--gate power`
447+
# on a flaky item passed on the strength of one good run out of K -- the precise case the
448+
# gate exists to catch.
449+
450+
451+
def _two_runs(gate=None):
452+
"""One passing run then one failing one, which pass@K clears and pass^K must not."""
453+
client = MagicMock()
454+
client.create_conversation.side_effect = ["conv-1", "conv-2"]
455+
client.send_message.side_effect = [_chat(_pair()), *[_chat([])] * 8]
456+
kwargs = {"gate": gate} if gate is not None else {}
457+
with (
458+
patch(f"{_MODULE}.ChatClient", return_value=client),
459+
patch(f"{_MODULE}.generate_simulated_what_if_response", return_value="adjust revenue by 10%"),
460+
):
461+
return evaluate_agentic_what_if(
462+
host="http://h",
463+
token="tok",
464+
workspace_id="ws1",
465+
question="What if revenue rose 10%?",
466+
expected_output=_EXPECTED,
467+
k=2,
468+
**kwargs,
469+
)
470+
471+
472+
def test_power_gate_fails_an_item_where_only_some_runs_passed():
473+
with pytest.raises(WhatIfAssertionError) as excinfo:
474+
_two_runs(gate="power")
475+
assert "pass^2" in str(excinfo.value)
476+
assert excinfo.value.runs_passed == 1
477+
assert excinfo.value.runs_effective == 2
478+
479+
480+
def test_any_gate_still_passes_the_same_item():
481+
"""The default is unchanged: one passing run out of K clears pass@K."""
482+
outcome = _two_runs(gate="any")
483+
assert outcome.runs_passed == 1
484+
assert outcome.runs_effective == 2
485+
486+
487+
def test_the_default_gate_is_pass_at_k():
488+
assert _two_runs().runs_passed == 1

0 commit comments

Comments
 (0)