|
25 | 25 | from dataclasses import dataclass, field |
26 | 26 | from typing import Any |
27 | 27 |
|
| 28 | +from gooddata_eval.core.agentic._gate import ( |
| 29 | + DEFAULT_GATE, |
| 30 | + EvalGate, |
| 31 | + gate_failure_note, |
| 32 | + gate_passed, |
| 33 | + log_gate_scores, |
| 34 | + stamp_gate_metadata, |
| 35 | +) |
28 | 36 | from gooddata_eval.core.agentic._trace_linker import ( |
29 | 37 | RunIdentity, |
30 | 38 | RunTraceContext, |
@@ -469,6 +477,7 @@ def evaluate_agentic_what_if( |
469 | 477 | k: int = _DEFAULT_K, |
470 | 478 | max_iterations: int = _DEFAULT_MAX_ITERATIONS, |
471 | 479 | initial_conversation_id: str | None = None, |
| 480 | + gate: EvalGate = DEFAULT_GATE, |
472 | 481 | agent_id: str | None = None, |
473 | 482 | langfuse: object | None = None, |
474 | 483 | dataset_item_id: str = "", |
@@ -501,6 +510,8 @@ def evaluate_agentic_what_if( |
501 | 510 | window_end = utc_now() |
502 | 511 |
|
503 | 512 | def _write_scores(ctx: RunTraceContext) -> None: |
| 513 | + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) |
| 514 | + |
504 | 515 | for run_idx, run in enumerate(summary.run_results): |
505 | 516 | pt = ctx.trace(run.conversation_id) |
506 | 517 | ev = run.evaluation |
@@ -528,6 +539,7 @@ def _write_scores(ctx: RunTraceContext) -> None: |
528 | 539 | with ctx.observe(pt, run_idx) as tid: |
529 | 540 | for score_name, value in strict_checks.items(): |
530 | 541 | ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") |
| 542 | + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) |
531 | 543 | ctx.quality( |
532 | 544 | tid, |
533 | 545 | strict_checks=strict_checks, |
@@ -572,9 +584,10 @@ def _write_scores(ctx: RunTraceContext) -> None: |
572 | 584 | detail = _detail(best) |
573 | 585 | runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) |
574 | 586 |
|
575 | | - if not summary.pass_at_k: |
| 587 | + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): |
| 588 | + gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) |
576 | 589 | message = ( |
577 | | - f"What-if assertion failed. strict_pass={ev.strict_pass} " |
| 590 | + f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " |
578 | 591 | f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " |
579 | 592 | f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, " |
580 | 593 | f"maql_correct={ev.maql_correct}, scenario_count_correct={ev.scenario_count_correct}, " |
|
0 commit comments