From edcf045ca3503b29794b15334f33712dcb24203e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:50:29 -0700 Subject: [PATCH 1/7] fix(gates): nothing measured is not a pass A gate fed an empty or all-errored result read it as green. Each one now exits 1 with an "incomplete" line and the counts: - repo tasks check / repo run: an export with no task is refused (repo run crashed with IndexError on one); tasks check exits 1 when no task passed the self-check, even with --allow-unsupported; repo run exits 1 when no cell has a result, and counts unsupported cells in its summary line. - eval judge-wording: a condition with no scored pair is NOT MEASURED (errored-pair counts printed), not "No rewrite moves any variant"; --strict exits 1 on it. - eval cross-judge: an arm no voting measure compared is NOT MEASURED, not NO DETECTABLE EFFECT; --strict exits 1 on it or on no arm. - run --strict: exits 1 when no document was measured. Partial errors change no threshold. Signed-off-by: Nikolay Petrov --- ...21_120000_napetrov_outcome-completeness.md | 3 + dokimi/docs/benchmarking-and-comparison.md | 8 ++- dokimi/docs/repo-evaluation-contract.md | 9 ++- dokimi/dokimi/commands/judge_reliability.py | 22 ++++-- dokimi/dokimi/commands/repo_run.py | 10 ++- dokimi/dokimi/commands/run.py | 5 ++ dokimi/dokimi/eval/cross_judge.py | 5 ++ dokimi/dokimi/eval/judge_wording.py | 17 ++++- dokimi/dokimi/repos/workspace.py | 6 +- dokimi/tests/test_gate_call_sites.py | 17 +++++ dokimi/tests/test_judge_reliability.py | 72 +++++++++++++++++++ dokimi/tests/test_repo_host.py | 48 ++++++++++++- dokimi/tests/test_repo_run.py | 2 +- 13 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_outcome-completeness.md diff --git a/dokimi/changelog.d/20261021_120000_napetrov_outcome-completeness.md b/dokimi/changelog.d/20261021_120000_napetrov_outcome-completeness.md new file mode 100644 index 00000000..d4276509 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_outcome-completeness.md @@ -0,0 +1,3 @@ +### Fixed + +- **Nothing measured is not a pass** — gates that read an empty or all-errored result as green now exit 1 with an `incomplete` line and the counts. `repo tasks check` and `repo run` refuse an export with no task (`repo run` crashed with an IndexError on one); `repo tasks check` exits 1 when no task passed the self-check, even with `--allow-unsupported`; `repo run` exits 1 when no cell of the batch has a result, and its summary line now counts unsupported cells. `eval judge-wording` reports a condition with no scored pair as NOT MEASURED (with errored-pair counts) instead of "No rewrite moves any variant", and `--strict` exits 1 on it. `eval cross-judge` labels an arm `NOT MEASURED` when no voting measure compared a single question, and `--strict` exits 1 on that or when no arm was compared. `run --strict` exits 1 when no document was measured. Partial errors change no threshold. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index d2b3a74f..6021aa4a 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -445,7 +445,9 @@ to the baseline's; `--escalate-model` is the same cheap-then-expensive cascade. under several judges (absolute, and with `--grounded` against each question's `expected_topics`), takes `eval pairwise` and `eval checklist` reports as more measures, and labels each arm `CONSISTENT ±`, `LEANING ±`, `JUDGE-DEPENDENT` or -`NO DETECTABLE EFFECT`, on Holm-adjusted p over the measures that vote. With +`NO DETECTABLE EFFECT`, on Holm-adjusted p over the measures that vote, or +`NOT MEASURED` when no voting measure compared a single question (every call +errored, or every judge was gated out); `--strict` exits 1 on that too. With `--validity` reports, a judge that failed its gate is shown but does not vote (its grounded and checklist measures included, in any context); `--require-validated` leaves only judges that passed. The stored measure is @@ -482,7 +484,9 @@ python cli.py eval cross-judge --answers run.json --questions golden.json --grou under meaning-preserving rewrites of it and flags a degradation FRAGILE when a rewrite moves it on the same pairs: a paired sign test against the original prompt, Holm-adjusted over every rewrite and variant. Re-draws of the original -are shown as a noise floor but do not decide the flag. Use at least 30 bases; at +are shown as a noise floor but do not decide the flag. A condition with no +scored pair (every call errored) is reported NOT MEASURED, not as "no rewrite +moves anything", and `--strict` exits 1 on it. Use at least 30 bases; at 20 bases only a large wording effect is detectable. `eval checklist` asks, per expected topic and in both polarities, whether the diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d69d2163..ccd8bfff 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -449,6 +449,13 @@ apart, neither failed nor missing. If the probe itself fails, a task with `host` requirements is not run either, and `repo run` exits 1 when one of its cells is left unrun; a cell that already has a `result.json` keeps it. +Nothing checked is not a pass. Both commands refuse an export with no task, and +exit 1 as `incomplete` when nothing was measured: `tasks check` when no task +passed the self-check (every one unsupported, even with `--allow-unsupported`), +`repo run` when no cell of the batch has a result (every one unsupported). The +`repo run` summary line counts unsupported cells beside ran, done, crashed and +not started. + Each task runs with its own network (`image.network` plus its `egress`). `tasks check --warm` runs the same null and oracle steps with network and writable caches, to fill the caches before offline runs (§2). @@ -851,7 +858,7 @@ prompts disabled`) and the fix; any other git failure shows its last `fatal:` line, else its last `error:` line, else its last line. ```bash -# self-check: no LLM; exits 1 if any task is broken or unsupported +# self-check: no LLM; exits 1 if any task is broken or unsupported, or none is ok python cli.py repo tasks check ~/oneDAL@ --manifest dev/agent_evals/repo-eval.yaml \ --task fix_finiteness_mut --build-network host --cache bazel-repo-cache=/cache/bazel-repo -j 3 diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 3c0277f7..ca15a443 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -218,7 +218,12 @@ def progress(cond: str, done: int, total: int) -> None: print(report.format_summary()) if args.output: print(f"\n✅ Wording report saved to {out}") - if args.strict and report.summary()["fragile_variants"]: + summary = report.summary() + if args.strict and not summary["measured"]: + print(f"✗ incomplete: no pair scored under {', '.join(summary['unmeasured_conditions']) or 'any condition'} " + f"({len(pairs)} pairs, errors per condition: {summary['n_errors']})", file=sys.stderr) + sys.exit(1) + if args.strict and summary["fragile_variants"]: sys.exit(1) @@ -380,7 +385,14 @@ def score(library, question, answer, ground_truth, context): print() print(md) print(f"✅ Cross-judge report saved to {out} (+ .md)") - if args.strict and any(a["label"] == "JUDGE-DEPENDENT" for a in report.summary()["arms"].values()): + arms = report.summary()["arms"] + unmeasured = sorted(a for a, r in arms.items() if r["label"] == "NOT MEASURED") + if args.strict and (not arms or unmeasured): + why = f"{len(unmeasured)} of {len(arms)} arm(s) NOT MEASURED: {', '.join(unmeasured)}" if arms else \ + "no arm has a scored question to compare" + print(f"✗ incomplete against `{report.baseline_arm}`: {why}", file=sys.stderr) + sys.exit(1) + if args.strict and any(a["label"] == "JUDGE-DEPENDENT" for a in arms.values()): sys.exit(1) @@ -418,7 +430,8 @@ def register(eval_sub, positive_int) -> None: "flag is a paired test against the original and does not use them (default: 1)") jw.add_argument("--seed", type=int, default=0) jw.add_argument("--min-accuracy", type=float, default=DEFAULT_MIN_ACCURACY, dest="min_accuracy") - jw.add_argument("--strict", action="store_true", help="Exit 1 when any variant is FRAGILE") + jw.add_argument("--strict", action="store_true", + help="Exit 1 when any variant is FRAGILE or a condition has no scored pair") jw.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) jw.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) jw.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", @@ -456,7 +469,8 @@ def register(eval_sub, positive_int) -> None: cj.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Judge sampling temperature, recorded as judge_temperature in the output") cj.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY) - cj.add_argument("--strict", action="store_true", help="Exit 1 when any arm is JUDGE-DEPENDENT") + cj.add_argument("--strict", action="store_true", + help="Exit 1 when any arm is JUDGE-DEPENDENT or NOT MEASURED, or no arm was compared") cj.add_argument("--output", default=None, help="Output JSON (default: _crossjudge.json); Markdown written next to it") cj.set_defaults(func=cmd_eval_cross_judge) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 4c67b35f..483ae1f1 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -157,6 +157,9 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: + ", ".join(r["task"] for r in allowed)) if len(bad) > len(allowed): sys.exit(1) + if len(bad) == len(results): # --allow-unsupported passed every task by: none was checked + print(f"✗ incomplete: no task passed the self-check (0 ok, {len(allowed)} unsupported)", file=sys.stderr) + sys.exit(1) def cmd_repo_run(args: argparse.Namespace) -> None: @@ -211,12 +214,17 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock print(f"✗ {exc}", file=sys.stderr) sys.exit(1) print(f"{summary['ran']} ran, {summary['skipped']} already done, {len(summary['errors'])} crashed, " - f"{summary['budget_stopped']} not started (batch budget); ${summary['spent_usd']:.4f}") + f"{summary['budget_stopped']} not started (batch budget), {summary['unsupported']} unsupported; " + f"${summary['spent_usd']:.4f}") if summary["probe_failed"]: print(f"✗ the host probe failed, so these tasks have cells that were not run: {', '.join(summary['probe_failed'])}", file=sys.stderr) if summary["errors"] or summary["budget_stopped"] or summary["probe_failed"]: sys.exit(1) + if not summary["ran"] and not summary["skipped"]: # every cell unsupported: the batch holds no result + print(f"✗ incomplete: no cell has a result ({summary['cells']} cells, {summary['unsupported']} unsupported)", + file=sys.stderr) + sys.exit(1) def _pin_arms(arms: list[Arm], git_dir: Path, source: str) -> list[str]: diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py index 82d1d66c..314abf19 100644 --- a/dokimi/dokimi/commands/run.py +++ b/dokimi/dokimi/commands/run.py @@ -57,6 +57,11 @@ def cmd_run(args: argparse.Namespace) -> None: print(f" - {v.condition}: {v.actual:.4f} < {v.threshold:.4f}", file=sys.stderr) sys.exit(1) + # A score over no document passes any gate set at 0 or switched off. + if not result["summary"].get("docs"): + print(f"\n❌ incomplete: no document was measured under {root} (0 docs)", file=sys.stderr) + sys.exit(1) + def cmd_compare(args: argparse.Namespace) -> None: """Compare two snapshots and show regression analysis.""" diff --git a/dokimi/dokimi/eval/cross_judge.py b/dokimi/dokimi/eval/cross_judge.py index df9c18ae..c747c7af 100644 --- a/dokimi/dokimi/eval/cross_judge.py +++ b/dokimi/dokimi/eval/cross_judge.py @@ -26,6 +26,9 @@ other way, but some point estimates disagree. * ``JUDGE-DEPENDENT`` — significant in both directions. Do not ship on it. * ``NO DETECTABLE EFFECT`` — no measure significant. +* ``NOT MEASURED`` — no measure that votes scored a single question of the arm + against baseline (every call errored, or every judge is gated out). It is not + "no effect": nothing was compared. A label whose significant measures all rest on one judge says so — ``CONSISTENT + (one judge)`` — because then it is one judge's verdict, not @@ -132,6 +135,8 @@ def label(rows: Sequence[Dict[str, Any]], *, require_validated: bool = False) -> """Verdict over the rows allowed to vote (see :func:`votes`), on Holm-adjusted p.""" apply_holm(rows, require_validated=require_validated) rows = [r for r in rows if r["votes"]] + if not any(r.get("n", 1) for r in rows): # no voter compared a single question + return "NOT MEASURED" sig = {r["sign"] for r in rows if r["significant"] and r["sign"]} if not sig: return "NO DETECTABLE EFFECT" diff --git a/dokimi/dokimi/eval/judge_wording.py b/dokimi/dokimi/eval/judge_wording.py index e0773d5c..30aad729 100644 --- a/dokimi/dokimi/eval/judge_wording.py +++ b/dokimi/dokimi/eval/judge_wording.py @@ -168,10 +168,16 @@ def summary(self) -> Dict[str, Any]: "fragile": spread >= MIN_FLAGGED_SPREAD and p_min is not None and p_min < ALPHA, } pooled = {c: r.summary()["accuracy"] for c, r in self.reports.items()} + # A condition with no scored pair (none built, or every call errored) was + # not measured: "no rewrite moves anything" would be read off zeros. + unmeasured = [c for c, r in self.reports.items() if not r.scored] return { "judge": self.judge, "conditions": list(self.reports), "pooled_accuracy": pooled, + "n_errors": {c: len(r.errors) for c, r in self.reports.items()}, + "unmeasured_conditions": unmeasured, + "measured": bool(self.reports) and not unmeasured, "per_variant": rows, "fragile_variants": sorted(v for v, r in rows.items() if r["fragile"]), } @@ -200,14 +206,23 @@ def format_summary(self) -> str: lines.append( f" {'pooled (graded)':20s} " + " ".join(f"{s['pooled_accuracy'][c]:10.2f}" for c in conds) ) + if any(s["n_errors"].values()): + lines.append( + f" {'errored pairs':20s} " + " ".join(f"{s['n_errors'][c]:10d}" for c in conds) + ) lines.append("") lines.append(" * neutral variant: column is unmoved rate, not accuracy") + if not s["measured"]: + lines.append( + " NOT MEASURED — no pair scored under: " + + (", ".join(s["unmeasured_conditions"]) or "any condition") + ) if s["fragile_variants"]: lines.append( " FRAGILE — a rewrite moves these on the same pairs (paired sign test, Holm): " + ", ".join(s["fragile_variants"]) ) - else: + elif s["measured"]: lines.append(" No rewrite moves any variant on the same pairs (paired sign test, Holm).") return "\n".join(lines) diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 548a0a28..fce3ac94 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -186,10 +186,14 @@ def score(self) -> dict[str, Any] | None: def load_tasks(content_dir: Path, manifest: RepoManifest, ids: list[str] | None = None) -> list[Task]: - """Tasks of the exported eval dir; ``ids`` selects (and must name existing tasks).""" + """Tasks of the exported eval dir; ``ids`` selects (and must name existing tasks). + + No task at all is refused: a check or run over none would report 0/0 as a pass.""" eval_dir, tasks = manifest.run_paths() tasks_dir = content_dir / PurePosixPath(tasks).relative_to(eval_dir) found = {p.parent.name: p for p in sorted(tasks_dir.glob("*/task.json"))} + if not found: + raise PrepError(f"no task under {manifest.tasks} (no */task.json): nothing to check or run") wanted = ids or sorted(found) missing = [t for t in wanted if t not in found] if missing: diff --git a/dokimi/tests/test_gate_call_sites.py b/dokimi/tests/test_gate_call_sites.py index d0d24f44..afe67908 100644 --- a/dokimi/tests/test_gate_call_sites.py +++ b/dokimi/tests/test_gate_call_sites.py @@ -87,3 +87,20 @@ def test_runner_passes_soft_kind(tmp_path): for p in ps: p.stop() assert spy.call_args.args[2] == "soft" + + +def test_strict_exit_1_when_no_document_was_measured(tmp_path, capsys): + # A gate at 0 (or a disabled one) passes a score over nothing. + for future in ({"hard_gate": G(0.0)}, {}): + with patch("dokimi.runner.run.discover_markdown", return_value=[]): + sp = tmp_path / "spec.yaml" + sp.write_text(yaml.safe_dump(dict(SPEC, future=future))) + args = argparse.Namespace(root=str(tmp_path), spec=str(sp), out_json=str(tmp_path / "o.json"), + out_md=str(tmp_path / "o.md"), strict=True) + try: + cmd_run(args) + code = 0 + except SystemExit as e: + code = e.code + assert code == 1 + assert "incomplete: no document was measured" in capsys.readouterr().err diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index f4ef6890..12a4d34b 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -1693,3 +1693,75 @@ def test_a_pass_in_one_context_never_names_the_context_of_a_grounded_measure(): report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[grounded]) assert report.apply_validity([{"judge": "p/j", "context": "stored", "passed": True}]) == ["p/j"] assert grounded.gate is None + + +# --- nothing measured is not a pass ----------------------------------------- + + +def _strict_exit(argv): + from dokimi.cli import build_parser + + args = build_parser().parse_args(argv) + try: + args.func(args) + except SystemExit as e: + return e.code + return 0 + + +def test_cross_judge_strict_exits_1_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge as judge_mod + + class ErroringJudge: + def __init__(self, **_): + pass + + def score_answer(self, **_): + raise RuntimeError("ThrottlingException") + + monkeypatch.setattr(judge_mod, "Judge", ErroringJudge) + errored = tmp_path / "errored.json" + errored.write_text(json.dumps(_stamped_run()), encoding="utf-8") + # Nothing to compare: the stored judge scored only the baseline arm. + unpaired = _stamped_run() + for ev in unpaired["evaluations"]: + del ev["scores"]["skill"] + empty = tmp_path / "empty.json" + empty.write_text(json.dumps(unpaired), encoding="utf-8") + for argv, arms in ((["--answers", str(errored), "--no-stored", "--judges", "amazon-bedrock/m"], {"skill": "NOT MEASURED"}), + (["--answers", str(empty)], {})): + code = _strict_exit(["eval", "cross-judge", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) + assert code == 1 + assert "✗ incomplete" in capsys.readouterr().err + summary = json.loads((tmp_path / "cj.json").read_text())["summary"] + assert {a: r["label"] for a, r in summary["arms"].items()} == arms + + +def test_judge_wording_strict_exits_1_when_a_condition_scored_no_pair(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge_wording + from dokimi.eval.judge_validity import ValidityReport + from dokimi.eval.validity_scoring import ValidityRow + + from tests.test_judge_validity import _answers as validity_answers + + def errored(n): + return ValidityReport(judge="fake", rows=[ + ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", error="ThrottlingException") + for i in range(n)]) + + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(validity_answers()), encoding="utf-8") + for n in (0, 4): # no pair built; every call errored + report = WordingReport(judge="fake", reports={c: errored(n) for c in ("original", "persona")}) + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, _r=report, **k: _r) + code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) + assert code == 1 + out = capsys.readouterr() + assert "NOT MEASURED — no pair scored under: original, persona" in out.out + assert "No rewrite moves" not in out.out and "incomplete" in out.err + assert report.summary()["n_errors"] == {"original": n, "persona": n} diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index e592e0d0..607f80a9 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -227,6 +227,31 @@ def test_allow_unsupported_lets_host_unsupported_tasks_pass_the_exit_code(tmp_pa assert race["verdict"] == "unsupported" and race["unsupported_by"] == "host" +def _taskless_repo(tmp_path): + from repo_eval_fixture import git + + root, _ = make_repo(tmp_path) + git(root, "rm", "-rqf", ".agent-eval/tasks") + (root / ".agent-eval/tasks").mkdir(parents=True) + (root / ".agent-eval/tasks/.keep").write_text("") + git(root, "add", "-A") + git(root, "commit", "-qm", "no tasks") + return root + + +def test_tasks_check_with_no_task_checked_is_not_a_pass(tmp_path, monkeypatch, capsys): + # No task at all: 0/0 ok was exit 0. + with pytest.raises(SystemExit) as exc: + _check(_taskless_repo(tmp_path / "e"), tmp_path / "a", HostRunner(_probe_out()), monkeypatch) + assert exc.value.code == 1 and "no task under" in capsys.readouterr().err + # Every task unsupported and allowed: 0/1 ok was exit 0, with nothing self-checked. + with pytest.raises(SystemExit) as exc: + _check(_repo(tmp_path), tmp_path / "b", HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)), + monkeypatch, "--allow-unsupported", "--task", "fix_race") + assert exc.value.code == 1 + assert "incomplete: no task passed the self-check (0 ok, 1 unsupported)" in capsys.readouterr().err + + @pytest.mark.parametrize("extra, why", [ ({"fix_nobug": {"family": "fix", "grader": "calc", "gate": ["tests_pass"], "prompt": "x"}}, "broken"), ({"fix_pip": {"family": "fix", "grader": "calc", "egress": ["pypi.org"], "prompt": "x"}}, "egress"), @@ -301,10 +326,26 @@ def test_run_skips_unsupported_cells_and_the_report_shows_them(tmp_path, monkeyp assert len(load_cells(batch)[0]) == 4 and not list(batch.glob("*/unsupported.json")) +def test_a_run_with_no_cell_result_is_not_a_pass(tmp_path, monkeypatch, capsys): + # No task at all: this crashed with IndexError in the isolation probe. + with pytest.raises(SystemExit) as exc: + _run(_taskless_repo(tmp_path / "e"), tmp_path / "e/runs", HostRunner(_probe_out()), monkeypatch) + assert exc.value.code == 1 and "no task under" in capsys.readouterr().err + # Every cell unsupported on this host: "0 ran, 0 already done, 0 crashed" was exit 0. + with pytest.raises(SystemExit) as exc: + _run(_repo(tmp_path), tmp_path / "runs", HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)), + monkeypatch, "--task", "fix_race") + assert exc.value.code == 1 + logged = capsys.readouterr() + assert "0 ran, 0 already done, 0 crashed, 0 not started (batch budget), 2 unsupported" in logged.out + assert "incomplete: no cell has a result (2 cells, 2 unsupported)" in logged.err + + def test_a_capable_run_that_stops_early_leaves_the_unreached_cells_missing(tmp_path, monkeypatch, capsys): root = _repo(tmp_path) out = tmp_path / "runs" - _run(root, out, HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)), monkeypatch, "--task", "fix_race") + with pytest.raises(SystemExit): # every cell unsupported: no result + _run(root, out, HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)), monkeypatch, "--task", "fix_race") with pytest.raises(SystemExit): # the budget stops it after one cell _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--task", "fix_race", "--max-batch-usd", "0.005") @@ -588,8 +629,9 @@ def test_a_host_miss_still_checks_the_oracle_when_only_cpus_are_short(tmp_path, def test_a_host_miss_still_probes_isolation_and_skips_only_null(tmp_path, monkeypatch, capsys): root = _repo(tmp_path, host={"min_cpus": 4}) - _check(root, tmp_path / "a", HostRunner(_probe_out(affinity=2)), monkeypatch, "--task", "fix_race", - "--allow-unsupported") + with pytest.raises(SystemExit): # the only task is unsupported: nothing was self-checked + _check(root, tmp_path / "a", HostRunner(_probe_out(affinity=2)), monkeypatch, "--task", "fix_race", + "--allow-unsupported") race = json.loads((tmp_path / "a/check/check.json").read_text())["tasks"][0] assert race["verdict"] == "unsupported" and race["unsupported_by"] == "host" assert "isolation" in race and "oracle" in race and "null" not in race diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 9cba295d..ea9bb22c 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -241,7 +241,7 @@ def test_timed_out_cell_counts_toward_batch_budget(setup, monkeypatch, capsys): _run(setup, "--arm", "guidance:raw", "--task", "fix_add", "--reps", "2", "--timeout", "2", "--budget", "0.5", "--max-batch-usd", "0.01") assert exc.value.code == 1 - assert "1 ran" in (out := capsys.readouterr().out) and "1 not started (batch budget); $0.5000" in out + assert "1 ran" in (out := capsys.readouterr().out) and "1 not started (batch budget), 0 unsupported; $0.5000" in out rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) assert rec["agent"]["cost_usd"] is None and rec["agent"]["charged_usd"] == 0.5 assert _report(setup)["rows"][0]["cost_unknown"] == 1 From c1d1be111efac801075b7dad31307522882e096a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 14:26:53 +0000 Subject: [PATCH 2/7] docs: note that run --strict exits 1 when no document was measured Signed-off-by: Anton Volkov --- dokimi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/README.md b/dokimi/README.md index e7e45d7d..499e2da5 100644 --- a/dokimi/README.md +++ b/dokimi/README.md @@ -202,7 +202,7 @@ The default static benchmark is configured in `benchmarks/spec.v1.yaml` and curr | `readability` | Estimates text readability with a Flesch-Kincaid-style score. | | `example_pass_rate` | Executes supported fenced code examples in isolation. | -Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. +Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. `--strict` also exits 1 when no document was measured. ## LLM and treatment evaluation workflow From 9e7b29af086ab3d33c8425cebc1e3b1999b5eaf5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:21:31 -0700 Subject: [PATCH 3/7] test(judge): pin any-vs-all in the cross-judge and wording NOT MEASURED verdicts Signed-off-by: Nikolay Petrov --- dokimi/tests/test_judge_reliability.py | 63 ++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index 12a4d34b..f5acc136 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -360,6 +360,13 @@ def test_verdict_labels(rows, expected): assert cj.label(rows) == expected +def test_one_voter_that_compared_nothing_does_not_unmeasure_what_another_compared(): + measured = {"sign": 1, "p_sign": 0.001, "n": 8} + errored = {"sign": 0, "p_sign": 1.0, "n": 0} + assert cj.label([measured, errored]) == "CONSISTENT + (one judge)" + assert cj.label([errored, dict(errored)]) == "NOT MEASURED" + + def test_measures_pair_by_question_and_skip_unscored(): data = {"evaluations": [ {"question_id": f"Q{i}", "scores": {"baseline": {"aggregate": 80}, "skill": {"aggregate": 90}}} @@ -1739,6 +1746,62 @@ def score_answer(self, **_): assert {a: r["label"] for a, r in summary["arms"].items()} == arms +def test_cross_judge_strict_exits_1_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge as judge_mod + + class HalfErroringJudge: + def __init__(self, **_): + pass + + def score_answer(self, *, answer, **_): + if answer.startswith("t"): # the third arm's answers + raise RuntimeError("ThrottlingException") + return {"aggregate": 90.0 if answer.startswith("s") else 50.0} + + monkeypatch.setattr(judge_mod, "Judge", HalfErroringJudge) + data = _stamped_run() + for ans in data["answers"]: + ans["arms"]["skill2"] = {"answer": "t" + ans["arms"]["skill"]["answer"][1:], + "metadata": {"arm_kind": "skill"}} + path = tmp_path / "answers.json" + path.write_text(json.dumps(data), encoding="utf-8") + code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--no-stored", "--judges", + "amazon-bedrock/m", "--output", str(tmp_path / "cj.json"), "--strict"]) + summary = json.loads((tmp_path / "cj.json").read_text())["summary"] + assert summary["arms"]["skill"]["label"] != "NOT MEASURED" + assert summary["arms"]["skill2"]["label"] == "NOT MEASURED" + assert code == 1 + assert "1 of 2 arm(s) NOT MEASURED: skill2" in capsys.readouterr().err + + +def test_judge_wording_strict_exits_1_when_one_condition_scored_and_another_did_not(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge_wording + from dokimi.eval.judge_validity import ValidityReport + from dokimi.eval.validity_scoring import ValidityRow + + from tests.test_judge_validity import _answers as validity_answers + + def rows(**kw): + return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", **kw) for i in range(4)] + + report = WordingReport(judge="fake", reports={ + "original": ValidityReport(judge="fake", rows=rows(intact_score=90.0, degraded_score=40.0)), + "persona": ValidityReport(judge="fake", rows=rows(error="ThrottlingException"))}) + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(validity_answers()), encoding="utf-8") + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) + code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) + out = capsys.readouterr() + assert report.summary()["unmeasured_conditions"] == ["persona"] + assert code == 1 + assert "No rewrite moves" not in out.out and "incomplete" in out.err + + def test_judge_wording_strict_exits_1_when_a_condition_scored_no_pair(tmp_path, monkeypatch, capsys): import json From 2cc66db6b405d211deee41c4eaca80fbf37fb487 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 01:38:18 -0700 Subject: [PATCH 4/7] refactor(judge): keep cross-judge --strict and its tests inside the growth baseline On main, the growth ratchet fails this branch: cmd_eval_cross_judge went from 32 to 33 complexity and tests/test_judge_reliability.py from 1695 to 1830 lines. Move the strict exit into _strict_cross_judge_exit and the new tests into tests/test_judge_not_measured.py; behaviour is unchanged. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/judge_reliability.py | 15 ++- dokimi/tests/test_judge_not_measured.py | 142 ++++++++++++++++++++ dokimi/tests/test_judge_reliability.py | 135 ------------------- 3 files changed, 152 insertions(+), 140 deletions(-) create mode 100644 dokimi/tests/test_judge_not_measured.py diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index ca15a443..065d6962 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -9,7 +9,7 @@ import json import sys from pathlib import Path -from typing import Optional +from typing import Any, Optional from dokimi.defaults import ( DEFAULT_CONCURRENCY, @@ -385,14 +385,19 @@ def score(library, question, answer, ground_truth, context): print() print(md) print(f"✅ Cross-judge report saved to {out} (+ .md)") - arms = report.summary()["arms"] + if args.strict: + _strict_cross_judge_exit(report.summary()["arms"], report.baseline_arm) + + +def _strict_cross_judge_exit(arms: dict[str, dict[str, Any]], baseline_arm: str) -> None: + """Exit 1 when no arm was compared, an arm is NOT MEASURED, or an arm is JUDGE-DEPENDENT.""" unmeasured = sorted(a for a, r in arms.items() if r["label"] == "NOT MEASURED") - if args.strict and (not arms or unmeasured): + if not arms or unmeasured: why = f"{len(unmeasured)} of {len(arms)} arm(s) NOT MEASURED: {', '.join(unmeasured)}" if arms else \ "no arm has a scored question to compare" - print(f"✗ incomplete against `{report.baseline_arm}`: {why}", file=sys.stderr) + print(f"✗ incomplete against `{baseline_arm}`: {why}", file=sys.stderr) sys.exit(1) - if args.strict and any(a["label"] == "JUDGE-DEPENDENT" for a in arms.values()): + if any(a["label"] == "JUDGE-DEPENDENT" for a in arms.values()): sys.exit(1) diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py new file mode 100644 index 00000000..1b543ff1 --- /dev/null +++ b/dokimi/tests/test_judge_not_measured.py @@ -0,0 +1,142 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Nothing measured is not a pass: the cross-judge and judge-wording verdicts, and their +`--strict` exits, when no voter or condition scored a single question.""" + +from dokimi.eval import cross_judge as cj +from dokimi.eval.judge_wording import WordingReport + +from tests.test_judge_reliability import _stamped_run + + +def test_one_voter_that_compared_nothing_does_not_unmeasure_what_another_compared(): + measured = {"sign": 1, "p_sign": 0.001, "n": 8} + errored = {"sign": 0, "p_sign": 1.0, "n": 0} + assert cj.label([measured, errored]) == "CONSISTENT + (one judge)" + assert cj.label([errored, dict(errored)]) == "NOT MEASURED" + + +def _strict_exit(argv): + from dokimi.cli import build_parser + + args = build_parser().parse_args(argv) + try: + args.func(args) + except SystemExit as e: + return e.code + return 0 + + +def test_cross_judge_strict_exits_1_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge as judge_mod + + class ErroringJudge: + def __init__(self, **_): + pass + + def score_answer(self, **_): + raise RuntimeError("ThrottlingException") + + monkeypatch.setattr(judge_mod, "Judge", ErroringJudge) + errored = tmp_path / "errored.json" + errored.write_text(json.dumps(_stamped_run()), encoding="utf-8") + # Nothing to compare: the stored judge scored only the baseline arm. + unpaired = _stamped_run() + for ev in unpaired["evaluations"]: + del ev["scores"]["skill"] + empty = tmp_path / "empty.json" + empty.write_text(json.dumps(unpaired), encoding="utf-8") + for argv, arms in ((["--answers", str(errored), "--no-stored", "--judges", "amazon-bedrock/m"], {"skill": "NOT MEASURED"}), + (["--answers", str(empty)], {})): + code = _strict_exit(["eval", "cross-judge", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) + assert code == 1 + assert "✗ incomplete" in capsys.readouterr().err + summary = json.loads((tmp_path / "cj.json").read_text())["summary"] + assert {a: r["label"] for a, r in summary["arms"].items()} == arms + + +def test_cross_judge_strict_exits_1_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge as judge_mod + + class HalfErroringJudge: + def __init__(self, **_): + pass + + def score_answer(self, *, answer, **_): + if answer.startswith("t"): # the third arm's answers + raise RuntimeError("ThrottlingException") + return {"aggregate": 90.0 if answer.startswith("s") else 50.0} + + monkeypatch.setattr(judge_mod, "Judge", HalfErroringJudge) + data = _stamped_run() + for ans in data["answers"]: + ans["arms"]["skill2"] = {"answer": "t" + ans["arms"]["skill"]["answer"][1:], + "metadata": {"arm_kind": "skill"}} + path = tmp_path / "answers.json" + path.write_text(json.dumps(data), encoding="utf-8") + code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--no-stored", "--judges", + "amazon-bedrock/m", "--output", str(tmp_path / "cj.json"), "--strict"]) + summary = json.loads((tmp_path / "cj.json").read_text())["summary"] + assert summary["arms"]["skill"]["label"] != "NOT MEASURED" + assert summary["arms"]["skill2"]["label"] == "NOT MEASURED" + assert code == 1 + assert "1 of 2 arm(s) NOT MEASURED: skill2" in capsys.readouterr().err + + +def test_judge_wording_strict_exits_1_when_one_condition_scored_and_another_did_not(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge_wording + from dokimi.eval.judge_validity import ValidityReport + from dokimi.eval.validity_scoring import ValidityRow + + from tests.test_judge_validity import _answers as validity_answers + + def rows(**kw): + return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", **kw) for i in range(4)] + + report = WordingReport(judge="fake", reports={ + "original": ValidityReport(judge="fake", rows=rows(intact_score=90.0, degraded_score=40.0)), + "persona": ValidityReport(judge="fake", rows=rows(error="ThrottlingException"))}) + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(validity_answers()), encoding="utf-8") + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) + code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) + out = capsys.readouterr() + assert report.summary()["unmeasured_conditions"] == ["persona"] + assert code == 1 + assert "No rewrite moves" not in out.out and "incomplete" in out.err + + +def test_judge_wording_strict_exits_1_when_a_condition_scored_no_pair(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge_wording + from dokimi.eval.judge_validity import ValidityReport + from dokimi.eval.validity_scoring import ValidityRow + + from tests.test_judge_validity import _answers as validity_answers + + def errored(n): + return ValidityReport(judge="fake", rows=[ + ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", error="ThrottlingException") + for i in range(n)]) + + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(validity_answers()), encoding="utf-8") + for n in (0, 4): # no pair built; every call errored + report = WordingReport(judge="fake", reports={c: errored(n) for c in ("original", "persona")}) + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, _r=report, **k: _r) + code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) + assert code == 1 + out = capsys.readouterr() + assert "NOT MEASURED — no pair scored under: original, persona" in out.out + assert "No rewrite moves" not in out.out and "incomplete" in out.err + assert report.summary()["n_errors"] == {"original": n, "persona": n} diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index f5acc136..f4ef6890 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -360,13 +360,6 @@ def test_verdict_labels(rows, expected): assert cj.label(rows) == expected -def test_one_voter_that_compared_nothing_does_not_unmeasure_what_another_compared(): - measured = {"sign": 1, "p_sign": 0.001, "n": 8} - errored = {"sign": 0, "p_sign": 1.0, "n": 0} - assert cj.label([measured, errored]) == "CONSISTENT + (one judge)" - assert cj.label([errored, dict(errored)]) == "NOT MEASURED" - - def test_measures_pair_by_question_and_skip_unscored(): data = {"evaluations": [ {"question_id": f"Q{i}", "scores": {"baseline": {"aggregate": 80}, "skill": {"aggregate": 90}}} @@ -1700,131 +1693,3 @@ def test_a_pass_in_one_context_never_names_the_context_of_a_grounded_measure(): report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[grounded]) assert report.apply_validity([{"judge": "p/j", "context": "stored", "passed": True}]) == ["p/j"] assert grounded.gate is None - - -# --- nothing measured is not a pass ----------------------------------------- - - -def _strict_exit(argv): - from dokimi.cli import build_parser - - args = build_parser().parse_args(argv) - try: - args.func(args) - except SystemExit as e: - return e.code - return 0 - - -def test_cross_judge_strict_exits_1_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): - import json - - from dokimi.eval import judge as judge_mod - - class ErroringJudge: - def __init__(self, **_): - pass - - def score_answer(self, **_): - raise RuntimeError("ThrottlingException") - - monkeypatch.setattr(judge_mod, "Judge", ErroringJudge) - errored = tmp_path / "errored.json" - errored.write_text(json.dumps(_stamped_run()), encoding="utf-8") - # Nothing to compare: the stored judge scored only the baseline arm. - unpaired = _stamped_run() - for ev in unpaired["evaluations"]: - del ev["scores"]["skill"] - empty = tmp_path / "empty.json" - empty.write_text(json.dumps(unpaired), encoding="utf-8") - for argv, arms in ((["--answers", str(errored), "--no-stored", "--judges", "amazon-bedrock/m"], {"skill": "NOT MEASURED"}), - (["--answers", str(empty)], {})): - code = _strict_exit(["eval", "cross-judge", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) - assert code == 1 - assert "✗ incomplete" in capsys.readouterr().err - summary = json.loads((tmp_path / "cj.json").read_text())["summary"] - assert {a: r["label"] for a, r in summary["arms"].items()} == arms - - -def test_cross_judge_strict_exits_1_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): - import json - - from dokimi.eval import judge as judge_mod - - class HalfErroringJudge: - def __init__(self, **_): - pass - - def score_answer(self, *, answer, **_): - if answer.startswith("t"): # the third arm's answers - raise RuntimeError("ThrottlingException") - return {"aggregate": 90.0 if answer.startswith("s") else 50.0} - - monkeypatch.setattr(judge_mod, "Judge", HalfErroringJudge) - data = _stamped_run() - for ans in data["answers"]: - ans["arms"]["skill2"] = {"answer": "t" + ans["arms"]["skill"]["answer"][1:], - "metadata": {"arm_kind": "skill"}} - path = tmp_path / "answers.json" - path.write_text(json.dumps(data), encoding="utf-8") - code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--no-stored", "--judges", - "amazon-bedrock/m", "--output", str(tmp_path / "cj.json"), "--strict"]) - summary = json.loads((tmp_path / "cj.json").read_text())["summary"] - assert summary["arms"]["skill"]["label"] != "NOT MEASURED" - assert summary["arms"]["skill2"]["label"] == "NOT MEASURED" - assert code == 1 - assert "1 of 2 arm(s) NOT MEASURED: skill2" in capsys.readouterr().err - - -def test_judge_wording_strict_exits_1_when_one_condition_scored_and_another_did_not(tmp_path, monkeypatch, capsys): - import json - - from dokimi.eval import judge_wording - from dokimi.eval.judge_validity import ValidityReport - from dokimi.eval.validity_scoring import ValidityRow - - from tests.test_judge_validity import _answers as validity_answers - - def rows(**kw): - return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", **kw) for i in range(4)] - - report = WordingReport(judge="fake", reports={ - "original": ValidityReport(judge="fake", rows=rows(intact_score=90.0, degraded_score=40.0)), - "persona": ValidityReport(judge="fake", rows=rows(error="ThrottlingException"))}) - answers = tmp_path / "answers.json" - answers.write_text(json.dumps(validity_answers()), encoding="utf-8") - monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) - code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", - "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) - out = capsys.readouterr() - assert report.summary()["unmeasured_conditions"] == ["persona"] - assert code == 1 - assert "No rewrite moves" not in out.out and "incomplete" in out.err - - -def test_judge_wording_strict_exits_1_when_a_condition_scored_no_pair(tmp_path, monkeypatch, capsys): - import json - - from dokimi.eval import judge_wording - from dokimi.eval.judge_validity import ValidityReport - from dokimi.eval.validity_scoring import ValidityRow - - from tests.test_judge_validity import _answers as validity_answers - - def errored(n): - return ValidityReport(judge="fake", rows=[ - ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", error="ThrottlingException") - for i in range(n)]) - - answers = tmp_path / "answers.json" - answers.write_text(json.dumps(validity_answers()), encoding="utf-8") - for n in (0, 4): # no pair built; every call errored - report = WordingReport(judge="fake", reports={c: errored(n) for c in ("original", "persona")}) - monkeypatch.setattr(judge_wording, "run_wording", lambda *a, _r=report, **k: _r) - code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", - "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) - assert code == 1 - out = capsys.readouterr() - assert "NOT MEASURED — no pair scored under: original, persona" in out.out - assert "No rewrite moves" not in out.out and "incomplete" in out.err - assert report.summary()["n_errors"] == {"original": n, "persona": n} From 2d97d0eff600f31ae6a1f22fec2c031aeb312384 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 01:38:43 -0700 Subject: [PATCH 5/7] test(judge): pin NOT MEASURED when the validity gate leaves no voter label() returns NOT MEASURED for an empty voter set (every judge FAILed, or --require-validated with no passing validity), and cross-judge --strict exits 1 on it. No test reached that branch: `if rows and not any(...)` passed the whole suite. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_judge_not_measured.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py index 1b543ff1..6979ce35 100644 --- a/dokimi/tests/test_judge_not_measured.py +++ b/dokimi/tests/test_judge_not_measured.py @@ -140,3 +140,21 @@ def errored(n): assert "NOT MEASURED — no pair scored under: original, persona" in out.out assert "No rewrite moves" not in out.out and "incomplete" in out.err assert report.summary()["n_errors"] == {"original": n, "persona": n} + + +def test_no_voter_left_by_the_validity_gate_is_not_measured(): + significant = {"sign": 1, "p_sign": 0.001, "n": 8} + assert cj.label([{**significant, "gate": "FAIL"}]) == "NOT MEASURED" + assert cj.label([dict(significant)], require_validated=True) == "NOT MEASURED" + + +def test_cross_judge_strict_exits_1_when_require_validated_leaves_no_voter(tmp_path, capsys): + import json + + path = tmp_path / "run.json" + path.write_text(json.dumps(_stamped_run()), encoding="utf-8") + code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--require-validated", + "--output", str(tmp_path / "cj.json"), "--strict"]) + assert code == 1 + assert "1 of 1 arm(s) NOT MEASURED: skill" in capsys.readouterr().err + assert json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"]["skill"]["label"] == "NOT MEASURED" From 9ab2184032a2639fd4700aea1e5c9a7d50ac7ca9 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 01:39:25 -0700 Subject: [PATCH 6/7] docs(repo): say in --allow-unsupported help that no ok task still exits 1 tasks check now exits 1 as incomplete when every task checked is unsupported, but the --allow-unsupported help still promised exit 0 for that case. State the exception in the help and in the contract sentence. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/commands/repo_run.py | 3 ++- dokimi/tests/test_repo_host.py | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5ce2a53e..64062293 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -443,7 +443,8 @@ For every task, before any paid run: A task whose `host` requirements (§3) the probe container does not meet is `unsupported` with the reason, like a task whose `egress` needs the missing proxy, and `tasks check` exits 1 for it. With `--allow-unsupported` it is -still printed with its reason but does not make the exit code 1; broken tasks, +still printed with its reason but does not make the exit code 1 unless no task +is ok (below); broken tasks, tasks that need egress and tasks whose host probe failed still do. `repo run` does not run its cells: each holds an `unsupported.json` with the reason, and `repo report` lists them apart, neither failed nor missing. If the probe itself fails, a task with diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 414e6277..81e8f1fe 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -355,7 +355,8 @@ def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: check_p.add_argument("--allow-unsupported", action="store_true", dest="allow_unsupported", help="Exit 0 when the only tasks that are not ok are unsupported because this host " "misses their task.json `host` requirements (still printed with the reason). Broken " - "tasks, tasks that need egress and tasks whose host probe failed still exit 1") + "tasks, tasks that need egress and tasks whose host probe failed still exit 1, and so " + "does a check in which no task is ok") check_p.add_argument("--out", default="repo-check", help="Work directory (default: repo-check)") check_p.set_defaults(func=cmd_repo_tasks_check) diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index 607f80a9..8941213b 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -252,6 +252,14 @@ def test_tasks_check_with_no_task_checked_is_not_a_pass(tmp_path, monkeypatch, c assert "incomplete: no task passed the self-check (0 ok, 1 unsupported)" in capsys.readouterr().err +def test_allow_unsupported_help_says_a_check_with_no_ok_task_still_exits_1(capsys): + from dokimi.cli import build_parser + + with pytest.raises(SystemExit): + build_parser().parse_args(["repo", "tasks", "check", "--help"]) + assert "and so does a check in which no task is ok" in " ".join(capsys.readouterr().out.split()) + + @pytest.mark.parametrize("extra, why", [ ({"fix_nobug": {"family": "fix", "grader": "calc", "gate": ["tests_pass"], "prompt": "x"}}, "broken"), ({"fix_pip": {"family": "fix", "grader": "calc", "egress": ["pypi.org"], "prompt": "x"}}, "egress"), From 1983d7d11de5462e2720e03f70a341df8f16eb7f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:43:33 -0700 Subject: [PATCH 7/7] fix(judge): no FRAGILE flag when a wording condition was not measured; pin --strict on judge-wording and cross-judge exits Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/judge_wording.py | 12 +++-- dokimi/tests/test_judge_not_measured.py | 63 ++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/eval/judge_wording.py b/dokimi/dokimi/eval/judge_wording.py index 30aad729..e09e9235 100644 --- a/dokimi/dokimi/eval/judge_wording.py +++ b/dokimi/dokimi/eval/judge_wording.py @@ -148,6 +148,11 @@ def summary(self) -> Dict[str, Any]: tests = [(v, c) for v in sorted(acc) for c in rewrites if c in acc[v] and "original" in acc[v]] raw = [self._paired_p(v, c) for v, c in tests] holm = dict(zip(tests, adjust_pvalues(raw, method="holm", rounded=False), strict=True)) + # A condition with no scored pair (none built, or every call errored) was + # not measured: "no rewrite moves anything" would be read off zeros, and an + # errored pair counts 0 in the sign test, so nothing is flagged either. + unmeasured = [c for c, r in self.reports.items() if not r.scored] + measured = bool(self.reports) and not unmeasured rows = {} for variant, by in sorted(acc.items()): noise = [by[c] for c in floor if c in by] @@ -165,19 +170,16 @@ def summary(self) -> Dict[str, Any]: "spread": spread, "worst_wording": worst, "p_holm": None if p_min is None else round(p_min, 4), - "fragile": spread >= MIN_FLAGGED_SPREAD and p_min is not None and p_min < ALPHA, + "fragile": measured and spread >= MIN_FLAGGED_SPREAD and p_min is not None and p_min < ALPHA, } pooled = {c: r.summary()["accuracy"] for c, r in self.reports.items()} - # A condition with no scored pair (none built, or every call errored) was - # not measured: "no rewrite moves anything" would be read off zeros. - unmeasured = [c for c, r in self.reports.items() if not r.scored] return { "judge": self.judge, "conditions": list(self.reports), "pooled_accuracy": pooled, "n_errors": {c: len(r.errors) for c, r in self.reports.items()}, "unmeasured_conditions": unmeasured, - "measured": bool(self.reports) and not unmeasured, + "measured": measured, "per_variant": rows, "fragile_variants": sorted(v for v, r in rows.items() if r["fragile"]), } diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py index 6979ce35..0739ae8c 100644 --- a/dokimi/tests/test_judge_not_measured.py +++ b/dokimi/tests/test_judge_not_measured.py @@ -7,7 +7,7 @@ from dokimi.eval import cross_judge as cj from dokimi.eval.judge_wording import WordingReport -from tests.test_judge_reliability import _stamped_run +from tests.test_judge_reliability import _material_run, _stamped_run def test_one_voter_that_compared_nothing_does_not_unmeasure_what_another_compared(): @@ -98,7 +98,7 @@ def test_judge_wording_strict_exits_1_when_one_condition_scored_and_another_did_ from tests.test_judge_validity import _answers as validity_answers def rows(**kw): - return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", **kw) for i in range(4)] + return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", **kw) for i in range(8)] report = WordingReport(judge="fake", reports={ "original": ValidityReport(judge="fake", rows=rows(intact_score=90.0, degraded_score=40.0)), @@ -112,6 +112,15 @@ def rows(**kw): assert report.summary()["unmeasured_conditions"] == ["persona"] assert code == 1 assert "No rewrite moves" not in out.out and "incomplete" in out.err + # The errored pairs count 0 in the sign test: that is not a finding, so no FRAGILE flag either. + assert "NOT MEASURED" in out.out and "FRAGILE" not in out.out + assert report.summary()["fragile_variants"] == [] + assert all(not r["fragile"] for r in report.summary()["per_variant"].values()) + # Report-only without --strict: the same run prints NOT MEASURED and exits 0. + code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m"]) + assert code == 0 + assert "NOT MEASURED" in capsys.readouterr().out def test_judge_wording_strict_exits_1_when_a_condition_scored_no_pair(tmp_path, monkeypatch, capsys): @@ -158,3 +167,53 @@ def test_cross_judge_strict_exits_1_when_require_validated_leaves_no_voter(tmp_p assert code == 1 assert "1 of 1 arm(s) NOT MEASURED: skill" in capsys.readouterr().err assert json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"]["skill"]["label"] == "NOT MEASURED" + + +def test_cross_judge_judge_dependent_exits_1_only_with_strict(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge as judge_mod + + class OppositeJudge: + def __init__(self, **_): + pass + + def score_answer(self, *, answer, **_): # prefers the baseline, the stored judge the skill + return {"aggregate": 90.0 if answer.startswith("b") else 50.0} + + monkeypatch.setattr(judge_mod, "Judge", OppositeJudge) + path = tmp_path / "run.json" + path.write_text(json.dumps(_material_run()), encoding="utf-8") + argv = ["eval", "cross-judge", "--answers", str(path), "--judges", "amazon-bedrock/m", + "--output", str(tmp_path / "cj.json")] + assert _strict_exit(argv) == 0 + capsys.readouterr() + assert json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"]["skill"]["label"] == "JUDGE-DEPENDENT" + assert _strict_exit([*argv, "--strict"]) == 1 + + +def test_judge_wording_fragile_variant_exits_1_only_with_strict(tmp_path, monkeypatch, capsys): + import json + + from dokimi.eval import judge_wording + from dokimi.eval.judge_validity import ValidityReport + from dokimi.eval.validity_scoring import ValidityRow + + from tests.test_judge_validity import _answers as validity_answers + + def rows(degraded): + return [ValidityRow(base_id=f"Q{i}", arm="baseline", variant="wrong_numbers", + intact_score=90.0, degraded_score=degraded) for i in range(8)] + + report = WordingReport(judge="fake", reports={ + "original": ValidityReport(judge="fake", rows=rows(40.0)), + "persona": ValidityReport(judge="fake", rows=rows(95.0))}) # the rewrite stops seeing the damage + assert report.summary()["fragile_variants"] == ["wrong_numbers"] and report.summary()["measured"] + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(validity_answers()), encoding="utf-8") + monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) + argv = ["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + "--judge-provider", "amazon-bedrock", "--judge-model", "m"] + assert _strict_exit(argv) == 0 + assert "FRAGILE" in capsys.readouterr().out + assert _strict_exit([*argv, "--strict"]) == 1