diff --git a/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md b/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md new file mode 100644 index 00000000..40af7a4b --- /dev/null +++ b/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md @@ -0,0 +1,3 @@ +### Fixed + +- **Nothing measured is not a pass (second batch)** — `tasks run` exits 1 as `incomplete` when every cell errored (the artifact and report are still written; partly errored runs keep exit 0 and their counts). The novelty gate's status is `NOT MEASURED`, not `PASS`, for a cell with no check measured. `run` refuses with one line and exit 1 when a selected document does not load, instead of a KeyError or OSError traceback. `baseline compare` says "Nothing to compare (0 scored)" when no question is scored on both sides, and its "No notable changes" line counts the questions compared; the snapshot `compare` report says "Nothing to compare (0 scored)" instead of "No regressions" when either snapshot has no document. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 6276ea94..91cb3a87 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -446,7 +446,10 @@ a verdict. The rules mirror intel/skills' `tools/check_harbor_job.py`: | no trial file (with or without the job's), more than one trial, or not a JSON object | errored | An errored cell is out of the pass-rate denominator and the paired test, as -for `docker-*` and BenchFlow. `metrics.n_trials` counts the trial files and +for `docker-*` and BenchFlow. When every cell of a `tasks run` errored, it +still writes the artifact and the report, then exits 1 as `incomplete` +(`0 scored, N errored of N cells`); a run with some cells scored exits 0 and +prints the errored count per harness. `metrics.n_trials` counts the trial files and `metrics.result_json` names the one read. A `harbor` that `dokimi` kills at the task budget is a failed cell (reward 0, returncode 124), as above. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 00c93d7c..651ca856 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -755,7 +755,9 @@ a threshold on `added` flipped its verdict on up to 24% of artifacts at a 0.25 p blocking novelty check would be a blocking opinion about probe count. Checks whose input a run did not produce are marked `measured: false` at `info` severity and listed in the verdict's `not_measured` (rendered as `PASS (k not measured)`) rather than passed silently, so a skipped stage never reads as a -cleared one; failing `warn`-severity checks and the artifact's own `warnings` also surface in the +cleared one. +A cell with no check measured at all is `NOT MEASURED`, not `PASS`. +Failing `warn`-severity checks and the artifact's own `warnings` also surface in the scorecard's top-level `warnings`. ## Cost, and how it is kept down diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index e5be783d..29ccf521 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -165,22 +165,7 @@ def cmd_baseline_list(args: argparse.Namespace) -> None: ) -def cmd_baseline_compare(args: argparse.Namespace) -> None: - """Compare an eval result against a saved baseline.""" - eval_path = Path(args.eval) - if not eval_path.exists(): - print(f"❌ Eval file not found: {eval_path}") - raise SystemExit(1) - - eval_data = json.loads(eval_path.read_text(encoding="utf-8")) - manifest = _baseline_manifest() - baselines = manifest.get("baselines", []) - - if not baselines: - print("No baselines found. Save one first:") - print(" python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB") - raise SystemExit(1) - +def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> dict: # Find baseline: by name or latest for same product if args.baseline: entry = next((b for b in baselines if b["name"] == args.baseline), None) @@ -193,20 +178,10 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: if not candidates: candidates = baselines entry = candidates[-1] # most recent + return entry - baseline_data = json.loads(Path(entry["path"]).read_text(encoding="utf-8")) - - from dokimi.records import RecordError, records - - # Read both record lists before the model checks: a malformed file must exit 2, not reach the --strict verdict. - try: - baseline_evals = {e.key: e for e in records(baseline_data.get("evaluations", []), - source=entry["path"], kind="evaluations")} - current_evals = records(eval_data.get("evaluations", []), source=str(eval_path), kind="evaluations") - except RecordError as exc: - print(f"❌ {exc}", file=sys.stderr) - raise SystemExit(2) from exc # 1 is the --strict verdict +def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: dict, eval_data: dict) -> None: mismatches, unverified = _model_mismatches(baseline_data, eval_data) strict = getattr(args, "strict", False) for role in unverified: @@ -227,9 +202,56 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: f" the changes below mix the model change with the content change.{detail}", file=sys.stderr) + +def _print_metric_table(cur_summary: dict, base_summary: dict) -> None: + print(f"{'Metric':<20} {'Baseline':>10} {'Current':>10} {'Change':>8}") + print("─" * 52) + for k, label in [("avg_with", "Context avg"), ("avg_without", "Baseline avg"), ("avg_delta", "Avg delta")]: + b_val = base_summary.get(k, "?") + c_val = cur_summary.get(k, "?") + try: + change = f"{c_val - b_val:+.2f}" + except TypeError: + change = "?" + print(f"{label:<20} {str(b_val):>10} {str(c_val):>10} {change:>8}") + + +def cmd_baseline_compare(args: argparse.Namespace) -> None: + """Compare an eval result against a saved baseline.""" + eval_path = Path(args.eval) + if not eval_path.exists(): + print(f"❌ Eval file not found: {eval_path}") + raise SystemExit(1) + + eval_data = json.loads(eval_path.read_text(encoding="utf-8")) + manifest = _baseline_manifest() + baselines = manifest.get("baselines", []) + + if not baselines: + print("No baselines found. Save one first:") + print(" python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB") + raise SystemExit(1) + + entry = _pick_entry(args, baselines, eval_data) + baseline_data = json.loads(Path(entry["path"]).read_text(encoding="utf-8")) + + from dokimi.records import RecordError, records + + # Read both record lists before the model checks: a malformed file must exit 2, not reach the --strict verdict. + try: + baseline_evals = {e.key: e for e in records(baseline_data.get("evaluations", []), + source=entry["path"], kind="evaluations")} + current_evals = records(eval_data.get("evaluations", []), source=str(eval_path), kind="evaluations") + except RecordError as exc: + print(f"❌ {exc}", file=sys.stderr) + raise SystemExit(2) from exc # 1 is the --strict verdict + + _report_model_checks(args, entry, baseline_data, eval_data) + print(f"Comparing against baseline: {entry['name']} (saved {entry.get('saved_at', '?')})\n") deltas_changed = [] + n_compared = 0 # questions scored on both sides: the only ones a "no change" can be about for e in current_evals: q_id = e.question_id base_e = baseline_evals.get(e.key) @@ -238,6 +260,7 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: cur_delta = e.delta base_delta = base_e.delta if cur_delta is not None and base_delta is not None: + n_compared += 1 moved = cur_delta - base_delta if abs(moved) >= 1: deltas_changed.append((q_id, base_delta, cur_delta, moved, e.question_text[:60])) @@ -246,16 +269,7 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: cur_summary = _compute_summary(eval_data) base_summary = entry.get("summary", {}) - print(f"{'Metric':<20} {'Baseline':>10} {'Current':>10} {'Change':>8}") - print("─" * 52) - for k, label in [("avg_with", "Context avg"), ("avg_without", "Baseline avg"), ("avg_delta", "Avg delta")]: - b_val = base_summary.get(k, "?") - c_val = cur_summary.get(k, "?") - try: - change = f"{c_val - b_val:+.2f}" - except TypeError: - change = "?" - print(f"{label:<20} {str(b_val):>10} {str(c_val):>10} {change:>8}") + _print_metric_table(cur_summary, base_summary) if deltas_changed: print("\nQuestions with notable changes (|Δ| ≥ 1):") @@ -263,8 +277,11 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: print(" " + "─" * 80) for q_id, bd, cd, ch, txt in sorted(deltas_changed, key=lambda x: -abs(x[3])): print(f" {q_id:<20} {bd:>7} {cd:>7} {ch:>+7} {txt}") + elif not n_compared: + print(f"\n⚪ Nothing to compare (0 scored): no question is scored in both the baseline and " + f"the current eval ({len(current_evals)} current, {len(baseline_evals)} baseline evaluations).") else: - print("\n✅ No notable changes vs baseline.") + print(f"\n✅ No notable changes vs baseline ({n_compared} of {len(current_evals)} questions compared).") def register(sub, positive_int) -> None: diff --git a/dokimi/dokimi/commands/run.py b/dokimi/dokimi/commands/run.py index 82d1d66c..c385bbf8 100644 --- a/dokimi/dokimi/commands/run.py +++ b/dokimi/dokimi/commands/run.py @@ -13,7 +13,7 @@ from dokimi.report.json_report import write_json from dokimi.report.markdown_report import write_compare_report, write_run_report from dokimi.runner.compare import compare_snapshots -from dokimi.runner.run import run_benchmark, save_snapshot +from dokimi.runner.run import DocLoadError, run_benchmark, save_snapshot def cmd_run(args: argparse.Namespace) -> None: @@ -23,7 +23,11 @@ def cmd_run(args: argparse.Namespace) -> None: out_json = Path(args.out_json).resolve() out_md = Path(args.out_md).resolve() - result = run_benchmark(root, spec_path) + try: + result = run_benchmark(root, spec_path) + except DocLoadError as exc: + print(f"❌ {exc}", file=sys.stderr) + sys.exit(1) save_snapshot(result, out_json) write_run_report(result, out_md) print(json.dumps(result["summary"], indent=2)) diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index 5995aac9..0564c8fc 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -78,8 +78,8 @@ def location(name: str) -> str | None: for h in harnesses if h != baseline and (loc := location(h)) not in (None, base)] -def cmd_tasks_run(args: argparse.Namespace) -> None: - from dokimi.harnesses import PreflightError, TaskSuiteRunner, discover_tasks, load_task +def _select_tasks(args: argparse.Namespace) -> list: + from dokimi.harnesses import discover_tasks, load_task root = _tasks_root(args) try: @@ -92,6 +92,39 @@ def cmd_tasks_run(args: argparse.Namespace) -> None: raise SystemExit(str(exc)) from None if not tasks: raise SystemExit("No tasks selected. Pass --tasks or --all.") + return tasks + + +def _print_summary(output: dict) -> None: + for harness, stats in output["summary"]["per_harness"].items(): + rate = stats["pass_rate"] + rate_s = "n/a" if rate is None else f"{rate:.2%}" + cost = stats.get("cost", {}).get("total_cost_usd") + cost_s = "" if cost is None else f", cost=${cost:.4f}" + scored = stats.get("scored", stats["n"]) + errored_s = f", {stats['errored']} errored" if stats.get("errored") else "" + print( + f"{harness}: {stats['passed']}/{scored} pass ({rate_s}){errored_s}, " + f"ops={stats['operation_count']}, elapsed={stats['elapsed_sec']:.2f}s{cost_s}" + ) + for harness, cmp_row in output["summary"]["comparisons"].items(): + delta = cmp_row["pass_rate_delta"] + delta_s = "n/a" if delta is None else f"{delta:+.2%}" + cost_delta = cmp_row.get("cost_delta_usd") + cost_s = "" if cost_delta is None else f", cost delta ${cost_delta:+.4f}" + print(f"{harness} vs {cmp_row['baseline_harness']}: pass-rate delta {delta_s}{cost_s}") + per_harness = output["summary"]["per_harness"].values() + cells = sum(s["n"] for s in per_harness) + errored = sum(s.get("errored", 0) for s in per_harness) + if cells and errored == cells: # no cell has a verdict: "0/0 pass (n/a)" is not a result + print(f"✗ incomplete: every cell errored (0 scored, {errored} errored of {cells} cells)", file=sys.stderr) + sys.exit(1) + + +def cmd_tasks_run(args: argparse.Namespace) -> None: + from dokimi.harnesses import PreflightError, TaskSuiteRunner + + tasks = _select_tasks(args) harnesses = [h.strip() for h in args.harnesses.split(",") if h.strip()] if not harnesses: @@ -136,23 +169,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None: out_md.write_text(render_task_runs_report(output), encoding="utf-8") print(f"Saved task run report: {out_md}") - for harness, stats in output["summary"]["per_harness"].items(): - rate = stats["pass_rate"] - rate_s = "n/a" if rate is None else f"{rate:.2%}" - cost = stats.get("cost", {}).get("total_cost_usd") - cost_s = "" if cost is None else f", cost=${cost:.4f}" - scored = stats.get("scored", stats["n"]) - errored_s = f", {stats['errored']} errored" if stats.get("errored") else "" - print( - f"{harness}: {stats['passed']}/{scored} pass ({rate_s}){errored_s}, " - f"ops={stats['operation_count']}, elapsed={stats['elapsed_sec']:.2f}s{cost_s}" - ) - for harness, cmp_row in output["summary"]["comparisons"].items(): - delta = cmp_row["pass_rate_delta"] - delta_s = "n/a" if delta is None else f"{delta:+.2%}" - cost_delta = cmp_row.get("cost_delta_usd") - cost_s = "" if cost_delta is None else f", cost delta ${cost_delta:+.4f}" - print(f"{harness} vs {cmp_row['baseline_harness']}: pass-rate delta {delta_s}{cost_s}") + _print_summary(output) def register(sub, positive_int) -> None: diff --git a/dokimi/dokimi/gate/novelty_gate.py b/dokimi/dokimi/gate/novelty_gate.py index e37cd27f..05f135e4 100644 --- a/dokimi/dokimi/gate/novelty_gate.py +++ b/dokimi/dokimi/gate/novelty_gate.py @@ -125,10 +125,16 @@ def notes(self) -> list[NoveltyCheck]: @property def status(self) -> str: - """``PASS`` / ``WARN`` / ``FAIL``, without a build meaning attached.""" + """``PASS`` / ``WARN`` / ``FAIL`` / ``NOT MEASURED``, without a build meaning attached. + + ``NOT MEASURED`` when no check was measured: every check passes unmeasured, so + ``PASS`` would be read off a run that cleared nothing. + """ if self.blocking: return "FAIL" - return "WARN" if self.warnings else "PASS" + if self.warnings: + return "WARN" + return "PASS" if any(c.measured for c in self.checks) else "NOT MEASURED" @property def not_measured(self) -> list[str]: diff --git a/dokimi/dokimi/report/markdown_report.py b/dokimi/dokimi/report/markdown_report.py index c339d080..466ba6b3 100644 --- a/dokimi/dokimi/report/markdown_report.py +++ b/dokimi/dokimi/report/markdown_report.py @@ -65,13 +65,20 @@ def write_compare_report(compare_data: dict, path: Path) -> None: reg = compare_data["regressions"] lines.extend(["", "## Regression Analysis"]) - # Overall status + # Overall status. A snapshot with no doc scored nothing: a zero diff against it is not "no regressions". + unscored = [side for side in ("base", "candidate") + if side in compare_data and not compare_data[side].get("docs")] + nothing = f"⚪ **Nothing to compare (0 scored)**: no document in {' and '.join(unscored)}" if reg["has_critical"]: lines.append("🔴 **CRITICAL regressions detected**") elif reg["has_warnings"]: lines.append("🟡 **Warnings detected**") + elif unscored: + lines.append(nothing) else: lines.append("✅ **No regressions**") + if unscored and (reg["has_critical"] or reg["has_warnings"]): + lines.append(nothing) lines.append("") diff --git a/dokimi/dokimi/runner/run.py b/dokimi/dokimi/runner/run.py index 73633251..a4f99e85 100644 --- a/dokimi/dokimi/runner/run.py +++ b/dokimi/dokimi/runner/run.py @@ -69,6 +69,24 @@ def _load_spec(spec_path: Path) -> dict: return data +class DocLoadError(RuntimeError): + """A document the run selected could not be read: the run refuses rather than score around it.""" + + +def _load_found_docs(docs: list[Path]) -> dict[str, str]: + """``load_docs`` over the selected docs; a doc that does not load is a clean error, not a traceback.""" + try: + loaded = load_docs(docs) + except OSError as exc: + raise DocLoadError(f"could not read {exc.filename or 'a document'}: {exc.strerror or exc} " + f"({len(docs)} document(s) selected)") from None + unloaded = [str(p) for p in docs if str(p) not in loaded] + if unloaded: + raise DocLoadError(f"{len(unloaded)} of {len(docs)} selected document(s) did not load: " + + ", ".join(unloaded[:5]) + (" ..." if len(unloaded) > 5 else "")) + return loaded + + def run_benchmark(root: Path, spec_path: Path) -> dict: """Run benchmark on markdown docs and return snapshot payload.""" spec = _load_spec(spec_path) @@ -87,7 +105,7 @@ def run_benchmark(root: Path, spec_path: Path) -> dict: else: # Legacy/minimal specs without a manifest: discover all markdown under docs/. docs = discover_markdown(root / "docs") - loaded = load_docs(docs) + loaded = _load_found_docs(docs) # Collect per-doc metrics and cached example results rows: list[dict] = [] diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 14f2e7e8..b2cb4800 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -18,7 +18,7 @@ "functions": { "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, - "dokimi/commands/baseline.py::cmd_baseline_compare": {"complexity": 19, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/commands/baseline.py::cmd_baseline_compare": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22}, "dokimi/commands/catalog.py::cmd_catalog_scan": {"complexity": 12}, @@ -30,7 +30,6 @@ "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "grew on main after the #93 seed and before it merged; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, - "dokimi/commands/tasks.py::cmd_tasks_run": {"complexity": 11}, "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11}, "dokimi/context/trace.py::_apply": {"complexity": 18}, "dokimi/dashboard/aggregator.py::ResultsAggregator._load_snapshot": {"complexity": 13}, diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index ed57961b..72ffc373 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -205,3 +205,34 @@ def test_panel_score_records_the_answer_model_for_compare(tmp_path): data = json.loads(path.read_text(encoding="utf-8")) assert baseline_mod._recorded_models(data) == { "answer": ("openai", "a"), "judge": ("panel", "anthropic/m")} + + + +def _unscored() -> dict: + data = _eval() + data["evaluations"][0]["delta"] = None + return data + + +@pytest.mark.parametrize("current", [_unscored(), {**_eval(), "evaluations": []}], ids=["unscored", "empty"]) +def test_compare_with_nothing_scored_on_both_sides_is_not_no_change(compare, capsys, current): + # An unscored or empty current eval printed "✅ No notable changes vs baseline." + compare(_eval(), current) + out = capsys.readouterr().out + assert "Nothing to compare (0 scored)" in out + assert "No notable changes" not in out + + +def test_compare_with_a_scored_pair_and_no_move_says_how_many_it_compared(compare, capsys): + compare(_eval(), _eval(delta=2.5)) + assert "No notable changes vs baseline (1 of 1 questions compared)" in capsys.readouterr().out + + +def test_compare_counts_only_questions_scored_on_both_sides(compare, capsys): + base, cur = _eval(), _eval(delta=2.5) + extra = [{**cur["evaluations"][0], "question_id": f"extra{i}", "delta": 1.0} for i in range(2)] + for e in extra: + e.pop("key", None) + cur["evaluations"] += extra + compare(base, cur) + assert "(1 of 3 questions compared)" in capsys.readouterr().out diff --git a/dokimi/tests/test_novelty_gate.py b/dokimi/tests/test_novelty_gate.py index 1065210d..79640462 100644 --- a/dokimi/tests/test_novelty_gate.py +++ b/dokimi/tests/test_novelty_gate.py @@ -474,3 +474,16 @@ def test_a_capped_atom_set_does_not_clear_harmful_atoms(): "harmful_atoms") assert found.measured is True and found.passed is False assert "lower bound" in found.message + + +def test_an_artifact_with_no_check_measured_is_not_a_pass(): + # `{}` carries no stage: every check passed unmeasured and the cell read PASS. + empty = evaluate_novelty({}) + assert len(empty.not_measured) == len(empty.checks) > 0 + assert empty.status == "NOT MEASURED" + assert empty.as_dict()["status"] == "NOT MEASURED" + assert "— NOT MEASURED (10 of 10 not measured)" in format_novelty_block([empty]) + # One measured check is enough for PASS; the rest stay counted as not measured. + assert evaluate_novelty(_artifact( + categories={}, shortening=[], attribution={}, controls={}, spans={"n": 60}, + bundle={}, probes={}, items={})).status == "PASS" diff --git a/dokimi/tests/test_runner_and_report.py b/dokimi/tests/test_runner_and_report.py index f103c716..5870a6b2 100644 --- a/dokimi/tests/test_runner_and_report.py +++ b/dokimi/tests/test_runner_and_report.py @@ -244,12 +244,39 @@ def test_write_compare_report_no_regressions(tmp_path): assert "No regressions" in text +def test_write_compare_report_with_no_scored_doc_is_not_no_regressions(tmp_path): + # Two 0-doc snapshots diff to 0 everywhere: that read "No regressions". + out = tmp_path / "compare.md" + data = _compare_data(with_regressions=True) + data["base"], data["candidate"] = _snapshot(docs=0)["summary"], _snapshot(docs=0)["summary"] + write_compare_report(data, out) + text = out.read_text(encoding="utf-8") + assert "Nothing to compare (0 scored)**: no document in base and candidate" in text + assert "No regressions" not in text + + +@pytest.mark.parametrize(("severity", "headline"), [ + ("has_critical", "🔴 **CRITICAL regressions detected**"), + ("has_warnings", "🟡 **Warnings detected**"), +], ids=["critical", "warn"]) +def test_write_compare_report_empty_candidate_keeps_the_headline(tmp_path, severity, headline): + # A candidate that lost every document is a regression; "Nothing to compare" must not replace the headline. + out = tmp_path / "compare.md" + data = _compare_data(with_regressions=True, **{severity: True}) + data["candidate"] = _snapshot(docs=0, score=0.0)["summary"] + write_compare_report(data, out) + lines = out.read_text(encoding="utf-8").splitlines() + assert headline in lines + assert any("Nothing to compare (0 scored)**: no document in candidate" in ln for ln in lines) + + def test_write_compare_report_critical_regressions(tmp_path): out = tmp_path / "compare.md" write_compare_report(_compare_data(with_regressions=True, has_critical=True), out) text = out.read_text(encoding="utf-8") assert "CRITICAL" in text assert "🔴" in text + assert "Nothing to compare" not in text def test_write_compare_report_warnings(tmp_path): @@ -257,6 +284,7 @@ def test_write_compare_report_warnings(tmp_path): write_compare_report(_compare_data(with_regressions=True, has_warnings=True), out) text = out.read_text(encoding="utf-8") assert "Warnings" in text or "🟡" in text + assert "Nothing to compare" not in text def test_write_compare_report_creates_parent_dirs(tmp_path): diff --git a/dokimi/tests/test_runner_run.py b/dokimi/tests/test_runner_run.py index 6dc84f8a..764c4276 100644 --- a/dokimi/tests/test_runner_run.py +++ b/dokimi/tests/test_runner_run.py @@ -105,3 +105,44 @@ def test_save_snapshot(tmp_path): save_snapshot({"summary": {"docs": 0}}, out) assert out.exists() assert json.loads(out.read_text())["summary"]["docs"] == 0 + + +def _spec(tmp_path): + spec = tmp_path / "spec.yaml" + spec.write_text( + "weights:\n coverage: 0.4\n freshness_lite: 0.3\n readability: 0.3\n" + "metrics:\n freshness_lite:\n max_age_days: 365\n readability:\n grade_max: 12\n" + ) + return spec + + +def test_a_found_doc_that_did_not_load_is_a_clean_error(tmp_path): + from dokimi.runner.run import DocLoadError + + found = [tmp_path / "docs/a.md", tmp_path / "docs/b.md"] + # Missing from the loaded mapping: this was a KeyError on the first doc. + with patch("dokimi.runner.run.discover_markdown", return_value=found), \ + patch("dokimi.runner.run.load_docs", return_value={str(found[1]): "hello"}): + with pytest.raises(DocLoadError, match=r"1 of 2 selected document\(s\) did not load: .*a\.md"): + run_benchmark(tmp_path, _spec(tmp_path)) + # Unreadable: this was a PermissionError traceback out of load_docs. + denied = PermissionError(13, "Permission denied", str(found[0])) + with patch("dokimi.runner.run.discover_markdown", return_value=found), \ + patch("dokimi.ingest.loader.load_text", side_effect=denied): + with pytest.raises(DocLoadError, match=r"could not read .*a\.md: Permission denied \(2 document"): + run_benchmark(tmp_path, _spec(tmp_path)) + + +def test_run_command_exits_1_without_a_traceback_when_a_doc_does_not_load(tmp_path, capsys): + from dokimi.cli import build_parser + + found = [tmp_path / "docs/a.md"] + args = build_parser().parse_args(["run", "--root", str(tmp_path), "--spec", str(_spec(tmp_path)), + "--out-json", str(tmp_path / "o.json"), "--out-md", str(tmp_path / "o.md")]) + with patch("dokimi.runner.run.discover_markdown", return_value=found), \ + patch("dokimi.runner.run.load_docs", return_value={}): + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + assert "1 of 1 selected document(s) did not load" in capsys.readouterr().err + assert not (tmp_path / "o.json").exists() # no snapshot over the docs that did load diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py index 4acaaf58..de1204ea 100644 --- a/dokimi/tests/test_tasks_command.py +++ b/dokimi/tests/test_tasks_command.py @@ -17,6 +17,10 @@ def _fixture_task(tmp_path: Path) -> Path: task.mkdir() (task / "instruction.md").write_text("Do it.", encoding="utf-8") (task / "task.toml").write_text("[metadata]\ntags = [\"toy\"]\n", encoding="utf-8") + # A task Harbor can run: without these every cell errors and `tasks run` exits 1. + (task / "environment").mkdir() + (task / "tests").mkdir() + (task / "tests" / "test.sh").write_text("exit 0\n", encoding="utf-8") return task @@ -135,3 +139,44 @@ def test_tasks_list_prints_tasks_from_a_named_root(tmp_path, capsys): out = capsys.readouterr().out assert "toy-task" in out assert "toy" in out + + +class _RaisingHarness: + def run(self, task, **_): + raise RuntimeError("docker build failed") + + +def _run_tasks(tmp_path, monkeypatch, harness_for): + from dokimi.harnesses import runner as runner_mod + + _fixture_task(tmp_path) + real = runner_mod.create_harness + monkeypatch.setattr(runner_mod, "create_harness", + lambda hid, **kw: harness_for(hid) or real(hid, **kw)) + args = build_parser().parse_args([ + "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path), + "--harnesses", "codex", "--baseline-harness", "claude-code", "--model", "m", + "--out-json", str(tmp_path / "runs.json"), "--dry-run", + ]) + try: + args.func(args) + except SystemExit as exc: + return exc.code + return 0 + + +def test_tasks_run_with_every_cell_errored_is_incomplete(tmp_path, monkeypatch, capsys): + # "codex: 0/0 pass (n/a), 1 errored" was exit 0. + code = _run_tasks(tmp_path, monkeypatch, lambda hid: _RaisingHarness()) + out, err = capsys.readouterr() + assert code == 1 + assert "codex: 0/0 pass (n/a), 1 errored" in out + assert "incomplete: every cell errored (0 scored, 2 errored of 2 cells)" in err + assert (tmp_path / "runs.json").is_file() # the artifact is still written + + +def test_tasks_run_with_some_cells_errored_keeps_exit_0_and_counts(tmp_path, monkeypatch, capsys): + code = _run_tasks(tmp_path, monkeypatch, lambda hid: _RaisingHarness() if hid == "codex" else None) + out, err = capsys.readouterr() + assert code == 0 + assert "codex: 0/0 pass (n/a), 1 errored" in out and "incomplete" not in err