fix: nothing measured is not a pass, second batch (tasks run, novelty, run, compares)

dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- **Nothing measured is not a pass (second batch)** — `tasks run` exits 1 as `incomplete` when every cell errored (the artifact and report are still written; partly errored runs keep exit 0 and their counts). The novelty gate's status is `NOT MEASURED`, not `PASS`, for a cell with no check measured. `run` refuses with one line and exit 1 when a selected document does not load, instead of a KeyError or OSError traceback. `baseline compare` says "Nothing to compare (0 scored)" when no question is scored on both sides, and its "No notable changes" line counts the questions compared; the snapshot `compare` report says "Nothing to compare (0 scored)" instead of "No regressions" when either snapshot has no document.

dokimi/docs/coding-harnesses.md

@@ -446,7 +446,10 @@ a verdict. The rules mirror intel/skills' `tools/check_harbor_job.py`:
 | no trial file (with or without the job's), more than one trial, or not a JSON object | errored |
 
 An errored cell is out of the pass-rate denominator and the paired test, as
-for `docker-*` and BenchFlow. `metrics.n_trials` counts the trial files and
+for `docker-*` and BenchFlow. When every cell of a `tasks run` errored, it
+still writes the artifact and the report, then exits 1 as `incomplete`
+(`0 scored, N errored of N cells`); a run with some cells scored exits 0 and
+prints the errored count per harness. `metrics.n_trials` counts the trial files and
 `metrics.result_json` names the one read. A `harbor` that `dokimi` kills at
 the task budget is a failed cell (reward 0, returncode 124), as above.
 

dokimi/docs/skill-novelty.md

@@ -755,7 +755,9 @@ a threshold on `added` flipped its verdict on up to 24% of artifacts at a 0.25 p
 blocking novelty check would be a blocking opinion about probe count. Checks whose input a run did
 not produce are marked `measured: false` at `info` severity and listed in the verdict's `not_measured`
 (rendered as `PASS (k not measured)`) rather than passed silently, so a skipped stage never reads as a
-cleared one; failing `warn`-severity checks and the artifact's own `warnings` also surface in the
+cleared one.
+A cell with no check measured at all is `NOT MEASURED`, not `PASS`.
+Failing `warn`-severity checks and the artifact's own `warnings` also surface in the
 scorecard's top-level `warnings`.
 
 ## Cost, and how it is kept down

dokimi/dokimi/commands/baseline.py

@@ -165,22 +165,7 @@ def cmd_baseline_list(args: argparse.Namespace) -> None:
         )
 
 
-def cmd_baseline_compare(args: argparse.Namespace) -> None:
-    """Compare an eval result against a saved baseline."""
-    eval_path = Path(args.eval)
-    if not eval_path.exists():
-        print(f"❌ Eval file not found: {eval_path}")
-        raise SystemExit(1)
-
-    eval_data = json.loads(eval_path.read_text(encoding="utf-8"))
-    manifest = _baseline_manifest()
-    baselines = manifest.get("baselines", [])
-
-    if not baselines:
-        print("No baselines found. Save one first:")
-        print("  python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB")
-        raise SystemExit(1)
-
+def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> dict:
     # Find baseline: by name or latest for same product
     if args.baseline:
         entry = next((b for b in baselines if b["name"] == args.baseline), None)
@@ -193,20 +178,10 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None:
         if not candidates:
             candidates = baselines
         entry = candidates[-1]   # most recent
+    return entry
 
-    baseline_data = json.loads(Path(entry["path"]).read_text(encoding="utf-8"))
-
-    from dokimi.records import RecordError, records
-
-    # Read both record lists before the model checks: a malformed file must exit 2, not reach the --strict verdict.
-    try:
-        baseline_evals = {e.key: e for e in records(baseline_data.get("evaluations", []),
-                                                    source=entry["path"], kind="evaluations")}
-        current_evals = records(eval_data.get("evaluations", []), source=str(eval_path), kind="evaluations")
-    except RecordError as exc:
-        print(f"❌ {exc}", file=sys.stderr)
-        raise SystemExit(2) from exc  # 1 is the --strict verdict
 
+def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: dict, eval_data: dict) -> None:
     mismatches, unverified = _model_mismatches(baseline_data, eval_data)
     strict = getattr(args, "strict", False)
     for role in unverified:
@@ -227,9 +202,56 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None:
               f" the changes below mix the model change with the content change.{detail}",
               file=sys.stderr)
 
+
+def _print_metric_table(cur_summary: dict, base_summary: dict) -> None:
+    print(f"{'Metric':<20} {'Baseline':>10} {'Current':>10} {'Change':>8}")
+    print("─" * 52)
+    for k, label in [("avg_with", "Context avg"), ("avg_without", "Baseline avg"), ("avg_delta", "Avg delta")]:
+        b_val = base_summary.get(k, "?")
+        c_val = cur_summary.get(k, "?")
+        try:
+            change = f"{c_val - b_val:+.2f}"
+        except TypeError:
+            change = "?"
+        print(f"{label:<20} {str(b_val):>10} {str(c_val):>10} {change:>8}")
+
+
+def cmd_baseline_compare(args: argparse.Namespace) -> None:
+    """Compare an eval result against a saved baseline."""
+    eval_path = Path(args.eval)
+    if not eval_path.exists():
+        print(f"❌ Eval file not found: {eval_path}")
+        raise SystemExit(1)
+
+    eval_data = json.loads(eval_path.read_text(encoding="utf-8"))
+    manifest = _baseline_manifest()
+    baselines = manifest.get("baselines", [])
+
+    if not baselines:
+        print("No baselines found. Save one first:")
+        print("  python cli.py baseline save --from-eval eval/oneTBB.json --product oneTBB")
+        raise SystemExit(1)
+
+    entry = _pick_entry(args, baselines, eval_data)
+    baseline_data = json.loads(Path(entry["path"]).read_text(encoding="utf-8"))
+
+    from dokimi.records import RecordError, records
+
+    # Read both record lists before the model checks: a malformed file must exit 2, not reach the --strict verdict.
+    try:
+        baseline_evals = {e.key: e for e in records(baseline_data.get("evaluations", []),
+                                                    source=entry["path"], kind="evaluations")}
+        current_evals = records(eval_data.get("evaluations", []), source=str(eval_path), kind="evaluations")
+    except RecordError as exc:
+        print(f"❌ {exc}", file=sys.stderr)
+        raise SystemExit(2) from exc  # 1 is the --strict verdict
+
+    _report_model_checks(args, entry, baseline_data, eval_data)
+
     print(f"Comparing against baseline: {entry['name']} (saved {entry.get('saved_at', '?')})\n")
 
     deltas_changed = []
+    n_compared = 0  # questions scored on both sides: the only ones a "no change" can be about
     for e in current_evals:
         q_id = e.question_id
         base_e = baseline_evals.get(e.key)
@@ -238,6 +260,7 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None:
         cur_delta = e.delta
         base_delta = base_e.delta
         if cur_delta is not None and base_delta is not None:
+            n_compared += 1
             moved = cur_delta - base_delta
             if abs(moved) >= 1:
                 deltas_changed.append((q_id, base_delta, cur_delta, moved, e.question_text[:60]))
@@ -246,25 +269,19 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None:
     cur_summary = _compute_summary(eval_data)
     base_summary = entry.get("summary", {})
 
-    print(f"{'Metric':<20} {'Baseline':>10} {'Current':>10} {'Change':>8}")
-    print("─" * 52)
-    for k, label in [("avg_with", "Context avg"), ("avg_without", "Baseline avg"), ("avg_delta", "Avg delta")]:
-        b_val = base_summary.get(k, "?")
-        c_val = cur_summary.get(k, "?")
-        try:
-            change = f"{c_val - b_val:+.2f}"
-        except TypeError:
-            change = "?"
-        print(f"{label:<20} {str(b_val):>10} {str(c_val):>10} {change:>8}")
+    _print_metric_table(cur_summary, base_summary)
 
     if deltas_changed:
         print("\nQuestions with notable changes (|Δ| ≥ 1):")
         print(f"  {'ID':<20} {'base_δ':>7} {'cur_δ':>7} {'change':>7}  Question")
         print("  " + "─" * 80)
         for q_id, bd, cd, ch, txt in sorted(deltas_changed, key=lambda x: -abs(x[3])):
             print(f"  {q_id:<20} {bd:>7} {cd:>7} {ch:>+7}  {txt}")
+    elif not n_compared:
+        print(f"\n⚪ Nothing to compare (0 scored): no question is scored in both the baseline and "
+              f"the current eval ({len(current_evals)} current, {len(baseline_evals)} baseline evaluations).")
     else:
-        print("\n✅ No notable changes vs baseline.")
+        print(f"\n✅ No notable changes vs baseline ({n_compared} of {len(current_evals)} questions compared).")
 
 
 def register(sub, positive_int) -> None:

dokimi/dokimi/commands/run.py

@@ -13,7 +13,7 @@
 from dokimi.report.json_report import write_json
 from dokimi.report.markdown_report import write_compare_report, write_run_report
 from dokimi.runner.compare import compare_snapshots
-from dokimi.runner.run import run_benchmark, save_snapshot
+from dokimi.runner.run import DocLoadError, run_benchmark, save_snapshot
 
 
 def cmd_run(args: argparse.Namespace) -> None:
@@ -23,7 +23,11 @@ def cmd_run(args: argparse.Namespace) -> None:
     out_json = Path(args.out_json).resolve()
     out_md = Path(args.out_md).resolve()
 
-    result = run_benchmark(root, spec_path)
+    try:
+        result = run_benchmark(root, spec_path)
+    except DocLoadError as exc:
+        print(f"❌ {exc}", file=sys.stderr)
+        sys.exit(1)
     save_snapshot(result, out_json)
     write_run_report(result, out_md)
     print(json.dumps(result["summary"], indent=2))

dokimi/dokimi/commands/tasks.py

@@ -78,8 +78,8 @@ def location(name: str) -> str | None:
             for h in harnesses if h != baseline and (loc := location(h)) not in (None, base)]
 
 
-def cmd_tasks_run(args: argparse.Namespace) -> None:
-    from dokimi.harnesses import PreflightError, TaskSuiteRunner, discover_tasks, load_task
+def _select_tasks(args: argparse.Namespace) -> list:
+    from dokimi.harnesses import discover_tasks, load_task
 
     root = _tasks_root(args)
     try:
@@ -92,6 +92,39 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
         raise SystemExit(str(exc)) from None
     if not tasks:
         raise SystemExit("No tasks selected. Pass --tasks <name[,name]> or --all.")
+    return tasks
+
+
+def _print_summary(output: dict) -> None:
+    for harness, stats in output["summary"]["per_harness"].items():
+        rate = stats["pass_rate"]
+        rate_s = "n/a" if rate is None else f"{rate:.2%}"
+        cost = stats.get("cost", {}).get("total_cost_usd")
+        cost_s = "" if cost is None else f", cost=${cost:.4f}"
+        scored = stats.get("scored", stats["n"])
+        errored_s = f", {stats['errored']} errored" if stats.get("errored") else ""
+        print(
+            f"{harness}: {stats['passed']}/{scored} pass ({rate_s}){errored_s}, "
+            f"ops={stats['operation_count']}, elapsed={stats['elapsed_sec']:.2f}s{cost_s}"
+        )
+    for harness, cmp_row in output["summary"]["comparisons"].items():
+        delta = cmp_row["pass_rate_delta"]
+        delta_s = "n/a" if delta is None else f"{delta:+.2%}"
+        cost_delta = cmp_row.get("cost_delta_usd")
+        cost_s = "" if cost_delta is None else f", cost delta ${cost_delta:+.4f}"
+        print(f"{harness} vs {cmp_row['baseline_harness']}: pass-rate delta {delta_s}{cost_s}")
+    per_harness = output["summary"]["per_harness"].values()
+    cells = sum(s["n"] for s in per_harness)
+    errored = sum(s.get("errored", 0) for s in per_harness)
+    if cells and errored == cells:  # no cell has a verdict: "0/0 pass (n/a)" is not a result
+        print(f"✗ incomplete: every cell errored (0 scored, {errored} errored of {cells} cells)", file=sys.stderr)
+        sys.exit(1)
+
+
+def cmd_tasks_run(args: argparse.Namespace) -> None:
+    from dokimi.harnesses import PreflightError, TaskSuiteRunner
+
+    tasks = _select_tasks(args)
 
     harnesses = [h.strip() for h in args.harnesses.split(",") if h.strip()]
     if not harnesses:
@@ -136,23 +169,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None:
     out_md.write_text(render_task_runs_report(output), encoding="utf-8")
     print(f"Saved task run report:   {out_md}")
 
-    for harness, stats in output["summary"]["per_harness"].items():
-        rate = stats["pass_rate"]
-        rate_s = "n/a" if rate is None else f"{rate:.2%}"
-        cost = stats.get("cost", {}).get("total_cost_usd")
-        cost_s = "" if cost is None else f", cost=${cost:.4f}"
-        scored = stats.get("scored", stats["n"])
-        errored_s = f", {stats['errored']} errored" if stats.get("errored") else ""
-        print(
-            f"{harness}: {stats['passed']}/{scored} pass ({rate_s}){errored_s}, "
-            f"ops={stats['operation_count']}, elapsed={stats['elapsed_sec']:.2f}s{cost_s}"
-        )
-    for harness, cmp_row in output["summary"]["comparisons"].items():
-        delta = cmp_row["pass_rate_delta"]
-        delta_s = "n/a" if delta is None else f"{delta:+.2%}"
-        cost_delta = cmp_row.get("cost_delta_usd")
-        cost_s = "" if cost_delta is None else f", cost delta ${cost_delta:+.4f}"
-        print(f"{harness} vs {cmp_row['baseline_harness']}: pass-rate delta {delta_s}{cost_s}")
+    _print_summary(output)
 
 
 def register(sub, positive_int) -> None:

dokimi/dokimi/gate/novelty_gate.py

@@ -125,10 +125,16 @@ def notes(self) -> list[NoveltyCheck]:
 
     @property
     def status(self) -> str:
-        """``PASS`` / ``WARN`` / ``FAIL``, without a build meaning attached."""
+        """``PASS`` / ``WARN`` / ``FAIL`` / ``NOT MEASURED``, without a build meaning attached.
+
+        ``NOT MEASURED`` when no check was measured: every check passes unmeasured, so
+        ``PASS`` would be read off a run that cleared nothing.
+        """
         if self.blocking:
             return "FAIL"
-        return "WARN" if self.warnings else "PASS"
+        if self.warnings:
+            return "WARN"
+        return "PASS" if any(c.measured for c in self.checks) else "NOT MEASURED"
 
     @property
     def not_measured(self) -> list[str]:

dokimi/dokimi/report/markdown_report.py

@@ -65,13 +65,20 @@ def write_compare_report(compare_data: dict, path: Path) -> None:
         reg = compare_data["regressions"]
         lines.extend(["", "## Regression Analysis"])
 
-        # Overall status
+        # Overall status. A snapshot with no doc scored nothing: a zero diff against it is not "no regressions".
+        unscored = [side for side in ("base", "candidate")
+                    if side in compare_data and not compare_data[side].get("docs")]
+        nothing = f"⚪ **Nothing to compare (0 scored)**: no document in {' and '.join(unscored)}"
         if reg["has_critical"]:
             lines.append("🔴 **CRITICAL regressions detected**")
         elif reg["has_warnings"]:
             lines.append("🟡 **Warnings detected**")
+        elif unscored:
+            lines.append(nothing)
         else:
             lines.append("✅ **No regressions**")
+        if unscored and (reg["has_critical"] or reg["has_warnings"]):
+            lines.append(nothing)
 
         lines.append("")
 

dokimi/dokimi/runner/run.py

@@ -69,6 +69,24 @@ def _load_spec(spec_path: Path) -> dict:
     return data
 
 
+class DocLoadError(RuntimeError):
+    """A document the run selected could not be read: the run refuses rather than score around it."""
+
+
+def _load_found_docs(docs: list[Path]) -> dict[str, str]:
+    """``load_docs`` over the selected docs; a doc that does not load is a clean error, not a traceback."""
+    try:
+        loaded = load_docs(docs)
+    except OSError as exc:
+        raise DocLoadError(f"could not read {exc.filename or 'a document'}: {exc.strerror or exc} "
+                           f"({len(docs)} document(s) selected)") from None
+    unloaded = [str(p) for p in docs if str(p) not in loaded]
+    if unloaded:
+        raise DocLoadError(f"{len(unloaded)} of {len(docs)} selected document(s) did not load: "
+                           + ", ".join(unloaded[:5]) + (" ..." if len(unloaded) > 5 else ""))
+    return loaded
+
+
 def run_benchmark(root: Path, spec_path: Path) -> dict:
     """Run benchmark on markdown docs and return snapshot payload."""
     spec = _load_spec(spec_path)
@@ -87,7 +105,7 @@ def run_benchmark(root: Path, spec_path: Path) -> dict:
     else:
         # Legacy/minimal specs without a manifest: discover all markdown under docs/.
         docs = discover_markdown(root / "docs")
-    loaded = load_docs(docs)
+    loaded = _load_found_docs(docs)
 
     # Collect per-doc metrics and cached example results
     rows: list[dict] = []

dokimi/scripts/growth_baseline.json

@@ -18,7 +18,7 @@
   "functions": {
     "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13},
     "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11},
-    "dokimi/commands/baseline.py::cmd_baseline_compare": {"complexity": 19, "rationale": "pre-existing when the gate was introduced"},
+    "dokimi/commands/baseline.py::cmd_baseline_compare": {"complexity": 11, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11},
     "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 22},
     "dokimi/commands/catalog.py::cmd_catalog_scan": {"complexity": 12},
@@ -30,7 +30,6 @@
     "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"},
     "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "grew on main after the #93 seed and before it merged; split it in its own PR"},
     "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12},
-    "dokimi/commands/tasks.py::cmd_tasks_run": {"complexity": 11},
     "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11},
     "dokimi/context/trace.py::_apply": {"complexity": 18},
     "dokimi/dashboard/aggregator.py::ResultsAggregator._load_snapshot": {"complexity": 13},

dokimi/tests/test_baseline_compare.py

@@ -205,3 +205,34 @@ def test_panel_score_records_the_answer_model_for_compare(tmp_path):
     data = json.loads(path.read_text(encoding="utf-8"))
     assert baseline_mod._recorded_models(data) == {
         "answer": ("openai", "a"), "judge": ("panel", "anthropic/m")}
+
+
+
+def _unscored() -> dict:
+    data = _eval()
+    data["evaluations"][0]["delta"] = None
+    return data
+
+
+@pytest.mark.parametrize("current", [_unscored(), {**_eval(), "evaluations": []}], ids=["unscored", "empty"])
+def test_compare_with_nothing_scored_on_both_sides_is_not_no_change(compare, capsys, current):
+    # An unscored or empty current eval printed "✅ No notable changes vs baseline."
+    compare(_eval(), current)
+    out = capsys.readouterr().out
+    assert "Nothing to compare (0 scored)" in out
+    assert "No notable changes" not in out
+
+
+def test_compare_with_a_scored_pair_and_no_move_says_how_many_it_compared(compare, capsys):
+    compare(_eval(), _eval(delta=2.5))
+    assert "No notable changes vs baseline (1 of 1 questions compared)" in capsys.readouterr().out
+
+
+def test_compare_counts_only_questions_scored_on_both_sides(compare, capsys):
+    base, cur = _eval(), _eval(delta=2.5)
+    extra = [{**cur["evaluations"][0], "question_id": f"extra{i}", "delta": 1.0} for i in range(2)]
+    for e in extra:
+        e.pop("key", None)
+    cur["evaluations"] += extra
+    compare(base, cur)
+    assert "(1 of 3 questions compared)" in capsys.readouterr().out