fix(routing): triage queries skip code fences, zero-query runs say why

dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md

@@ -0,0 +1,17 @@
+### Fixed
+
+- **`novelty routing --queries-from-headings`** no longer reads a `##` line inside a
+  fenced code block as a request, so a shell comment does not become a query.
+- **`novelty routing` with no usable query** says why and suggests `--queries`, and
+  writes the result with `"status": "no_queries"` (the reason is in `reason`), where it
+  used to exit with a bare "no usable queries". A successful result carries `"status": "ok"`.
+- **Every `novelty routing` rate line** prints its `n_queries`, and a self-derived rate
+  carries `triage: self-derived` on the same line. A run that scores fewer than five
+  queries prints a warning, since one query moves a rate by 20 points.
+
+### Changed
+
+- **`novelty routing` exits 1 when no query is scored** — a run whose queries are non-empty
+  but all filtered out (no content word left) now exits non-zero with `"status": "no_queries"`
+  and a `reason`, where it used to exit 0 with null rates. A script that read the null rates
+  as a result must check the exit code or `status` instead.

dokimi/docs/skill-novelty.md

@@ -622,7 +622,11 @@ Three properties worth knowing before quoting either:
 - **The queries must come from outside the artifact**, or it defines both the problem scope
   and its own relevance. `--queries-from-headings` exists for triage, warns, and stamps
   `queries_self_derived: true` into the result; corpus-wide those self-derived queries gave
-  recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667.
+  recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667. Headings inside
+  fenced code blocks are not offered as queries. A run with no usable query exits non-zero,
+  says why, and still writes the result with `"status": "no_queries"`. Every rate line
+  prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer
+  than five scored queries draw a warning.
 - **The model figure is an upper bound on harness behaviour.** A harness whose own skill
   listing truncates or omits descriptions selects on less text than this command shows —
   which has been observed. Only a run inside the harness settles it.

dokimi/dokimi/commands/novelty.py

@@ -17,7 +17,7 @@
 import math
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, NoReturn
 
 from dokimi.defaults import (
     DEFAULT_ANSWER_MODEL,
@@ -320,14 +320,17 @@ def card(ref: str) -> tuple[Candidate, str]:
             "pass --queries <file> with requests written independently of the artifact, or "
             "--queries-from-headings for a self-derived triage run"
         )
-    if not queries:
-        raise SystemExit("no usable queries")
-
     candidates = [gold, *competitors]
     lexical = dict(lexical_routing(candidates, queries, gold=0),
                    queries_self_derived=self_derived)
+    out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    if not lexical["n_queries_scored"]:
+        _refuse_no_queries(args, queries, self_derived, out, len(candidates))
+
     payload = {
         "artifact": args.artifact,
+        "status": "ok",
         "n_candidates": len(candidates),
         "queries_self_derived": self_derived,
         "lexical": lexical,
@@ -339,28 +342,61 @@ def card(ref: str) -> tuple[Candidate, str]:
             queries_self_derived=self_derived,
         )
 
-    out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json"
-    out.parent.mkdir(parents=True, exist_ok=True)
     out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
     print(f"OK Saved routing measurement: {out}")
 
+    _print_routing_summary(payload, len(competitors), self_derived)
+
+def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None:
+    """The lexical line, the model line and their caveats, printed after a routing run."""
+    # Every rate is printed with the number of queries behind it, and a self-derived one
+    # carries the triage tag on the same line, so a copied figure cannot lose either.
+    tag = "  triage: self-derived" if self_derived else ""
     lexical = payload["lexical"]
     print(f"lexical  R@1 {lexical['recall_at_1']}  R@3 {lexical['recall_at_3']}  "
-          f"MRR {lexical['mrr']}  over {lexical['n_queries_scored']} of "
-          f"{lexical['n_queries_given']} queries vs {len(competitors)} competitors"
+          f"MRR {lexical['mrr']}  n_queries {lexical['n_queries_scored']} of "
+          f"{lexical['n_queries_given']} given, vs {n_competitors} competitors{tag}"
           + ("  (R@3 needs more than 3 candidates to mean anything)"
              if lexical["recall_at_3"] is None else ""))
+    if lexical["n_queries_scored"] < 5:
+        n_scored = lexical["n_queries_scored"]
+        noun = "query" if n_scored == 1 else "queries"
+        print(f"WARNING: only {n_scored} {noun} scored; one query moves "
+              "a rate by 20 points or more, so do not quote it as a routing rate")
     if "model" in payload:
         row = payload["model"]
         print(f"model    selection {row['selection_rate']}  "
               f"({row['selection_rate_among_answered']} among answered; "
-              f"{row['n_abstained']} abstained, {row['n_failed']} failed)  [{row['model']}]")
+              f"{row['n_abstained']} abstained, {row['n_failed']} failed)  "
+              f"[{row['model']}]  n_queries {row['n_queries']} given{tag}")
         print("The model figure is an upper bound on harness behaviour: a harness that "
               "truncates descriptions in its own listing selects on less text.")
     if self_derived:
         print("Self-derived queries: read as triage only.")
 
 
+
+def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool,
+                       out: Path, n_candidates: int) -> NoReturn:
+    """Write the ``no_queries`` record, then exit saying why no query was scored."""
+    # Written too, so a batch run leaves a record of why it has no rate.
+    if not queries and self_derived:
+        reason = (f"no section heading of two or more words outside code fences in "
+                  f"{args.artifact}; headings inside fenced code are not requests")
+    elif not queries:
+        reason = f"--queries {args.queries} holds no non-blank query"
+    else:
+        reason = (f"none of the {len(queries)} queries has a content word, since words "
+                  "like a, the, how do not count")
+    out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries",
+                               "reason": reason, "queries_self_derived": self_derived,
+                               "n_candidates": n_candidates}, indent=2),
+                   encoding="utf-8")
+    raise SystemExit(f"no usable queries: {reason}. Pass --queries <file> with requests "
+                     f"written independently of the artifact, one per line or a JSON "
+                     f"list. Wrote {out} with status no_queries")
+
+
 def cmd_novelty_shorten(args: argparse.Namespace) -> None:
     """Score shortening candidates on the five axes and optionally write the best one."""
     from dokimi.novelty.analyze import load_artifact_text

dokimi/dokimi/novelty/routing.py

@@ -309,13 +309,39 @@ def model_routing(
     }
 
 
+# Only spaces indent a fence (a tab is four columns, so it makes indented code), and a
+# backtick opener may not carry a backtick in its info string; both are CommonMark rules.
+_FENCE = re.compile(r" {0,3}(`{3,}(?!.*`)|~{3,})")
+
+
+def _outside_fences(body: str) -> str:
+    """``body`` with its fenced code blocks removed, so a ``## comment`` in a shell block is
+    not read as a heading. A fence closes on the same character, at least as long, alone on
+    its line; an unclosed fence runs to the end of the body, as CommonMark has it.
+    """
+    kept: list[str] = []
+    opening: str | None = None
+    for line in body.splitlines():
+        fence = _FENCE.match(line)
+        if opening is None:
+            if fence:
+                opening = fence.group(1)
+            else:
+                kept.append(line)
+        elif (fence and fence.group(1)[0] == opening[0] and len(fence.group(1)) >= len(opening)
+              and not line.strip()[len(fence.group(1)):]):
+            opening = None
+    return "\n".join(kept)
+
+
 def headings_as_queries(body: str, *, limit: int = 40) -> list[str]:
     """Section headings of an artifact, for triage only.
 
     Queries taken from the artifact let it define both the problem scope and its own
     relevance. Callers must stamp ``queries_self_derived`` on any result built from these.
+    Headings inside fenced code blocks are not headings, so they are not offered.
     """
-    headings = [h.strip() for h in re.findall(r"^#{2,3}\s+(.+)$", body, re.M)]
+    headings = [h.strip() for h in re.findall(r"^#{2,3}[ \t]+(.+)$", _outside_fences(body), re.M)]
     return [h for h in headings if len(tokenize(h)) >= 2][:limit]
 
 

dokimi/scripts/growth_baseline.json

@@ -1,6 +1,6 @@
 {
   "files": {
-    "dokimi/commands/novelty.py": {"lines": 827},
+    "dokimi/commands/novelty.py": {"lines": 863, "rationale": "W3 adds routing no_queries handling; file split tracked separately to avoid conflicts across the W-series"},
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main after the #93 seed and before it merged; split it in its own PR"},
@@ -25,7 +25,7 @@
     "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18},
     "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"},
     "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"},
-    "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 14},
+    "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12},
     "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13},
     "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"},
     "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "grew on main after the #93 seed and before it merged; split it in its own PR"},