From 53d7374d9b1445bcaf5a83a29d91a4e234687dab Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 21:45:10 -0700 Subject: [PATCH 1/8] fix(routing): triage queries skip code fences, zero-query runs say why --queries-from-headings read a `## comment` inside a fenced code block as a request. Headings inside fences are now skipped (backtick and tilde fences, closed only by the same character at least as long). A run with no usable query exited with a bare "no usable queries" and wrote nothing. It now names the cause, suggests --queries, and writes the result with "status": "no_queries" and a "reason"; a successful result carries "status": "ok". Every routing rate line prints its n_queries, with "triage: self-derived" beside it when the queries come from the artifact's headings. Fewer than five scored queries draw a warning on stdout. Signed-off-by: Nikolay Petrov --- ...1_140000_napetrov_routing-triage-guards.md | 10 +++ dokimi/docs/skill-novelty.md | 6 +- dokimi/dokimi/commands/novelty.py | 39 +++++++-- dokimi/dokimi/novelty/routing.py | 26 +++++- dokimi/tests/test_novelty_routing.py | 81 +++++++++++++++++++ 5 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md diff --git a/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md b/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md new file mode 100644 index 00000000..593e9901 --- /dev/null +++ b/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md @@ -0,0 +1,10 @@ +### Fixed + +- **`novelty routing --queries-from-headings`** no longer reads a `##` line inside a + fenced code block as a request, so a shell comment does not become a query. +- **`novelty routing` with no usable query** says why and suggests `--queries`, and + writes the result with `"status": "no_queries"` (the reason is in `reason`), where it + used to exit with a bare "no usable queries". A successful result carries `"status": "ok"`. +- **Every `novelty routing` rate line** prints its `n_queries`, and a self-derived rate + carries `triage: self-derived` on the same line. A run that scores fewer than five + queries prints a warning, since one query moves a rate by 20 points. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bb..79a4fa97 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -621,7 +621,11 @@ Three properties worth knowing before quoting either: - **The queries must come from outside the artifact**, or it defines both the problem scope and its own relevance. `--queries-from-headings` exists for triage, warns, and stamps `queries_self_derived: true` into the result; corpus-wide those self-derived queries gave - recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667. + recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667. Headings inside + fenced code blocks are not offered as queries. A run with no usable query exits non-zero, + says why, and still writes the result with `"status": "no_queries"`. Every rate line + prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer + than five scored queries draw a warning. - **The model figure is an upper bound on harness behaviour.** A harness whose own skill listing truncates or omits descriptions selects on less text than this command shows — which has been observed. Only a run inside the harness settles it. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc943..4c76f7e6 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -320,14 +320,32 @@ def card(ref: str) -> tuple[Candidate, str]: "pass --queries with requests written independently of the artifact, or " "--queries-from-headings for a self-derived triage run" ) - if not queries: - raise SystemExit("no usable queries") - candidates = [gold, *competitors] lexical = dict(lexical_routing(candidates, queries, gold=0), queries_self_derived=self_derived) + out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" + out.parent.mkdir(parents=True, exist_ok=True) + if not lexical["n_queries_scored"]: + # Written too, so a batch run leaves a record of why it has no rate. + if not queries and self_derived: + reason = (f"no section heading of two or more words outside code fences in " + f"{args.artifact}; headings inside fenced code are not requests") + elif not queries: + reason = f"--queries {args.queries} holds no non-blank query" + else: + reason = (f"none of the {len(queries)} queries has a content word, since words " + "like a, the, how do not count") + out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries", + "reason": reason, "queries_self_derived": self_derived, + "n_candidates": len(candidates)}, indent=2), + encoding="utf-8") + raise SystemExit(f"no usable queries: {reason}. Pass --queries with requests " + f"written independently of the artifact, one per line or a JSON " + f"list. Wrote {out} with status no_queries") + payload = { "artifact": args.artifact, + "status": "ok", "n_candidates": len(candidates), "queries_self_derived": self_derived, "lexical": lexical, @@ -339,22 +357,27 @@ def card(ref: str) -> tuple[Candidate, str]: queries_self_derived=self_derived, ) - out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" - out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") + # Every rate is printed with the number of queries behind it, and a self-derived one + # carries the triage tag on the same line, so a copied figure cannot lose either. + tag = " triage: self-derived" if self_derived else "" lexical = payload["lexical"] print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " - f"MRR {lexical['mrr']} over {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} queries vs {len(competitors)} competitors" + f"MRR {lexical['mrr']} n_queries {lexical['n_queries_scored']} of " + f"{lexical['n_queries_given']} given, vs {len(competitors)} competitors{tag}" + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) + if lexical["n_queries_scored"] < 5: + print(f"WARNING: only {lexical['n_queries_scored']} queries scored; one query moves " + "a rate by 20 points or more, so do not quote it as a routing rate") if "model" in payload: row = payload["model"] print(f"model selection {row['selection_rate']} " f"({row['selection_rate_among_answered']} among answered; " - f"{row['n_abstained']} abstained, {row['n_failed']} failed) [{row['model']}]") + f"{row['n_abstained']} abstained, {row['n_failed']} failed) " + f"[{row['model']}] n_queries {row['n_queries']}{tag}") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") if self_derived: diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 36d13045..d1cbbc42 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -309,13 +309,37 @@ def model_routing( } +_FENCE = re.compile(r"\s{0,3}(`{3,}|~{3,})") + + +def _outside_fences(body: str) -> str: + """``body`` with its fenced code blocks removed, so a ``## comment`` in a shell block is + not read as a heading. A fence closes on the same character, at least as long, alone on + its line; an unclosed fence runs to the end of the body, as CommonMark has it. + """ + kept: list[str] = [] + opening: str | None = None + for line in body.splitlines(): + fence = _FENCE.match(line) + if opening is None: + if fence: + opening = fence.group(1) + else: + kept.append(line) + elif (fence and fence.group(1)[0] == opening[0] and len(fence.group(1)) >= len(opening) + and not line.strip()[len(fence.group(1)):]): + opening = None + return "\n".join(kept) + + def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: """Section headings of an artifact, for triage only. Queries taken from the artifact let it define both the problem scope and its own relevance. Callers must stamp ``queries_self_derived`` on any result built from these. + Headings inside fenced code blocks are not headings, so they are not offered. """ - headings = [h.strip() for h in re.findall(r"^#{2,3}\s+(.+)$", body, re.M)] + headings = [h.strip() for h in re.findall(r"^#{2,3}\s+(.+)$", _outside_fences(body), re.M)] return [h for h in headings if len(tokenize(h)) >= 2][:limit] diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index cd7ea078..0dbfa2ee 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -170,6 +170,19 @@ def test_headings_are_offered_only_as_self_derived_queries(): assert headings_as_queries(body) == ["Recording a profile", "Reading the report"] +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_a_heading_inside_a_code_fence_is_not_a_query(fence): + # A shell comment such as "## Not a heading" is a line of code, not a request. + body = (f"# Title\n\n## Recording a profile\n\n{fence}bash\n## Not a heading here\n" + f"perf record\n{fence}\n\n## Reading the report\n") + assert headings_as_queries(body) == ["Recording a profile", "Reading the report"] + + +def test_a_fence_closes_only_on_its_own_character_and_length(): + body = "````md\n```\n## Still inside\n````\n## Real one here\n" + assert headings_as_queries(body) == ["Real one here"] + + def _parse(*argv): return build_parser().parse_args(list(argv)) @@ -473,3 +486,71 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert names.count("perf") == 0, "perf should be excluded from catalog" assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" + +def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path): + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary", + body="```\n## only in code\n```\n"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries-from-headings", "--out", str(out)) + with pytest.raises(SystemExit, match="code fences.*--queries"): + args.func(args) + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["status"] == "no_queries" + assert "code fences" in payload["reason"] + + +def test_a_query_file_with_nothing_in_it_refuses_the_run(tmp_path): + queries = tmp_path / "q.txt" + queries.write_text("\n \n", encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(out)) + with pytest.raises(SystemExit, match="no non-blank query"): + args.func(args) + assert json.loads(out.read_text(encoding="utf-8"))["status"] == "no_queries" + + +def test_queries_without_a_content_word_refuse_the_run(tmp_path): + queries = tmp_path / "q.txt" + queries.write_text("the of a\nhow do I\n", encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(tmp_path / "routing.json")) + with pytest.raises(SystemExit, match="has a content word"): + args.func(args) + + +def test_every_rate_line_carries_its_query_count_and_triage_tag(tmp_path, capsys): + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary", + body="# B\n\n## Recording a profile\n\n" + "## Reading the report\n"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries-from-headings", "--out", str(out)) + args.func(args) + lines = capsys.readouterr().out.splitlines() + lexical = next(line for line in lines if line.startswith("lexical")) + assert "n_queries 2 of 2" in lexical + assert "triage: self-derived" in lexical + + +def test_fewer_than_five_scored_queries_are_warned_about_on_stdout(tmp_path, capsys): + def run(count): + queries = tmp_path / f"q{count}.txt" + queries.write_text("\n".join(f"profile a binary run {i}" for i in range(count)), + encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, f"a{count}", "profile a binary"), + "--competitors", _skill(tmp_path, f"b{count}", "tune a database"), + "--queries", str(queries), "--out", str(tmp_path / f"r{count}.json")) + args.func(args) + return capsys.readouterr().out + + assert "WARNING: only 2 queries scored" in run(2) + assert "WARNING" not in run(5) From 52d90ccc94412c8e28afc7b1c8e99d696da7ef73 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:07:38 -0700 Subject: [PATCH 2/8] fix(routing): a heading needs a space or tab after its hashes The heading pattern used \s+ after the hashes, which also matches a newline, so a bare "##" line read the next line as its heading. The separator is now [ \t]+, and a test pins "##\nfoo bar\n" to no queries. Changelog: the zero-scored routing exit (status no_queries, exit 1 where it used to exit 0 with null rates) is listed under Changed. Signed-off-by: Nikolay Petrov --- .../20261021_140000_napetrov_routing-triage-guards.md | 7 +++++++ dokimi/dokimi/novelty/routing.py | 2 +- dokimi/tests/test_novelty_routing.py | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md b/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md index 593e9901..3124fe57 100644 --- a/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md +++ b/dokimi/changelog.d/20261021_140000_napetrov_routing-triage-guards.md @@ -8,3 +8,10 @@ - **Every `novelty routing` rate line** prints its `n_queries`, and a self-derived rate carries `triage: self-derived` on the same line. A run that scores fewer than five queries prints a warning, since one query moves a rate by 20 points. + +### Changed + +- **`novelty routing` exits 1 when no query is scored** — a run whose queries are non-empty + but all filtered out (no content word left) now exits non-zero with `"status": "no_queries"` + and a `reason`, where it used to exit 0 with null rates. A script that read the null rates + as a result must check the exit code or `status` instead. diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index d1cbbc42..aa54cc40 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -339,7 +339,7 @@ def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: relevance. Callers must stamp ``queries_self_derived`` on any result built from these. Headings inside fenced code blocks are not headings, so they are not offered. """ - headings = [h.strip() for h in re.findall(r"^#{2,3}\s+(.+)$", _outside_fences(body), re.M)] + headings = [h.strip() for h in re.findall(r"^#{2,3}[ \t]+(.+)$", _outside_fences(body), re.M)] return [h for h in headings if len(tokenize(h)) >= 2][:limit] diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 0dbfa2ee..008ee5cd 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -178,6 +178,11 @@ def test_a_heading_inside_a_code_fence_is_not_a_query(fence): assert headings_as_queries(body) == ["Recording a profile", "Reading the report"] +def test_a_bare_hash_line_does_not_take_the_next_line_as_its_heading(): + # "\s+" matched the newline, so "##" followed by "foo bar" read as the heading "foo bar". + assert headings_as_queries("##\nfoo bar\n") == [] + + def test_a_fence_closes_only_on_its_own_character_and_length(): body = "````md\n```\n## Still inside\n````\n## Real one here\n" assert headings_as_queries(body) == ["Real one here"] From 46a08974395c05ba17b04e08f03703e4d95c79b5 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 15:33:38 +0000 Subject: [PATCH 3/8] Use singular 'query' in the few-queries warning for a count of one Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/novelty.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 4c76f7e6..f384d15b 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -370,7 +370,9 @@ def card(ref: str) -> tuple[Candidate, str]: + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) if lexical["n_queries_scored"] < 5: - print(f"WARNING: only {lexical['n_queries_scored']} queries scored; one query moves " + n_scored = lexical["n_queries_scored"] + noun = "query" if n_scored == 1 else "queries" + print(f"WARNING: only {n_scored} {noun} scored; one query moves " "a rate by 20 points or more, so do not quote it as a routing rate") if "model" in payload: row = payload["model"] From e5594d9f61b047dff53133cca830a4f98ebaf818 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:40:16 -0700 Subject: [PATCH 4/8] fix(routing): fence opener follows CommonMark; model row says n_queries is of all given Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 2 +- dokimi/dokimi/novelty/routing.py | 4 +- dokimi/tests/test_novelty_routing.py | 61 ++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index f384d15b..9646712a 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -379,7 +379,7 @@ def card(ref: str) -> tuple[Candidate, str]: print(f"model selection {row['selection_rate']} " f"({row['selection_rate_among_answered']} among answered; " f"{row['n_abstained']} abstained, {row['n_failed']} failed) " - f"[{row['model']}] n_queries {row['n_queries']}{tag}") + f"[{row['model']}] n_queries {row['n_queries']} given{tag}") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") if self_derived: diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index aa54cc40..2ec7cf0b 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -309,7 +309,9 @@ def model_routing( } -_FENCE = re.compile(r"\s{0,3}(`{3,}|~{3,})") +# Only spaces indent a fence (a tab is four columns, so it makes indented code), and a +# backtick opener may not carry a backtick in its info string; both are CommonMark rules. +_FENCE = re.compile(r" {0,3}(`{3,}(?!.*`)|~{3,})") def _outside_fences(body: str) -> str: diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 008ee5cd..74ae47f9 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -559,3 +559,64 @@ def run(count): assert "WARNING: only 2 queries scored" in run(2) assert "WARNING" not in run(5) + + +@pytest.mark.parametrize("opener", ["```js `x`", "\t```"]) +def test_a_line_that_is_not_a_fence_does_not_hide_the_headings_after_it(opener): + # An info string with a backtick, or a tab indent, is not a fence by CommonMark's rules. + assert headings_as_queries(f"{opener}\n## Real heading one\n") == ["Real heading one"] + + +def test_a_four_space_indented_fence_is_indented_code_not_a_fence(): + assert headings_as_queries(" ```\n## Real heading one\n") == ["Real heading one"] + + +def test_a_tilde_line_does_not_close_a_backtick_block(): + body = "```\n~~~\n## Hidden heading one\n```\n## Real heading one\n" + assert headings_as_queries(body) == ["Real heading one"] + + +def test_a_fence_with_an_info_string_does_not_close_a_block(): + body = "```\n```js\n## Hidden heading one\n```\n## Real heading one\n" + assert headings_as_queries(body) == ["Real heading one"] + + +def _routing_with_model(tmp_path, monkeypatch, queries_text): + from dokimi.commands import novelty as novelty_cmd + + calls = [] + + def transport(prompt, system): + calls.append(prompt) + return json.dumps({"pick": 1}), None + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model="fake", role=role, transport=transport)) + queries = tmp_path / "q.txt" + queries.write_text(queries_text, encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary run"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "m", "--out", str(out)) + return args, calls + + +def test_the_model_row_says_its_query_count_is_of_all_given(tmp_path, monkeypatch, capsys): + args, calls = _routing_with_model(tmp_path, monkeypatch, + "profile a binary run\nthe of a\nhow do I\n") + args.func(args) + lines = capsys.readouterr().out.splitlines() + lexical = next(line for line in lines if line.startswith("lexical")) + model = next(line for line in lines if line.startswith("model")) + assert "n_queries 1 of 3 given" in lexical + assert "n_queries 3 given" in model + assert len(calls) == 3 + + +def test_a_run_with_no_scorable_query_makes_no_model_call(tmp_path, monkeypatch): + args, calls = _routing_with_model(tmp_path, monkeypatch, "the of a\nhow do I\n") + with pytest.raises(SystemExit, match="no usable queries"): + args.func(args) + assert calls == [] + assert json.loads((tmp_path / "routing.json").read_text())["status"] == "no_queries" From 79eb3dea871dae1372bfa770446aaf5333e6202b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:04:02 -0700 Subject: [PATCH 5/8] refactor(novelty): extract the no-queries exit and routing summary; chore(growth): raise commands/novelty.py for W3 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 47 ++++++++++++++++++----------- dokimi/scripts/growth_baseline.json | 2 +- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 9646712a..58329fd6 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -17,7 +17,7 @@ import math import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NoReturn from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, @@ -326,22 +326,7 @@ def card(ref: str) -> tuple[Candidate, str]: out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" out.parent.mkdir(parents=True, exist_ok=True) if not lexical["n_queries_scored"]: - # Written too, so a batch run leaves a record of why it has no rate. - if not queries and self_derived: - reason = (f"no section heading of two or more words outside code fences in " - f"{args.artifact}; headings inside fenced code are not requests") - elif not queries: - reason = f"--queries {args.queries} holds no non-blank query" - else: - reason = (f"none of the {len(queries)} queries has a content word, since words " - "like a, the, how do not count") - out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries", - "reason": reason, "queries_self_derived": self_derived, - "n_candidates": len(candidates)}, indent=2), - encoding="utf-8") - raise SystemExit(f"no usable queries: {reason}. Pass --queries with requests " - f"written independently of the artifact, one per line or a JSON " - f"list. Wrote {out} with status no_queries") + _refuse_no_queries(args, queries, self_derived, out, len(candidates)) payload = { "artifact": args.artifact, @@ -360,13 +345,17 @@ def card(ref: str) -> tuple[Candidate, str]: out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") + _print_routing_summary(payload, len(competitors), self_derived) + +def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: + """The lexical line, the model line and their caveats, printed after a routing run.""" # Every rate is printed with the number of queries behind it, and a self-derived one # carries the triage tag on the same line, so a copied figure cannot lose either. tag = " triage: self-derived" if self_derived else "" lexical = payload["lexical"] print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " f"MRR {lexical['mrr']} n_queries {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} given, vs {len(competitors)} competitors{tag}" + f"{lexical['n_queries_given']} given, vs {n_competitors} competitors{tag}" + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) if lexical["n_queries_scored"] < 5: @@ -386,6 +375,28 @@ def card(ref: str) -> tuple[Candidate, str]: print("Self-derived queries: read as triage only.") + +def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, + out: Path, n_candidates: int) -> NoReturn: + """Write the ``no_queries`` record, then exit saying why no query was scored.""" + # Written too, so a batch run leaves a record of why it has no rate. + if not queries and self_derived: + reason = (f"no section heading of two or more words outside code fences in " + f"{args.artifact}; headings inside fenced code are not requests") + elif not queries: + reason = f"--queries {args.queries} holds no non-blank query" + else: + reason = (f"none of the {len(queries)} queries has a content word, since words " + "like a, the, how do not count") + out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries", + "reason": reason, "queries_self_derived": self_derived, + "n_candidates": n_candidates}, indent=2), + encoding="utf-8") + raise SystemExit(f"no usable queries: {reason}. Pass --queries with requests " + f"written independently of the artifact, one per line or a JSON " + f"list. Wrote {out} with status no_queries") + + def cmd_novelty_shorten(args: argparse.Namespace) -> None: """Score shortening candidates on the five axes and optionally write the best one.""" from dokimi.novelty.analyze import load_artifact_text diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..b4bee57c 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 863, "rationale": "W3 adds routing no_queries handling; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, From 50a15176d2616f2560a34ae750ca0917ee7f28dc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:04:03 -0700 Subject: [PATCH 6/8] chore(growth): lower dokimi/commands/novelty.py::cmd_novelty_routing baseline after the extraction Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b4bee57c..c9503e8c 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 14}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, From b6e3d7dbef27df569356c3af4d74da661a782f78 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 01:34:52 -0700 Subject: [PATCH 7/8] test(routing): pin indented fence closers, status ok, model-row tag, warning threshold A closer indented 1-3 spaces (fences in list items) could stop matching, hiding every heading after it, and no test went red. Also pin "status": "ok" on a successful result, the triage tag on the model row, the warning at 4 scored queries, and a tab after "##". Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_routing.py | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 74ae47f9..d61ce370 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -188,6 +188,20 @@ def test_a_fence_closes_only_on_its_own_character_and_length(): assert headings_as_queries(body) == ["Real one here"] +@pytest.mark.parametrize("fence", ["```", "~~~"]) +@pytest.mark.parametrize("open_indent", range(4)) +@pytest.mark.parametrize("close_indent", range(4)) +def test_a_fence_indented_up_to_three_spaces_opens_and_closes(fence, open_indent, close_indent): + # Fences in list items are indented; an indented closer must still end the block. + body = (f"{' ' * open_indent}{fence}\n## Hidden heading one\n" + f"{' ' * close_indent}{fence}\n## Real heading one\n") + assert headings_as_queries(body) == ["Real heading one"] + + +def test_a_tab_after_the_hashes_still_makes_a_heading(): + assert headings_as_queries("##\tTab heading words\n") == ["Tab heading words"] + + def _parse(*argv): return build_parser().parse_args(list(argv)) @@ -268,6 +282,7 @@ def test_self_derived_queries_are_stamped_on_the_result(tmp_path, capsys): "--queries-from-headings", "--out", str(out)) args.func(args) payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["status"] == "ok" assert payload["queries_self_derived"] is True assert "model" not in payload # no --model, no spend captured = capsys.readouterr() @@ -558,6 +573,7 @@ def run(count): return capsys.readouterr().out assert "WARNING: only 2 queries scored" in run(2) + assert "WARNING: only 4 queries scored" in run(4) assert "WARNING" not in run(5) @@ -620,3 +636,20 @@ def test_a_run_with_no_scorable_query_makes_no_model_call(tmp_path, monkeypatch) args.func(args) assert calls == [] assert json.loads((tmp_path / "routing.json").read_text())["status"] == "no_queries" + + +def test_the_model_row_carries_the_triage_tag_on_self_derived_queries(tmp_path, monkeypatch, + capsys): + from dokimi.commands import novelty as novelty_cmd + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model="fake", role=role, transport=lambda prompt, system: (json.dumps({"pick": 1}), None))) + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries-from-headings", "--model", "m", + "--out", str(tmp_path / "routing.json")) + args.func(args) + model = next(line for line in capsys.readouterr().out.splitlines() + if line.startswith("model")) + assert "triage: self-derived" in model From 6125691050d96ab8ce78646cbe8d9979ba1343b4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 08:12:21 -0700 Subject: [PATCH 8/8] test(novelty): pin the two-marker fence and no-space heading boundaries Signed-off-by: Nikolay Petrov --- dokimi/tests/test_novelty_routing.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index d61ce370..2c888224 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -583,6 +583,15 @@ def test_a_line_that_is_not_a_fence_does_not_hide_the_headings_after_it(opener): assert headings_as_queries(f"{opener}\n## Real heading one\n") == ["Real heading one"] +@pytest.mark.parametrize("opener", ["~~strike~~ text", "``x`` text", "``x text"]) +def test_a_run_of_two_markers_is_not_a_fence(opener): + assert headings_as_queries(f"{opener}\n## Real heading one\n") == ["Real heading one"] + + +def test_a_hash_run_without_a_space_is_not_a_heading(): + assert headings_as_queries("##hashtag word\n") == [] + + def test_a_four_space_indented_fence_is_indented_code_not_a_fence(): assert headings_as_queries(" ```\n## Real heading one\n") == ["Real heading one"]