From 6a6631c6893b916f58bcd497ff8ab179843d9d67 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:43:38 -0700 Subject: [PATCH 1/9] feat(novelty): --repeats for routing noise floor; keep declined replies `novelty routing --model` now asks each query N times (--repeats) with the same card order, and reports each query's agreement across its repeats plus the spread of the selection rate between repeats (`noise`), so a change of a few points can be read against run-to-run noise. Every abstention or unparseable pick keeps the model's raw reply under per_query[].raw_reply. Repeats are only real samples if the response cache sees a new draw, so LLMClient.text takes a `draw` index and forwards it to llm_call_with_usage. Draw 0 is the default and keeps every existing cache entry. Signed-off-by: Nikolay Petrov --- ...0261021_230000_napetrov_routing-repeats.md | 3 + dokimi/docs/skill-novelty.md | 10 +- dokimi/dokimi/commands/novelty.py | 22 ++- dokimi/dokimi/novelty/client.py | 14 +- dokimi/dokimi/novelty/routing.py | 121 +++++++++--- dokimi/tests/test_novelty_routing.py | 175 ++++++++++++++++++ 6 files changed, 314 insertions(+), 31 deletions(-) create mode 100644 dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md diff --git a/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md new file mode 100644 index 00000000..075669b1 --- /dev/null +++ b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md @@ -0,0 +1,3 @@ +### Added + +- **`novelty routing --repeats N` and the raw reply of a declined pick** — `--repeats N` (with `--model`) asks the model N times per query, with the same card order each time, so the spread between repeats is the model's own sampling noise. The `model` block gains `noise`: each query's agreement across its repeats (`per_query`), the selection rate of each repeat, and `half_range`, half the gap between the highest and lowest of those rates, a crude noise floor. Each repeat is a fresh sample, so with `DOKIMI_RESPONSE_CACHE` set a repeat is no longer a replay of the first reply. Every abstention or unparseable pick now keeps the model's raw reply under `per_query[].raw_reply` (first 2000 characters). `LLMClient.text` takes a `draw` index for the same reason. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bb..0f7cb9fe 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -587,7 +587,7 @@ the absence of detectable corruption — not proof that the wordings are faithfu ```bash python cli.py novelty routing --artifact skill:path/to/skill \ --competitors skill:sibling-a skill:sibling-b ... \ - --queries queries.json [--model M] + --queries queries.json [--model M [--repeats N]] ``` `--catalog ` takes the competitors from a skill directory instead: every `SKILL.md` @@ -606,6 +606,14 @@ Two numbers, reported side by side and never merged: `selection_rate`** — not picking anything is the routing failure — with `selection_rate_among_answered` beside it; a failed call is excluded from both and counted, and one failed query does not lose the rest. + Each abstention or unparseable pick keeps the model's raw reply under + `per_query[].raw_reply` (the first 2000 characters), so a decline can be read. + `--repeats N` asks each query N times with the same card order, so the spread between + repeats is the model's own sampling noise. The result's `noise` block gives each query's + agreement across its repeats (`per_query`), the selection rate of each repeat, and + `half_range`, half the gap between the highest and lowest of those rates: a crude noise + floor. Each repeat is a fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a + replay. Repeats need `--model`, and cost N times the calls. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc943..bd023195 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -264,6 +264,11 @@ def cmd_novelty_items(args: argparse.Namespace) -> None: def cmd_novelty_routing(args: argparse.Namespace) -> None: """Can this artifact be selected against its competitors? Lexical floor, model on top.""" + if args.repeats > 1 and not args.model: + raise SystemExit( + "--repeats needs --model: the lexical floor is deterministic, so repeating it " + "measures nothing" + ) from dokimi.novelty.routing import ( Candidate, RoutingInputError, @@ -335,7 +340,8 @@ def card(ref: str) -> tuple[Candidate, str]: if args.model: selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( - model_routing(candidates, queries, client=selector, gold=0, seed=args.seed), + model_routing(candidates, queries, client=selector, gold=0, seed=args.seed, + repeats=args.repeats), queries_self_derived=self_derived, ) @@ -355,6 +361,16 @@ def card(ref: str) -> tuple[Candidate, str]: print(f"model selection {row['selection_rate']} " f"({row['selection_rate_among_answered']} among answered; " f"{row['n_abstained']} abstained, {row['n_failed']} failed) [{row['model']}]") + if row["n_abstained"]: + print(f"{row['n_abstained']} abstained or unparseable: the raw reply is under " + "per_query[].raw_reply in the output") + if row["noise"]: + noise = row["noise"] + print(f"noise {row['repeats']} repeats per query: mean agreement " + f"{noise['mean_agreement']}, {noise['unstable_queries']} queries picked " + f"differently across repeats; selection by repeat " + f"{noise['selection_rate_by_repeat']}, half range " + f"{noise['half_range']} (a crude sampling floor)") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") if self_derived: @@ -603,6 +619,10 @@ def register(sub, positive_int) -> None: routing_p.add_argument("--seed", type=int, default=0, help="Seeds the per-query card permutation, so a position-biased " "model scores ~1/N and the run is reproducible") + routing_p.add_argument("--repeats", type=positive_int, default=1, + help="Ask the model this many times per query, with the same card " + "order each time, and report per-query agreement and the " + "run-to-run noise floor. Needs --model; costs N times the calls") routing_p.add_argument("--out", default=None) _add_spend_cap(routing_p) routing_p.set_defaults(func=_stops_at_cap(cmd_novelty_routing)) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index b8579cc8..ae65133a 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -106,7 +106,8 @@ class LLMClient: cap: SpendCap | None = None def text( - self, prompt: str, *, system: str | None = None, accept: Accept | None = None + self, prompt: str, *, system: str | None = None, accept: Accept | None = None, + draw: int = 0, ) -> str: """Return the model's reply as plain text. @@ -115,13 +116,18 @@ def text( ``DOKIMI_RESPONSE_CACHE`` set, a cut-off or unparseable reply is stored and every later run replays the failure without calling the model. An injected transport is the caller's own and is not given it. + + ``draw`` is the sample index of this prompt. With the response cache on, a repeat + of a byte-identical prompt replays draw 0 unless it has its own index, so a + repeated ask is only a new sample when it passes a new one. Draw 0 is the default + and keeps every existing cache entry. An injected transport is not given it. """ if self.cap is not None: self.cap.before_call(self.model) if self.transport is not None: reply, usage = self.transport(prompt, system) else: - reply, usage = self._default_transport(prompt, system, accept=accept) + reply, usage = self._default_transport(prompt, system, accept=accept, draw=draw) self.calls += 1 if usage is not None: # UsageRecord is frozen; re-stamp the role so cost rolls up per role. @@ -180,7 +186,8 @@ def cost_summary(self) -> dict[str, Any]: } def _default_transport( - self, prompt: str, system: str | None, *, accept: Accept | None = None + self, prompt: str, system: str | None, *, accept: Accept | None = None, + draw: int = 0, ) -> tuple[str, UsageRecord | None]: from dokimi.llm import llm_call_with_usage @@ -191,6 +198,7 @@ def _default_transport( system=system, max_tokens=self.max_tokens, cache_if=accept, + draw=draw, ) diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 36d13045..de496fb1 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -229,11 +229,15 @@ def _as_pick_position(value: object) -> int: return 0 +# A reply that declines to route is kept verbatim, up to this many characters. +_RAW_REPLY_CHARS = 2000 + + def model_routing( candidates: list[Candidate], queries: list[str], *, client: LLMClient, gold: int = 0, - seed: int = 0, + seed: int = 0, repeats: int = 1, ) -> dict: - """Ask a model to pick from the same cards, one call per query. + """Ask a model to pick from the same cards, ``repeats`` times per query. The candidate order is a seeded random permutation per query, so a model that always picks the first (or last) option scores about ``1/N`` rather than a figure set by how @@ -242,14 +246,23 @@ def model_routing( second-to-last -- exactly the positions where primacy and recency bias are strongest. The permutation is recorded per query, so a result can be re-read or replayed. + Each query keeps one card order across its repeats, so the spread between repeats is + the model's own sampling noise rather than a change of layout. Each repeat is its own + sample (``draw``), so with the response cache on a repeat is not a replay. + Returns: ``selection_rate``, ``n_queries``, ``n_abstained``, ``n_failed``, the per-query pick - with the position the gold card was shown at, and the caveat that this bounds - harness behaviour from above. + with the position the gold card was shown at, the raw reply of every abstention or + unparseable pick, and the caveat that this bounds harness behaviour from above. + With ``repeats > 1``, ``noise`` holds each query's agreement across its repeats and + the run-to-run spread of the selection rate; it is ``None`` for one repeat. """ if not 0 <= gold < len(candidates): raise ValueError(f"gold index {gold} is outside 0..{len(candidates) - 1}") + if repeats < 1: + raise ValueError(f"repeats must be at least 1, not {repeats}") picks: list[dict] = [] + groups: list[list[dict]] = [] for index_of_query, query in enumerate(queries): shuffler = random.Random(f"{seed}:{index_of_query}:{len(candidates)}") order = list(range(len(candidates))) @@ -258,28 +271,36 @@ def model_routing( f"{position + 1}. {candidates[index].card}" for position, index in enumerate(order) ) - try: - reply = client.text( - _SELECT_PROMPT.format(query=query, cards=cards), - accept=partial(_is_pick_reply, n_cards=len(order)), - ) - except Exception as exc: # noqa: BLE001 - one failed query must not lose the rest - picks.append({"query": query, "picked": None, "error": str(exc)[:200]}) - continue - # A reply that is not a usable pick is the model declining to route, not a - # transport failure: it stays in the denominator like an explicit abstention. - try: - raw = _as_pick_position(parse_json_object(reply).get("pick", 0)) - except ValueError: - raw = 0 - if raw < 1 or raw > len(order): - picks.append({"query": query, "picked": None, "abstained": True}) - continue - chosen = order[raw - 1] - picks.append({"query": query, "picked": candidates[chosen].name, - "correct": chosen == gold, "picked_at": raw, - "gold_shown_at": order.index(gold) + 1, - "order": [candidates[i].name for i in order]}) + prompt = _SELECT_PROMPT.format(query=query, cards=cards) + group: list[dict] = [] + for repeat in range(repeats): + base = {"query": query, "repeat": repeat} + try: + reply = client.text( + prompt, accept=partial(_is_pick_reply, n_cards=len(order)), draw=repeat, + ) + except Exception as exc: # noqa: BLE001 - one failed query must not lose the rest + row = {**base, "picked": None, "error": str(exc)[:200]} + else: + # A reply that is not a usable pick is the model declining to route, not a + # transport failure: it stays in the denominator like an explicit abstention, + # and its raw text is kept so the decline can be read rather than inferred. + try: + raw = _as_pick_position(parse_json_object(reply).get("pick", 0)) + except ValueError: + raw = 0 + if raw < 1 or raw > len(order): + row = {**base, "picked": None, "abstained": True, + "raw_reply": reply[:_RAW_REPLY_CHARS]} + else: + chosen = order[raw - 1] + row = {**base, "picked": candidates[chosen].name, + "correct": chosen == gold, "picked_at": raw, + "gold_shown_at": order.index(gold) + 1, + "order": [candidates[i].name for i in order]} + picks.append(row) + group.append(row) + groups.append(group) answered = [p for p in picks if "correct" in p] hits = sum(1 for p in answered if p["correct"]) @@ -289,6 +310,7 @@ def model_routing( "model": client.model, "n_candidates": len(candidates), "n_queries": len(queries), + "repeats": repeats, "n_abstained": sum(1 for p in picks if p.get("abstained")), "n_failed": sum(1 for p in picks if p.get("error")), "seed": seed, @@ -300,6 +322,7 @@ def model_routing( "selection_rate_among_answered": ( round(hits / len(answered), 4) if answered else None ), + "noise": _noise(groups) if repeats > 1 else None, "per_query": picks, "note": ( "An upper bound on harness behaviour, not a harness measurement: a harness " @@ -309,6 +332,52 @@ def model_routing( } +def _noise(groups: list[list[dict]]) -> dict: + """Agreement across the repeats of each query, and the run-to-run spread of the rate. + + A query's agreement is the share of its answered repeats that gave its most common + pick; an abstention counts as a pick. ``selection_rate_by_repeat`` reads each repeat as + its own run over the same queries, and ``half_range`` is half the gap between the + highest and the lowest of those rates. It is a crude sampling noise floor: a few repeats + estimate it coarsely, and it says nothing about a change of cards or queries. + + Args: + groups: One list per query, holding that query's rows in repeat order. + """ + per_query: list[dict] = [] + for group in groups: + labels = [ + "(abstained)" if row.get("abstained") else row["picked"] + for row in group if not row.get("error") + ] + modal, agreement = None, None + if labels: + modal, count = Counter(labels).most_common(1)[0] + agreement = round(count / len(labels), 4) + per_query.append({ + "query": group[0]["query"], + "modal_pick": modal, + "agreement": agreement, + "picks": labels, + "n_correct": sum(1 for row in group if row.get("correct")), + }) + repeats = len(groups[0]) if groups else 0 + rates: list[float | None] = [] + for repeat in range(repeats): + rows = [group[repeat] for group in groups if not group[repeat].get("error")] + rates.append(round(sum(1 for r in rows if r.get("correct")) / len(rows), 4) + if rows else None) + known = [rate for rate in rates if rate is not None] + agreements = [row["agreement"] for row in per_query if row["agreement"] is not None] + return { + "mean_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, + "unstable_queries": sum(1 for a in agreements if a < 1), + "selection_rate_by_repeat": rates, + "half_range": round((max(known) - min(known)) / 2, 4) if len(known) > 1 else None, + "per_query": per_query, + } + + def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: """Section headings of an artifact, for triage only. diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index cd7ea078..aa701ab2 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +from types import SimpleNamespace import pytest @@ -473,3 +474,177 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert names.count("perf") == 0, "perf should be excluded from catalog" assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" + + +def _answer_as(name: str): + """A transport reply that picks the card called ``name``, wherever it is shown.""" + def reply(prompt: str) -> str: + lines = prompt.split("TOOLS:", 1)[1].strip().splitlines() + return json.dumps({"pick": next(i + 1 for i, line in enumerate(lines) + if line.split(" ", 1)[1].startswith(name))}) + return reply + + +@pytest.mark.parametrize("reply", [ + "I think the best one is the third.", '{"pick": 0}', '{"pick": 999}', +]) +def test_a_declined_reply_is_recorded_verbatim(reply): + client = LLMClient(model="m", role="selector", transport=lambda p, s: (reply, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert row["abstained"] is True + assert row["raw_reply"] == reply + + +def test_a_very_long_declined_reply_is_cut_at_the_recording_limit(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: ("x" * 5000, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert len(row["raw_reply"]) == 2000 + + +def test_repeats_below_one_are_refused(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: (json.dumps({"pick": 1}), None)) + with pytest.raises(ValueError, match="at least 1"): + model_routing([PERF, TBB], ["q1"], client=client, repeats=0) + + +def test_one_repeat_has_no_noise_block(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: (json.dumps({"pick": 1}), None)) + result = model_routing([PERF, TBB], ["q1", "q2"], client=client) + assert (result["repeats"], result["noise"]) == (1, None) + assert [row["repeat"] for row in result["per_query"]] == [0, 0] + + +def test_noise_reports_each_query_s_agreement_and_the_spread_between_repeats(): + # Query 1 is answered PERF, PERF, TBB; query 2 is PERF three times. Calls run query by + # query, repeat by repeat, so this script is the order the model is asked in. + script = iter([PERF.name, PERF.name, TBB.name, PERF.name, PERF.name, PERF.name]) + + def transport(prompt: str, system: str | None): + return _answer_as(next(script))(prompt), None + + client = LLMClient(model="m", role="selector", transport=transport) + result = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=3) + noise = result["noise"] + assert result["repeats"] == 3 + assert [q["modal_pick"] for q in noise["per_query"]] == [PERF.name, PERF.name] + assert noise["per_query"][0]["agreement"] == pytest.approx(2 / 3, abs=1e-4) + assert noise["per_query"][1]["agreement"] == 1.0 + assert noise["per_query"][0]["n_correct"] == 2 + assert noise["unstable_queries"] == 1 + # Repeat 2 gets query 1 wrong: the rate goes 1.0, 1.0, 0.5, so the half range is 0.25. + assert noise["selection_rate_by_repeat"] == [1.0, 1.0, 0.5] + assert noise["half_range"] == 0.25 + assert [row["repeat"] for row in result["per_query"]] == [0, 1, 2, 0, 1, 2] + + +def test_a_query_that_failed_on_every_repeat_has_no_agreement_to_report(): + def transport(prompt: str, system: str | None): + raise RuntimeError("provider down") + + client = LLMClient(model="m", role="selector", transport=transport) + noise = model_routing([PERF, TBB], ["q1"], client=client, repeats=2)["noise"] + assert noise == { + "mean_agreement": None, "unstable_queries": 0, + "selection_rate_by_repeat": [None, None], "half_range": None, + "per_query": [{"query": "q1", "modal_pick": None, "agreement": None, "picks": [], + "n_correct": 0}], + } + + +def _completions(monkeypatch, tmp_path, replies: list[str]) -> list[list[dict]]: + """Route the real default transport into scripted completions, with the cache on.""" + import litellm + + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + monkeypatch.setenv("OPENAI_API_KEY", "k") + asked: list[list[dict]] = [] + + def completion(model, messages, api_key=None, **kw): + reply = replies[min(len(asked), len(replies) - 1)] + asked.append(messages) + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=reply), + finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5), + ) + + monkeypatch.setattr(litellm, "completion", completion) + return asked + + +def test_with_the_response_cache_on_each_repeat_is_asked_again(monkeypatch, tmp_path): + # Identical prompts: without a draw index the cache would replay the first reply and + # every repeat would agree by construction, reporting a noise floor of zero. + asked = _completions(monkeypatch, tmp_path, + [json.dumps({"pick": 1}), json.dumps({"pick": 2}), + json.dumps({"pick": 3})]) + client = LLMClient(model="m", provider="openai", role="selector") + result = model_routing([PERF, TBB, MKL], ["q1"], client=client, repeats=3) + assert result["n_failed"] == 0 # a refused draw would read as zero calls + assert len(asked) == 3 + assert [row["picked_at"] for row in result["per_query"]] == [1, 2, 3] + + +def test_a_repeat_run_through_the_cli_writes_the_noise_block(tmp_path, monkeypatch, capsys): + import dokimi.commands.novelty as novelty + + def answer(prompt: str) -> str: + return _answer_as("perf")(prompt) + + monkeypatch.setattr( + novelty, "_client", + lambda model, provider, role, args: LLMClient( + model=model, role=role, transport=lambda p, s: (answer(p), None)), + ) + artifact = _skill(tmp_path, "perf", "record and annotate a perf profile") + competitor = _skill(tmp_path, "tbb", "parallelize C++ loops with task groups") + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["record a perf profile", "profile my binary"]), + encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", "--artifact", artifact, "--competitors", competitor, + "--queries", str(queries), "--model", "m", "--repeats", "2", + "--out", str(out)) + args.func(args) + model = json.loads(out.read_text(encoding="utf-8"))["model"] + assert model["repeats"] == 2 + assert model["noise"]["selection_rate_by_repeat"] == [1.0, 1.0] + assert model["noise"]["half_range"] == 0.0 + assert "noise 2 repeats per query" in capsys.readouterr().out + + +def test_repeats_without_a_model_are_refused_on_the_cli(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--repeats", "3") + with pytest.raises(SystemExit, match="needs --model"): + args.func(args) + + +def test_a_declining_model_is_named_on_the_cli_with_where_its_replies_are( + tmp_path, monkeypatch, capsys): + import dokimi.commands.novelty as novelty + + monkeypatch.setattr( + novelty, "_client", + lambda model, provider, role, args: LLMClient( + model=model, role=role, + transport=lambda p, s: ("I would rather not choose.", None)), + ) + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["record a perf profile"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "record and annotate a perf profile"), + "--competitors", _skill(tmp_path, "tbb", "parallelize C++ loops"), + "--queries", str(queries), "--model", "m", "--out", str(out)) + args.func(args) + model = json.loads(out.read_text(encoding="utf-8"))["model"] + assert model["per_query"][0]["raw_reply"] == "I would rather not choose." + assert "1 abstained or unparseable" in capsys.readouterr().out From 85765e2ddbf7737a23d702362eb37b7fc086b05d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 14:03:51 +0000 Subject: [PATCH 2/9] Add n_calls to model routing; word abstain line per call Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/novelty.py | 5 +++-- dokimi/dokimi/novelty/routing.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index bd023195..707b2aaa 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -362,8 +362,9 @@ def card(ref: str) -> tuple[Candidate, str]: f"({row['selection_rate_among_answered']} among answered; " f"{row['n_abstained']} abstained, {row['n_failed']} failed) [{row['model']}]") if row["n_abstained"]: - print(f"{row['n_abstained']} abstained or unparseable: the raw reply is under " - "per_query[].raw_reply in the output") + of_calls = f" of {row['n_calls']} calls" if row["repeats"] > 1 else "" + print(f"{row['n_abstained']}{of_calls} abstained or unparseable: the raw reply " + "is under per_query[].raw_reply in the output") if row["noise"]: noise = row["noise"] print(f"noise {row['repeats']} repeats per query: mean agreement " diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index de496fb1..e3c0b531 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -251,7 +251,8 @@ def model_routing( sample (``draw``), so with the response cache on a repeat is not a replay. Returns: - ``selection_rate``, ``n_queries``, ``n_abstained``, ``n_failed``, the per-query pick + ``selection_rate``, ``n_queries``, ``n_calls`` (queries x repeats; ``n_abstained`` and + ``n_failed`` count calls), ``n_abstained``, ``n_failed``, the per-query pick with the position the gold card was shown at, the raw reply of every abstention or unparseable pick, and the caveat that this bounds harness behaviour from above. With ``repeats > 1``, ``noise`` holds each query's agreement across its repeats and @@ -311,6 +312,7 @@ def model_routing( "n_candidates": len(candidates), "n_queries": len(queries), "repeats": repeats, + "n_calls": len(picks), "n_abstained": sum(1 for p in picks if p.get("abstained")), "n_failed": sum(1 for p in picks if p.get("error")), "seed": seed, From 2b9a28f00d0c3d9f2f7ab3381cfde0bd64fae95e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:07:57 -0700 Subject: [PATCH 3/9] fix(novelty): noise rates over the queries that never failed; tests for the surviving mutants Each repeat's rate divided by its own non-errored rows, so a transient provider error read as half_range. Rates now cover the queries with no failed call in any repeat, reported as n_queries_compared. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/routing.py | 14 ++++-- dokimi/tests/test_novelty_routing.py | 68 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index e3c0b531..2e40649c 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -341,7 +341,9 @@ def _noise(groups: list[list[dict]]) -> dict: pick; an abstention counts as a pick. ``selection_rate_by_repeat`` reads each repeat as its own run over the same queries, and ``half_range`` is half the gap between the highest and the lowest of those rates. It is a crude sampling noise floor: a few repeats - estimate it coarsely, and it says nothing about a change of cards or queries. + estimate it coarsely, and it says nothing about a change of cards or queries. The + per-repeat rates cover only the queries with no failed call in any repeat + (``n_queries_compared``), so a transient error does not read as noise. Args: groups: One list per query, holding that query's rows in repeat order. @@ -365,16 +367,20 @@ def _noise(groups: list[list[dict]]) -> dict: }) repeats = len(groups[0]) if groups else 0 rates: list[float | None] = [] + # Every repeat's rate is taken over the same queries: those with no failed call in any + # repeat. Dividing each repeat by its own non-errored rows would compare different + # denominators and report a transient provider error as sampling noise. + complete = [group for group in groups if not any(row.get("error") for row in group)] for repeat in range(repeats): - rows = [group[repeat] for group in groups if not group[repeat].get("error")] - rates.append(round(sum(1 for r in rows if r.get("correct")) / len(rows), 4) - if rows else None) + rates.append(round(sum(1 for g in complete if g[repeat].get("correct")) / len(complete), 4) + if complete else None) known = [rate for rate in rates if rate is not None] agreements = [row["agreement"] for row in per_query if row["agreement"] is not None] return { "mean_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, "unstable_queries": sum(1 for a in agreements if a < 1), "selection_rate_by_repeat": rates, + "n_queries_compared": len(complete), "half_range": round((max(known) - min(known)) / 2, 4) if len(known) > 1 else None, "per_query": per_query, } diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index aa701ab2..e9083d1d 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -502,6 +502,13 @@ def test_a_very_long_declined_reply_is_cut_at_the_recording_limit(): assert len(row["raw_reply"]) == 2000 +def test_a_long_declined_reply_keeps_its_head_not_its_tail(): + reply = "a" * 2000 + "b" * 3000 + client = LLMClient(model="m", role="selector", transport=lambda p, s: (reply, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert row["raw_reply"] == reply[:2000] + + def test_repeats_below_one_are_refused(): client = LLMClient(model="m", role="selector", transport=lambda p, s: (json.dumps({"pick": 1}), None)) @@ -549,6 +556,7 @@ def transport(prompt: str, system: str | None): assert noise == { "mean_agreement": None, "unstable_queries": 0, "selection_rate_by_repeat": [None, None], "half_range": None, + "n_queries_compared": 0, "per_query": [{"query": "q1", "modal_pick": None, "agreement": None, "picks": [], "n_correct": 0}], } @@ -648,3 +656,63 @@ def test_a_declining_model_is_named_on_the_cli_with_where_its_replies_are( model = json.loads(out.read_text(encoding="utf-8"))["model"] assert model["per_query"][0]["raw_reply"] == "I would rather not choose." assert "1 abstained or unparseable" in capsys.readouterr().out + + +class _Boom(Exception): + pass + + +def _scripted(script): + """A selector whose n-th call answers a card name, abstains ("-"), or fails ("!").""" + calls = iter(script) + + def transport(prompt: str, system: str | None): + step = next(calls) + if step == "!": + raise _Boom("provider down") + if step == "-": + return json.dumps({"pick": 0}), None + return _answer_as(step)(prompt), None + + return LLMClient(model="m", role="selector", transport=transport) + + +def test_a_failed_repeat_is_not_reported_as_sampling_noise(): + # q1 is wrong in both repeats. q2 fails in repeat 0 and is right in repeat 1. Per-repeat + # rates over each repeat's own answered rows would be [0.0, 0.5]: a half range of 0.25 + # from an error, not from the model. + client = _scripted([TBB.name, TBB.name, "!", PERF.name]) + noise = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=2)["noise"] + assert noise["selection_rate_by_repeat"] == [0.0, 0.0] + assert noise["half_range"] == 0.0 + assert noise["n_queries_compared"] == 1 + + +def test_noise_counts_an_abstention_as_a_pick_and_an_error_as_no_answer(): + # One query, three repeats: PERF, abstained, failed. Agreement is over the two answered + # repeats (1/2), not the three calls (1/3), and the abstention has its own label. + client = _scripted([PERF.name, "-", "!"]) + noise = model_routing([PERF, TBB], ["q1"], client=client, repeats=3)["noise"] + (query,) = noise["per_query"] + assert query["picks"] == [PERF.name, "(abstained)"] + assert query["agreement"] == 0.5 + assert noise["mean_agreement"] == 0.5 and noise["unstable_queries"] == 1 + assert noise["n_queries_compared"] == 0 + assert noise["selection_rate_by_repeat"] == [None, None, None] + assert noise["half_range"] is None + + +def test_an_abstention_can_be_the_modal_pick(): + client = _scripted(["-", "-", PERF.name]) + (query,) = model_routing([PERF, TBB], ["q1"], client=client, repeats=3)["noise"][ + "per_query"] + assert query["modal_pick"] == "(abstained)" + assert query["agreement"] == pytest.approx(2 / 3, abs=1e-4) + + +def test_a_single_usable_repeat_has_no_half_range(): + from dokimi.novelty.routing import _noise + + noise = _noise([[{"query": "q", "repeat": 0, "picked": PERF.name, "correct": True}]]) + assert noise["selection_rate_by_repeat"] == [1.0] + assert noise["half_range"] is None From 8e7cc04fa6bc86cbd67fd5d8ea529b21f51c449c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:39:31 -0700 Subject: [PATCH 4/9] docs(novelty): say the repeat rates cover only queries with no failed call Signed-off-by: Nikolay Petrov --- .../changelog.d/20261021_230000_napetrov_routing-repeats.md | 2 +- dokimi/docs/skill-novelty.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md index 075669b1..4339eda7 100644 --- a/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md +++ b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md @@ -1,3 +1,3 @@ ### Added -- **`novelty routing --repeats N` and the raw reply of a declined pick** — `--repeats N` (with `--model`) asks the model N times per query, with the same card order each time, so the spread between repeats is the model's own sampling noise. The `model` block gains `noise`: each query's agreement across its repeats (`per_query`), the selection rate of each repeat, and `half_range`, half the gap between the highest and lowest of those rates, a crude noise floor. Each repeat is a fresh sample, so with `DOKIMI_RESPONSE_CACHE` set a repeat is no longer a replay of the first reply. Every abstention or unparseable pick now keeps the model's raw reply under `per_query[].raw_reply` (first 2000 characters). `LLMClient.text` takes a `draw` index for the same reason. +- **`novelty routing --repeats N` and the raw reply of a declined pick** — `--repeats N` (with `--model`) asks the model N times per query, with the same card order each time, so the spread between repeats is the model's own sampling noise. The `model` block gains `noise`: each query's agreement across its repeats (`per_query`), the selection rate of each repeat, and `half_range`, half the gap between the highest and lowest of those rates, a crude noise floor. The rates cover only the queries with no failed call in any repeat (`n_queries_compared`). Each repeat is a fresh sample, so with `DOKIMI_RESPONSE_CACHE` set a repeat is no longer a replay of the first reply. Every abstention or unparseable pick now keeps the model's raw reply under `per_query[].raw_reply` (first 2000 characters). `LLMClient.text` takes a `draw` index for the same reason. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 0f7cb9fe..2968ba44 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -612,7 +612,9 @@ Two numbers, reported side by side and never merged: repeats is the model's own sampling noise. The result's `noise` block gives each query's agreement across its repeats (`per_query`), the selection rate of each repeat, and `half_range`, half the gap between the highest and lowest of those rates: a crude noise - floor. Each repeat is a fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a + floor. The rates cover only the queries with no failed call in any repeat + (`n_queries_compared`), so a transient error is not read as noise. Each repeat is a + fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a replay. Repeats need `--model`, and cost N times the calls. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` From 2e1a17688f20fc2907fc4f04d23ea35f3847dcb0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:01:55 -0700 Subject: [PATCH 5/9] refactor(novelty): extract the routing summary; chore(growth): raise commands/novelty.py for W8 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 7 ++++++- dokimi/scripts/growth_baseline.json | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 707b2aaa..fcda5801 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -350,10 +350,15 @@ def card(ref: str) -> tuple[Candidate, str]: out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") + _print_routing_summary(payload, len(competitors), self_derived) + + +def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: + """The lexical line, the model line and its caveats, printed after a routing run.""" lexical = payload["lexical"] print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " f"MRR {lexical['mrr']} over {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} queries vs {len(competitors)} competitors" + f"{lexical['n_queries_given']} queries vs {n_competitors} competitors" + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) if "model" in payload: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..201f2c54 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 853, "rationale": "W8 adds a routing flag; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, From cb514f1b32d9392f1891fd9c199bf26ebc2a8e5b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:02:41 -0700 Subject: [PATCH 6/9] chore(growth): lower dokimi/commands/novelty.py::cmd_novelty_routing baseline after the extraction Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 201f2c54..baae4a86 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 14}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, From 881d5d26bf8108ecf57a7d748b9c12351672238b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 10:57:41 +0000 Subject: [PATCH 7/9] docs(novelty): say what the routing counts mean with --repeats Signed-off-by: Anton Volkov --- dokimi/docs/skill-novelty.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index cb8949c6..4f9632ec 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -616,7 +616,9 @@ Two numbers, reported side by side and never merged: floor. The rates cover only the queries with no failed call in any repeat (`n_queries_compared`), so a transient error is not read as noise. Each repeat is a fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a - replay. Repeats need `--model`, and cost N times the calls. + replay. Repeats need `--model`, and cost N times the calls. With repeats, `n_calls` is + queries x repeats, `n_abstained` and `n_failed` count calls, and each `per_query` row + carries its `repeat` index. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` From bebf8916121324fd80598368963740183e113e31 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:46:57 -0700 Subject: [PATCH 8/9] test(novelty): pin n_calls, per-repeat rate denominator and mean agreement Signed-off-by: Nikolay Petrov --- dokimi/tests/test_novelty_routing.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 392c1f59..f9f9f336 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -737,6 +737,23 @@ def test_noise_counts_an_abstention_as_a_pick_and_an_error_as_no_answer(): assert noise["half_range"] is None +def test_the_output_counts_calls_and_the_noise_means_each_agreement(): + # q1: PERF, PERF, TBB (agreement 2/3). q2: PERF three times (agreement 1.0). + client = _scripted([PERF.name, PERF.name, TBB.name, PERF.name, PERF.name, PERF.name]) + result = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=3) + assert result["n_calls"] == 6 # queries x repeats, not queries + assert result["noise"]["mean_agreement"] == pytest.approx(5 / 6, abs=1e-3) # the mean of 2/3 and 1.0, not the max + + +def test_a_failed_call_elsewhere_does_not_change_the_rate_of_the_complete_queries(): + # q1 is right in both repeats; q2 fails once. The rate is over the complete queries only + # (q1), so 1.0 in each repeat; dividing by all queries would give 0.5. + client = _scripted([PERF.name, PERF.name, "!", TBB.name]) + noise = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=2)["noise"] + assert noise["selection_rate_by_repeat"] == [1.0, 1.0] + assert noise["n_queries_compared"] == 1 + + def test_an_abstention_can_be_the_modal_pick(): client = _scripted(["-", "-", PERF.name]) (query,) = model_routing([PERF, TBB], ["q1"], client=client, repeats=3)["noise"][ From cf7b97fbf93f91c4d0c17ba8266818c901f07a67 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 13:30:54 -0700 Subject: [PATCH 9/9] chore(ci): say why the cmd_novelty_routing complexity baseline is raised Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 4921a23a..eb90bcc4 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -24,7 +24,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "went over the cap on main in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"},