diff --git a/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md new file mode 100644 index 00000000..4339eda7 --- /dev/null +++ b/dokimi/changelog.d/20261021_230000_napetrov_routing-repeats.md @@ -0,0 +1,3 @@ +### Added + +- **`novelty routing --repeats N` and the raw reply of a declined pick** — `--repeats N` (with `--model`) asks the model N times per query, with the same card order each time, so the spread between repeats is the model's own sampling noise. The `model` block gains `noise`: each query's agreement across its repeats (`per_query`), the selection rate of each repeat, and `half_range`, half the gap between the highest and lowest of those rates, a crude noise floor. The rates cover only the queries with no failed call in any repeat (`n_queries_compared`). Each repeat is a fresh sample, so with `DOKIMI_RESPONSE_CACHE` set a repeat is no longer a replay of the first reply. Every abstention or unparseable pick now keeps the model's raw reply under `per_query[].raw_reply` (first 2000 characters). `LLMClient.text` takes a `draw` index for the same reason. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index f0ef48df..39bc931d 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -588,7 +588,7 @@ the absence of detectable corruption — not proof that the wordings are faithfu ```bash python cli.py novelty routing --artifact skill:path/to/skill \ --competitors skill:sibling-a skill:sibling-b ... \ - --queries queries.json [--model M] + --queries queries.json [--model M [--repeats N]] ``` `--catalog ` takes the competitors from a skill directory instead: every `SKILL.md` @@ -608,6 +608,18 @@ Two numbers, reported side by side and never merged: `selection_rate`** — not picking anything is the routing failure — with `selection_rate_among_answered` beside it; a failed call is excluded from both and counted, and one failed query does not lose the rest. + Each abstention or unparseable pick keeps the model's raw reply under + `per_query[].raw_reply` (the first 2000 characters), so a decline can be read. + `--repeats N` asks each query N times with the same card order, so the spread between + repeats is the model's own sampling noise. The result's `noise` block gives each query's + agreement across its repeats (`per_query`), the selection rate of each repeat, and + `half_range`, half the gap between the highest and lowest of those rates: a crude noise + floor. The rates cover only the queries with no failed call in any repeat + (`n_queries_compared`), so a transient error is not read as noise. Each repeat is a + fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a + replay. Repeats need `--model`, and cost N times the calls. With repeats, `n_calls` is + queries x repeats, `n_abstained` and `n_failed` count calls, and each `per_query` row + carries its `repeat` index. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 6afb318f..e351f12d 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -264,6 +264,11 @@ def cmd_novelty_items(args: argparse.Namespace) -> None: def cmd_novelty_routing(args: argparse.Namespace) -> None: """Can this artifact be selected against its competitors? Lexical floor, model on top.""" + if args.repeats > 1 and not args.model: + raise SystemExit( + "--repeats needs --model: the lexical floor is deterministic, so repeating it " + "measures nothing" + ) from dokimi.novelty.routing import ( Candidate, RoutingInputError, @@ -338,7 +343,8 @@ def card(ref: str) -> tuple[Candidate, str]: if args.model: selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( - model_routing(candidates, queries, client=selector, gold=0, seed=args.seed), + model_routing(candidates, queries, client=selector, gold=0, seed=args.seed, + repeats=args.repeats), queries_self_derived=self_derived, ) @@ -369,6 +375,17 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool f"({row['selection_rate_among_answered']} among answered; " f"{row['n_abstained']} abstained, {row['n_failed']} failed) " f"[{row['model']}] n_queries {row['n_queries']} given{tag}") + if row["n_abstained"]: + of_calls = f" of {row['n_calls']} calls" if row["repeats"] > 1 else "" + print(f"{row['n_abstained']}{of_calls} abstained or unparseable: the raw reply " + "is under per_query[].raw_reply in the output") + if row["noise"]: + noise = row["noise"] + print(f"noise {row['repeats']} repeats per query: mean agreement " + f"{noise['mean_agreement']}, {noise['unstable_queries']} queries picked " + f"differently across repeats; selection by repeat " + f"{noise['selection_rate_by_repeat']}, half range " + f"{noise['half_range']} (a crude sampling floor)") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") if self_derived: @@ -640,6 +657,10 @@ def register(sub, positive_int) -> None: routing_p.add_argument("--seed", type=int, default=0, help="Seeds the per-query card permutation, so a position-biased " "model scores ~1/N and the run is reproducible") + routing_p.add_argument("--repeats", type=positive_int, default=1, + help="Ask the model this many times per query, with the same card " + "order each time, and report per-query agreement and the " + "run-to-run noise floor. Needs --model; costs N times the calls") routing_p.add_argument("--out", default=None) _add_spend_cap(routing_p) routing_p.set_defaults(func=_stops_at_cap(cmd_novelty_routing)) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index b8579cc8..ae65133a 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -106,7 +106,8 @@ class LLMClient: cap: SpendCap | None = None def text( - self, prompt: str, *, system: str | None = None, accept: Accept | None = None + self, prompt: str, *, system: str | None = None, accept: Accept | None = None, + draw: int = 0, ) -> str: """Return the model's reply as plain text. @@ -115,13 +116,18 @@ def text( ``DOKIMI_RESPONSE_CACHE`` set, a cut-off or unparseable reply is stored and every later run replays the failure without calling the model. An injected transport is the caller's own and is not given it. + + ``draw`` is the sample index of this prompt. With the response cache on, a repeat + of a byte-identical prompt replays draw 0 unless it has its own index, so a + repeated ask is only a new sample when it passes a new one. Draw 0 is the default + and keeps every existing cache entry. An injected transport is not given it. """ if self.cap is not None: self.cap.before_call(self.model) if self.transport is not None: reply, usage = self.transport(prompt, system) else: - reply, usage = self._default_transport(prompt, system, accept=accept) + reply, usage = self._default_transport(prompt, system, accept=accept, draw=draw) self.calls += 1 if usage is not None: # UsageRecord is frozen; re-stamp the role so cost rolls up per role. @@ -180,7 +186,8 @@ def cost_summary(self) -> dict[str, Any]: } def _default_transport( - self, prompt: str, system: str | None, *, accept: Accept | None = None + self, prompt: str, system: str | None, *, accept: Accept | None = None, + draw: int = 0, ) -> tuple[str, UsageRecord | None]: from dokimi.llm import llm_call_with_usage @@ -191,6 +198,7 @@ def _default_transport( system=system, max_tokens=self.max_tokens, cache_if=accept, + draw=draw, ) diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 2ec7cf0b..341c17d1 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -229,11 +229,15 @@ def _as_pick_position(value: object) -> int: return 0 +# A reply that declines to route is kept verbatim, up to this many characters. +_RAW_REPLY_CHARS = 2000 + + def model_routing( candidates: list[Candidate], queries: list[str], *, client: LLMClient, gold: int = 0, - seed: int = 0, + seed: int = 0, repeats: int = 1, ) -> dict: - """Ask a model to pick from the same cards, one call per query. + """Ask a model to pick from the same cards, ``repeats`` times per query. The candidate order is a seeded random permutation per query, so a model that always picks the first (or last) option scores about ``1/N`` rather than a figure set by how @@ -242,14 +246,24 @@ def model_routing( second-to-last -- exactly the positions where primacy and recency bias are strongest. The permutation is recorded per query, so a result can be re-read or replayed. + Each query keeps one card order across its repeats, so the spread between repeats is + the model's own sampling noise rather than a change of layout. Each repeat is its own + sample (``draw``), so with the response cache on a repeat is not a replay. + Returns: - ``selection_rate``, ``n_queries``, ``n_abstained``, ``n_failed``, the per-query pick - with the position the gold card was shown at, and the caveat that this bounds - harness behaviour from above. + ``selection_rate``, ``n_queries``, ``n_calls`` (queries x repeats; ``n_abstained`` and + ``n_failed`` count calls), ``n_abstained``, ``n_failed``, the per-query pick + with the position the gold card was shown at, the raw reply of every abstention or + unparseable pick, and the caveat that this bounds harness behaviour from above. + With ``repeats > 1``, ``noise`` holds each query's agreement across its repeats and + the run-to-run spread of the selection rate; it is ``None`` for one repeat. """ if not 0 <= gold < len(candidates): raise ValueError(f"gold index {gold} is outside 0..{len(candidates) - 1}") + if repeats < 1: + raise ValueError(f"repeats must be at least 1, not {repeats}") picks: list[dict] = [] + groups: list[list[dict]] = [] for index_of_query, query in enumerate(queries): shuffler = random.Random(f"{seed}:{index_of_query}:{len(candidates)}") order = list(range(len(candidates))) @@ -258,28 +272,36 @@ def model_routing( f"{position + 1}. {candidates[index].card}" for position, index in enumerate(order) ) - try: - reply = client.text( - _SELECT_PROMPT.format(query=query, cards=cards), - accept=partial(_is_pick_reply, n_cards=len(order)), - ) - except Exception as exc: # noqa: BLE001 - one failed query must not lose the rest - picks.append({"query": query, "picked": None, "error": str(exc)[:200]}) - continue - # A reply that is not a usable pick is the model declining to route, not a - # transport failure: it stays in the denominator like an explicit abstention. - try: - raw = _as_pick_position(parse_json_object(reply).get("pick", 0)) - except ValueError: - raw = 0 - if raw < 1 or raw > len(order): - picks.append({"query": query, "picked": None, "abstained": True}) - continue - chosen = order[raw - 1] - picks.append({"query": query, "picked": candidates[chosen].name, - "correct": chosen == gold, "picked_at": raw, - "gold_shown_at": order.index(gold) + 1, - "order": [candidates[i].name for i in order]}) + prompt = _SELECT_PROMPT.format(query=query, cards=cards) + group: list[dict] = [] + for repeat in range(repeats): + base = {"query": query, "repeat": repeat} + try: + reply = client.text( + prompt, accept=partial(_is_pick_reply, n_cards=len(order)), draw=repeat, + ) + except Exception as exc: # noqa: BLE001 - one failed query must not lose the rest + row = {**base, "picked": None, "error": str(exc)[:200]} + else: + # A reply that is not a usable pick is the model declining to route, not a + # transport failure: it stays in the denominator like an explicit abstention, + # and its raw text is kept so the decline can be read rather than inferred. + try: + raw = _as_pick_position(parse_json_object(reply).get("pick", 0)) + except ValueError: + raw = 0 + if raw < 1 or raw > len(order): + row = {**base, "picked": None, "abstained": True, + "raw_reply": reply[:_RAW_REPLY_CHARS]} + else: + chosen = order[raw - 1] + row = {**base, "picked": candidates[chosen].name, + "correct": chosen == gold, "picked_at": raw, + "gold_shown_at": order.index(gold) + 1, + "order": [candidates[i].name for i in order]} + picks.append(row) + group.append(row) + groups.append(group) answered = [p for p in picks if "correct" in p] hits = sum(1 for p in answered if p["correct"]) @@ -289,6 +311,8 @@ def model_routing( "model": client.model, "n_candidates": len(candidates), "n_queries": len(queries), + "repeats": repeats, + "n_calls": len(picks), "n_abstained": sum(1 for p in picks if p.get("abstained")), "n_failed": sum(1 for p in picks if p.get("error")), "seed": seed, @@ -300,6 +324,7 @@ def model_routing( "selection_rate_among_answered": ( round(hits / len(answered), 4) if answered else None ), + "noise": _noise(groups) if repeats > 1 else None, "per_query": picks, "note": ( "An upper bound on harness behaviour, not a harness measurement: a harness " @@ -309,6 +334,58 @@ def model_routing( } +def _noise(groups: list[list[dict]]) -> dict: + """Agreement across the repeats of each query, and the run-to-run spread of the rate. + + A query's agreement is the share of its answered repeats that gave its most common + pick; an abstention counts as a pick. ``selection_rate_by_repeat`` reads each repeat as + its own run over the same queries, and ``half_range`` is half the gap between the + highest and the lowest of those rates. It is a crude sampling noise floor: a few repeats + estimate it coarsely, and it says nothing about a change of cards or queries. The + per-repeat rates cover only the queries with no failed call in any repeat + (``n_queries_compared``), so a transient error does not read as noise. + + Args: + groups: One list per query, holding that query's rows in repeat order. + """ + per_query: list[dict] = [] + for group in groups: + labels = [ + "(abstained)" if row.get("abstained") else row["picked"] + for row in group if not row.get("error") + ] + modal, agreement = None, None + if labels: + modal, count = Counter(labels).most_common(1)[0] + agreement = round(count / len(labels), 4) + per_query.append({ + "query": group[0]["query"], + "modal_pick": modal, + "agreement": agreement, + "picks": labels, + "n_correct": sum(1 for row in group if row.get("correct")), + }) + repeats = len(groups[0]) if groups else 0 + rates: list[float | None] = [] + # Every repeat's rate is taken over the same queries: those with no failed call in any + # repeat. Dividing each repeat by its own non-errored rows would compare different + # denominators and report a transient provider error as sampling noise. + complete = [group for group in groups if not any(row.get("error") for row in group)] + for repeat in range(repeats): + rates.append(round(sum(1 for g in complete if g[repeat].get("correct")) / len(complete), 4) + if complete else None) + known = [rate for rate in rates if rate is not None] + agreements = [row["agreement"] for row in per_query if row["agreement"] is not None] + return { + "mean_agreement": round(sum(agreements) / len(agreements), 4) if agreements else None, + "unstable_queries": sum(1 for a in agreements if a < 1), + "selection_rate_by_repeat": rates, + "n_queries_compared": len(complete), + "half_range": round((max(known) - min(known)) / 2, 4) if len(known) > 1 else None, + "per_query": per_query, + } + + # Only spaces indent a fence (a tab is four columns, so it makes indented code), and a # backtick opener may not carry a backtick in its info string; both are CommonMark rules. _FENCE = re.compile(r" {0,3}(`{3,}(?!.*`)|~{3,})") diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index c0456236..642f99b9 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line; file split tracked separately to avoid conflicts across the W-series"}, + "dokimi/commands/novelty.py": {"lines": 885, "rationale": "W3 adds routing no_queries handling, W8 a routing flag, W10 a --catalog help line; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, @@ -24,7 +24,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "went over the cap on main in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 18c6ac60..f9f9f336 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +from types import SimpleNamespace import pytest @@ -509,6 +510,266 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" + +def _answer_as(name: str): + """A transport reply that picks the card called ``name``, wherever it is shown.""" + def reply(prompt: str) -> str: + lines = prompt.split("TOOLS:", 1)[1].strip().splitlines() + return json.dumps({"pick": next(i + 1 for i, line in enumerate(lines) + if line.split(" ", 1)[1].startswith(name))}) + return reply + + +@pytest.mark.parametrize("reply", [ + "I think the best one is the third.", '{"pick": 0}', '{"pick": 999}', +]) +def test_a_declined_reply_is_recorded_verbatim(reply): + client = LLMClient(model="m", role="selector", transport=lambda p, s: (reply, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert row["abstained"] is True + assert row["raw_reply"] == reply + + +def test_a_very_long_declined_reply_is_cut_at_the_recording_limit(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: ("x" * 5000, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert len(row["raw_reply"]) == 2000 + + +def test_a_long_declined_reply_keeps_its_head_not_its_tail(): + reply = "a" * 2000 + "b" * 3000 + client = LLMClient(model="m", role="selector", transport=lambda p, s: (reply, None)) + row = model_routing([PERF, TBB], ["q1"], client=client)["per_query"][0] + assert row["raw_reply"] == reply[:2000] + + +def test_repeats_below_one_are_refused(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: (json.dumps({"pick": 1}), None)) + with pytest.raises(ValueError, match="at least 1"): + model_routing([PERF, TBB], ["q1"], client=client, repeats=0) + + +def test_one_repeat_has_no_noise_block(): + client = LLMClient(model="m", role="selector", + transport=lambda p, s: (json.dumps({"pick": 1}), None)) + result = model_routing([PERF, TBB], ["q1", "q2"], client=client) + assert (result["repeats"], result["noise"]) == (1, None) + assert [row["repeat"] for row in result["per_query"]] == [0, 0] + + +def test_noise_reports_each_query_s_agreement_and_the_spread_between_repeats(): + # Query 1 is answered PERF, PERF, TBB; query 2 is PERF three times. Calls run query by + # query, repeat by repeat, so this script is the order the model is asked in. + script = iter([PERF.name, PERF.name, TBB.name, PERF.name, PERF.name, PERF.name]) + + def transport(prompt: str, system: str | None): + return _answer_as(next(script))(prompt), None + + client = LLMClient(model="m", role="selector", transport=transport) + result = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=3) + noise = result["noise"] + assert result["repeats"] == 3 + assert [q["modal_pick"] for q in noise["per_query"]] == [PERF.name, PERF.name] + assert noise["per_query"][0]["agreement"] == pytest.approx(2 / 3, abs=1e-4) + assert noise["per_query"][1]["agreement"] == 1.0 + assert noise["per_query"][0]["n_correct"] == 2 + assert noise["unstable_queries"] == 1 + # Repeat 2 gets query 1 wrong: the rate goes 1.0, 1.0, 0.5, so the half range is 0.25. + assert noise["selection_rate_by_repeat"] == [1.0, 1.0, 0.5] + assert noise["half_range"] == 0.25 + assert [row["repeat"] for row in result["per_query"]] == [0, 1, 2, 0, 1, 2] + + +def test_a_query_that_failed_on_every_repeat_has_no_agreement_to_report(): + def transport(prompt: str, system: str | None): + raise RuntimeError("provider down") + + client = LLMClient(model="m", role="selector", transport=transport) + noise = model_routing([PERF, TBB], ["q1"], client=client, repeats=2)["noise"] + assert noise == { + "mean_agreement": None, "unstable_queries": 0, + "selection_rate_by_repeat": [None, None], "half_range": None, + "n_queries_compared": 0, + "per_query": [{"query": "q1", "modal_pick": None, "agreement": None, "picks": [], + "n_correct": 0}], + } + + +def _completions(monkeypatch, tmp_path, replies: list[str]) -> list[list[dict]]: + """Route the real default transport into scripted completions, with the cache on.""" + import litellm + + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + monkeypatch.setenv("OPENAI_API_KEY", "k") + asked: list[list[dict]] = [] + + def completion(model, messages, api_key=None, **kw): + reply = replies[min(len(asked), len(replies) - 1)] + asked.append(messages) + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=reply), + finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5), + ) + + monkeypatch.setattr(litellm, "completion", completion) + return asked + + +def test_with_the_response_cache_on_each_repeat_is_asked_again(monkeypatch, tmp_path): + # Identical prompts: without a draw index the cache would replay the first reply and + # every repeat would agree by construction, reporting a noise floor of zero. + asked = _completions(monkeypatch, tmp_path, + [json.dumps({"pick": 1}), json.dumps({"pick": 2}), + json.dumps({"pick": 3})]) + client = LLMClient(model="m", provider="openai", role="selector") + result = model_routing([PERF, TBB, MKL], ["q1"], client=client, repeats=3) + assert result["n_failed"] == 0 # a refused draw would read as zero calls + assert len(asked) == 3 + assert [row["picked_at"] for row in result["per_query"]] == [1, 2, 3] + + +def test_a_repeat_run_through_the_cli_writes_the_noise_block(tmp_path, monkeypatch, capsys): + import dokimi.commands.novelty as novelty + + def answer(prompt: str) -> str: + return _answer_as("perf")(prompt) + + monkeypatch.setattr( + novelty, "_client", + lambda model, provider, role, args: LLMClient( + model=model, role=role, transport=lambda p, s: (answer(p), None)), + ) + artifact = _skill(tmp_path, "perf", "record and annotate a perf profile") + competitor = _skill(tmp_path, "tbb", "parallelize C++ loops with task groups") + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["record a perf profile", "profile my binary"]), + encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", "--artifact", artifact, "--competitors", competitor, + "--queries", str(queries), "--model", "m", "--repeats", "2", + "--out", str(out)) + args.func(args) + model = json.loads(out.read_text(encoding="utf-8"))["model"] + assert model["repeats"] == 2 + assert model["noise"]["selection_rate_by_repeat"] == [1.0, 1.0] + assert model["noise"]["half_range"] == 0.0 + assert "noise 2 repeats per query" in capsys.readouterr().out + + +def test_repeats_without_a_model_are_refused_on_the_cli(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--repeats", "3") + with pytest.raises(SystemExit, match="needs --model"): + args.func(args) + + +def test_a_declining_model_is_named_on_the_cli_with_where_its_replies_are( + tmp_path, monkeypatch, capsys): + import dokimi.commands.novelty as novelty + + monkeypatch.setattr( + novelty, "_client", + lambda model, provider, role, args: LLMClient( + model=model, role=role, + transport=lambda p, s: ("I would rather not choose.", None)), + ) + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["record a perf profile"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "record and annotate a perf profile"), + "--competitors", _skill(tmp_path, "tbb", "parallelize C++ loops"), + "--queries", str(queries), "--model", "m", "--out", str(out)) + args.func(args) + model = json.loads(out.read_text(encoding="utf-8"))["model"] + assert model["per_query"][0]["raw_reply"] == "I would rather not choose." + assert "1 abstained or unparseable" in capsys.readouterr().out + + +class _Boom(Exception): + pass + + +def _scripted(script): + """A selector whose n-th call answers a card name, abstains ("-"), or fails ("!").""" + calls = iter(script) + + def transport(prompt: str, system: str | None): + step = next(calls) + if step == "!": + raise _Boom("provider down") + if step == "-": + return json.dumps({"pick": 0}), None + return _answer_as(step)(prompt), None + + return LLMClient(model="m", role="selector", transport=transport) + + +def test_a_failed_repeat_is_not_reported_as_sampling_noise(): + # q1 is wrong in both repeats. q2 fails in repeat 0 and is right in repeat 1. Per-repeat + # rates over each repeat's own answered rows would be [0.0, 0.5]: a half range of 0.25 + # from an error, not from the model. + client = _scripted([TBB.name, TBB.name, "!", PERF.name]) + noise = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=2)["noise"] + assert noise["selection_rate_by_repeat"] == [0.0, 0.0] + assert noise["half_range"] == 0.0 + assert noise["n_queries_compared"] == 1 + + +def test_noise_counts_an_abstention_as_a_pick_and_an_error_as_no_answer(): + # One query, three repeats: PERF, abstained, failed. Agreement is over the two answered + # repeats (1/2), not the three calls (1/3), and the abstention has its own label. + client = _scripted([PERF.name, "-", "!"]) + noise = model_routing([PERF, TBB], ["q1"], client=client, repeats=3)["noise"] + (query,) = noise["per_query"] + assert query["picks"] == [PERF.name, "(abstained)"] + assert query["agreement"] == 0.5 + assert noise["mean_agreement"] == 0.5 and noise["unstable_queries"] == 1 + assert noise["n_queries_compared"] == 0 + assert noise["selection_rate_by_repeat"] == [None, None, None] + assert noise["half_range"] is None + + +def test_the_output_counts_calls_and_the_noise_means_each_agreement(): + # q1: PERF, PERF, TBB (agreement 2/3). q2: PERF three times (agreement 1.0). + client = _scripted([PERF.name, PERF.name, TBB.name, PERF.name, PERF.name, PERF.name]) + result = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=3) + assert result["n_calls"] == 6 # queries x repeats, not queries + assert result["noise"]["mean_agreement"] == pytest.approx(5 / 6, abs=1e-3) # the mean of 2/3 and 1.0, not the max + + +def test_a_failed_call_elsewhere_does_not_change_the_rate_of_the_complete_queries(): + # q1 is right in both repeats; q2 fails once. The rate is over the complete queries only + # (q1), so 1.0 in each repeat; dividing by all queries would give 0.5. + client = _scripted([PERF.name, PERF.name, "!", TBB.name]) + noise = model_routing([PERF, TBB, MKL], ["q1", "q2"], client=client, repeats=2)["noise"] + assert noise["selection_rate_by_repeat"] == [1.0, 1.0] + assert noise["n_queries_compared"] == 1 + + +def test_an_abstention_can_be_the_modal_pick(): + client = _scripted(["-", "-", PERF.name]) + (query,) = model_routing([PERF, TBB], ["q1"], client=client, repeats=3)["noise"][ + "per_query"] + assert query["modal_pick"] == "(abstained)" + assert query["agreement"] == pytest.approx(2 / 3, abs=1e-4) + + +def test_a_single_usable_repeat_has_no_half_range(): + from dokimi.novelty.routing import _noise + + noise = _noise([[{"query": "q", "repeat": 0, "picked": PERF.name, "correct": True}]]) + assert noise["selection_rate_by_repeat"] == [1.0] + assert noise["half_range"] is None + + def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path): out = tmp_path / "routing.json" args = _parse("novelty", "routing",