From 09f2ce72af33fdbe597da11a889a5b1f4f4d3b83 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 21:42:55 -0700 Subject: [PATCH 1/8] feat(novelty): usage and cost block for routing --model A model-backed `novelty routing` run wrote no record of what it spent, so a routing rate could not be read against its cost. The run now writes a `cost` block into routing.json (calls, input/output tokens, cache read/write tokens, and usd, null when any call is unpriced) and prints the same totals on stdout. Cache token totals are added to the per-role cost summary, so novelty analyze reports them too. Signed-off-by: Nikolay Petrov --- .../20261021_120000_napetrov_routing-usage.md | 3 + dokimi/docs/skill-novelty.md | 4 ++ dokimi/dokimi/commands/novelty.py | 9 +++ dokimi/dokimi/novelty/client.py | 6 ++ dokimi/tests/test_novelty_routing.py | 58 +++++++++++++++++++ 5 files changed, 80 insertions(+) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md new file mode 100644 index 00000000..fd101f6b --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -0,0 +1,3 @@ +### Added + +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bb..62e205d7 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -607,6 +607,10 @@ Two numbers, reported side by side and never merged: `selection_rate_among_answered` beside it; a failed call is excluded from both and counted, and one failed query does not lose the rest. +A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and +write tokens, and `usd`, which is `null` when any call had no known price) and prints the +same totals on stdout. A run without `--model` makes no calls and has no `cost` block. + `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` next to `n_queries_scored`, since a query with no content words is dropped rather than diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc943..b68ead9f 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -333,11 +333,14 @@ def card(ref: str) -> tuple[Candidate, str]: "lexical": lexical, } if args.model: + from dokimi.novelty.client import roll_up_cost + selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( model_routing(candidates, queries, client=selector, gold=0, seed=args.seed), queries_self_derived=self_derived, ) + payload["cost"] = roll_up_cost([selector]) out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" out.parent.mkdir(parents=True, exist_ok=True) @@ -357,6 +360,12 @@ def card(ref: str) -> tuple[Candidate, str]: f"{row['n_abstained']} abstained, {row['n_failed']} failed) [{row['model']}]") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") + cost = payload["cost"] + spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None + else "USD not known: a call had no price") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " + f"/ write {cost['cache_write_tokens']}, {spent} [{row['model']}]") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index b8579cc8..55fc9713 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -166,6 +166,8 @@ def cost_summary(self) -> dict[str, Any]: """ input_tokens = sum(u.prompt_tokens for u in self.usage) output_tokens = sum(u.completion_tokens for u in self.usage) + cache_read = sum(u.cache_read_tokens for u in self.usage) + cache_write = sum(u.cache_write_tokens for u in self.usage) costs = [u.cost_usd for u in self.usage] priced = costs and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None @@ -176,6 +178,8 @@ def cost_summary(self) -> dict[str, Any]: "llm_calls": self.calls, "input_tokens": input_tokens, "output_tokens": output_tokens, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, "usd": usd, } @@ -230,6 +234,8 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "llm_calls": sum(row["llm_calls"] for row in per_role), "input_tokens": sum(row["input_tokens"] for row in per_role), "output_tokens": sum(row["output_tokens"] for row in per_role), + "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), + "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index cd7ea078..0a981fae 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -473,3 +473,61 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert names.count("perf") == 0, "perf should be excluded from catalog" assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" + + +def _stub_provider(monkeypatch, usage): + """Replace the provider call the --model path makes; every reply picks card 1.""" + from dokimi import llm + + def stub(prompt, model, **_kwargs): + return '{"pick": 1}', usage + + monkeypatch.setattr(llm, "llm_call_with_usage", stub) + + +def _model_run(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cost_usd=0.002)) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["input_tokens"] == 200 + assert payload["cost"]["output_tokens"] == 10 + assert payload["cost"]["cache_read_tokens"] == 120 + assert payload["cost"]["cache_write_tokens"] == 0 + assert payload["cost"]["usd"] == 0.004 + assert "$0.0040" in capsys.readouterr().out + + +def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] is None # one unpriced call must not be read as free + assert "no price" in capsys.readouterr().out + + +def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(out)) + args.func(args) + assert "cost" not in json.loads(out.read_text(encoding="utf-8")) From 14a7e93205db1ee63da557006b356063cde367d3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:07:32 -0700 Subject: [PATCH 2/8] fix(novelty): declare cache token fields and tidy routing cost line Declare cache_read_tokens and cache_write_tokens (integer, minimum 0) in the novelty.v1 cost block, which routing and analyze both write. Say in skill-novelty.md that cache tokens are reported separately and may or may not be inside input_tokens, depending on the provider. The routing test stub now sets cache_write_tokens too, and asserts its total. The stdout cost line drops the trailing model tag, which the model line above already carries. Signed-off-by: Nikolay Petrov --- dokimi/docs/skill-novelty.md | 4 +++- dokimi/dokimi/commands/novelty.py | 2 +- dokimi/dokimi/schemas/novelty.v1.json | 2 ++ dokimi/tests/test_novelty_routing.py | 6 +++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 62e205d7..d6b95ead 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -609,7 +609,9 @@ Two numbers, reported side by side and never merged: A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the -same totals on stdout. A run without `--model` makes no calls and has no `cost` block. +same totals on stdout. Cache tokens are reported separately from input tokens, and depending on +the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes +no calls and has no `cost` block. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index b68ead9f..fada3e85 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -365,7 +365,7 @@ def card(ref: str) -> tuple[Candidate, str]: else "USD not known: a call had no price") print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " - f"/ write {cost['cache_write_tokens']}, {spent} [{row['model']}]") + f"/ write {cost['cache_write_tokens']}, {spent}") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index dc41beec..c0dd76b9 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -447,6 +447,8 @@ "llm_calls": {"type": "integer", "minimum": 0}, "input_tokens": {"type": "integer", "minimum": 0}, "output_tokens": {"type": "integer", "minimum": 0}, + "cache_read_tokens": {"type": "integer", "minimum": 0}, + "cache_write_tokens": {"type": "integer", "minimum": 0}, "usd": { "type": ["number", "null"], "description": "null when any call had unknown pricing — distinct from a genuinely free run." diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 0a981fae..312bf1f9 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -474,7 +474,6 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" - def _stub_provider(monkeypatch, usage): """Replace the provider call the --model path makes; every reply picks card 1.""" from dokimi import llm @@ -501,13 +500,14 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca from dokimi.metrics.usage import UsageRecord _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, - cache_read_tokens=60, cost_usd=0.002)) + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002)) payload = _model_run(tmp_path) assert payload["cost"]["llm_calls"] == 2 assert payload["cost"]["input_tokens"] == 200 assert payload["cost"]["output_tokens"] == 10 assert payload["cost"]["cache_read_tokens"] == 120 - assert payload["cost"]["cache_write_tokens"] == 0 + assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 assert "$0.0040" in capsys.readouterr().out From 714ce0debe9a11ace637f10af3f4592460df7e95 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:44:51 -0700 Subject: [PATCH 3/8] fix(novelty): routing cost line separates no calls, missing usage and cache replays Signed-off-by: Nikolay Petrov --- .../20261021_120000_napetrov_routing-usage.md | 2 +- dokimi/docs/skill-novelty.md | 4 +- dokimi/dokimi/commands/novelty.py | 16 ++++--- dokimi/dokimi/novelty/client.py | 24 ++++++---- dokimi/dokimi/schemas/novelty.v1.json | 1 + dokimi/tests/test_novelty_routing.py | 45 ++++++++++++++++++- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index fd101f6b..ec835918 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,3 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index d6b95ead..040b8963 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -611,7 +611,9 @@ A `--model` run also writes a `cost` block (calls, input and output tokens, cach write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout. Cache tokens are reported separately from input tokens, and depending on the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes -no calls and has no `cost` block. +no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a +call that raised is not counted. A call replayed from the response cache is counted in +`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index fada3e85..b3557942 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -361,11 +361,17 @@ def card(ref: str) -> tuple[Candidate, str]: print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") cost = payload["cost"] - spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None - else "USD not known: a call had no price") - print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " - f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " - f"/ write {cost['cache_write_tokens']}, {spent}") + if not cost["llm_calls"]: + print("cost 0 calls") + else: + spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None + else "USD not known: a call had no price or no usage") + replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost["replayed_calls"] else "") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read " + f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " + f"{spent}{replayed}") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 55fc9713..cec8039c 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -161,15 +161,21 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing, rather than - silently reporting the sum of the priced subset as the total. + ``usd`` stays ``None`` when any call had unknown pricing or returned no usage, + rather than silently reporting the sum of the priced subset as the total. + Calls served from the response cache are counted in ``replayed_calls`` and + left out of the token and dollar totals. """ - input_tokens = sum(u.prompt_tokens for u in self.usage) - output_tokens = sum(u.completion_tokens for u in self.usage) - cache_read = sum(u.cache_read_tokens for u in self.usage) - cache_write = sum(u.cache_write_tokens for u in self.usage) - costs = [u.cost_usd for u in self.usage] - priced = costs and None not in costs + # A replayed row is the response cache's copy of an earlier call: its tokens and + # price were not spent by this process, so it is counted apart, as the spend cap does. + billed = [u for u in self.usage if not u.replayed] + input_tokens = sum(u.prompt_tokens for u in billed) + output_tokens = sum(u.completion_tokens for u in billed) + cache_read = sum(u.cache_read_tokens for u in billed) + cache_write = sum(u.cache_write_tokens for u in billed) + costs = [u.cost_usd for u in billed] + # A call that returned no usage row has no price either, so it makes the total unknown. + priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None return { "model": self.model, @@ -180,6 +186,7 @@ def cost_summary(self) -> dict[str, Any]: "output_tokens": output_tokens, "cache_read_tokens": cache_read, "cache_write_tokens": cache_write, + "replayed_calls": len(self.usage) - len(billed), "usd": usd, } @@ -236,6 +243,7 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "output_tokens": sum(row["output_tokens"] for row in per_role), "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), + "replayed_calls": sum(row["replayed_calls"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index c0dd76b9..f302528d 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -449,6 +449,7 @@ "output_tokens": {"type": "integer", "minimum": 0}, "cache_read_tokens": {"type": "integer", "minimum": 0}, "cache_write_tokens": {"type": "integer", "minimum": 0}, + "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], "description": "null when any call had unknown pricing — distinct from a genuinely free run." diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 312bf1f9..13b5c657 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -509,7 +509,8 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert payload["cost"]["cache_read_tokens"] == 120 assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 - assert "$0.0040" in capsys.readouterr().out + assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " + "$0.0040") in capsys.readouterr().out def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): @@ -518,7 +519,7 @@ def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monke _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) payload = _model_run(tmp_path) assert payload["cost"]["usd"] is None # one unpriced call must not be read as free - assert "no price" in capsys.readouterr().out + assert "USD not known: a call had no price or no usage" in capsys.readouterr().out def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): @@ -531,3 +532,43 @@ def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): "--queries", str(queries), "--out", str(out)) args.func(args) assert "cost" not in json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_whose_every_call_fails_prints_no_price_blame(tmp_path, monkeypatch, capsys): + from dokimi import llm + + def failing(prompt, model, **_kwargs): + raise RuntimeError("provider refused") + + monkeypatch.setattr(llm, "llm_call_with_usage", failing) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 0 + out = capsys.readouterr().out + assert "cost 0 calls\n" in out + assert "USD not known" not in out + + +def test_a_call_with_no_usage_makes_the_total_unknown_not_the_priced_subset( + tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + replies = iter([None, UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001)]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["usd"] is None + assert "$0.0010" not in capsys.readouterr().out + + +def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cost_usd=0.002, replayed=True)) + payload = _model_run(tmp_path) + cost = payload["cost"] + assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) + assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert "2 replayed from the cache and not counted" in capsys.readouterr().out From 54eb79c101ad87147363028351024d782e1fe50c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:08:29 -0700 Subject: [PATCH 4/8] refactor(novelty): extract the routing summary; chore(growth): raise commands/novelty.py for W1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 7 ++++++- dokimi/scripts/growth_baseline.json | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index b3557942..1d2716fd 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -347,10 +347,15 @@ def card(ref: str) -> tuple[Candidate, str]: out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") + _print_routing_summary(payload, len(competitors), self_derived) + + +def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: + """The lexical line, the model line and its caveats, printed after a routing run.""" lexical = payload["lexical"] print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " f"MRR {lexical['mrr']} over {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} queries vs {len(competitors)} competitors" + f"{lexical['n_queries_given']} queries vs {n_competitors} competitors" + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) if "model" in payload: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..aefe430e 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 847, "rationale": "W1 adds routing cost output; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, From f6df38a838e030e65c6f3bb7b1075504e5051e5f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:08:30 -0700 Subject: [PATCH 5/8] chore(growth): lower dokimi/commands/novelty.py::cmd_novelty_routing baseline after the extraction Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index aefe430e..4bf2a57f 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 14}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, From a5b007e47d6a87bd41cf51a273e5662aef0f307e Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 02:28:07 -0700 Subject: [PATCH 6/8] test(novelty): pin roll_up_cost cache and replay totals across roles roll_up_cost sums the new cache_read/cache_write/replayed_calls fields over every role, but only single-client tests covered them, so a total taken from the first role alone (what novelty analyze would then report) passed. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_measure.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index a94373a8..152b1810 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -221,6 +221,30 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced(): assert by_role["judge"]["usd"] is None +def test_cost_rolls_up_cache_tokens_and_replays_across_every_role(): + def client(role, records): + rows = iter(records) + return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows))) + + extractor = client("extractor", [ + UsageRecord(prompt_tokens=10, cache_read_tokens=4, cache_write_tokens=1, cost_usd=0.1), + UsageRecord(prompt_tokens=10, cost_usd=0.1, replayed=True), + ]) + judge = client("judge", [ + UsageRecord(prompt_tokens=20, cache_read_tokens=8, cache_write_tokens=4, cost_usd=0.2), + UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True), + ]) + for c in (extractor, judge): + c.text("p") + c.text("p") + total = roll_up_cost([extractor, judge]) + # analyze rolls up several roles: a total taken from one of them would pass a + # single-client test, so each new field is summed over two. + assert total["llm_calls"] == 4 + assert total["cache_read_tokens"] == 12 + assert total["cache_write_tokens"] == 5 + assert total["replayed_calls"] == 2 + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): From 54f4ca8b5d01a15f32c7cbb25286da8b32cd009c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 12:59:42 +0000 Subject: [PATCH 7/8] style(tests): add the second blank line before the next top-level test Signed-off-by: Anton Volkov --- dokimi/tests/test_novelty_measure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index 152b1810..7191db32 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -245,6 +245,7 @@ def client(role, records): assert total["cache_write_tokens"] == 5 assert total["replayed_calls"] == 2 + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): From 2594a80d8e2464b7d39f9913817df6fda9fa029f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:41:44 -0700 Subject: [PATCH 8/8] test(novelty): pin usd None for no calls and replayed cache tokens; say 'this run paid for' Signed-off-by: Nikolay Petrov --- .../changelog.d/20261021_120000_napetrov_routing-usage.md | 2 +- dokimi/docs/skill-novelty.md | 2 +- dokimi/dokimi/novelty/client.py | 3 ++- dokimi/dokimi/schemas/novelty.v1.json | 2 +- dokimi/tests/test_novelty_routing.py | 6 +++++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index ec835918..77ee46b5 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,3 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index b1dd6df2..e7fbe633 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -609,7 +609,7 @@ Two numbers, reported side by side and never merged: counted, and one failed query does not lose the rest. A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and -write tokens, and `usd`, which is `null` when any call had no known price) and prints the +write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout. Cache tokens are reported separately from input tokens, and depending on the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index cec8039c..4f53b46b 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -161,7 +161,8 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing or returned no usage, + ``usd`` stays ``None`` when any call this run paid for had unknown pricing or + returned no usage (a replayed call is not paid for, so it cannot make it unknown), rather than silently reporting the sum of the priced subset as the total. Calls served from the response cache are counted in ``replayed_calls`` and left out of the token and dollar totals. diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index f302528d..5c7df79d 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -452,7 +452,7 @@ "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], - "description": "null when any call had unknown pricing — distinct from a genuinely free run." + "description": "null when any call this run paid for had unknown pricing — distinct from a genuinely free run." }, "by_role": {"type": "array", "items": {"type": "object"}} } diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index b644e28f..4c4702a2 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -545,7 +545,8 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " - "$0.0040") in capsys.readouterr().out + "$0.0040") in (out := capsys.readouterr().out) + assert "replayed" not in out def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): @@ -578,6 +579,7 @@ def failing(prompt, model, **_kwargs): monkeypatch.setattr(llm, "llm_call_with_usage", failing) payload = _model_run(tmp_path) assert payload["cost"]["llm_calls"] == 0 + assert payload["cost"]["usd"] is None # no calls: nothing known, not "free" out = capsys.readouterr().out assert "cost 0 calls\n" in out assert "USD not known" not in out @@ -601,11 +603,13 @@ def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, from dokimi.metrics.usage import UsageRecord _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, cost_usd=0.002, replayed=True)) payload = _model_run(tmp_path) cost = payload["cost"] assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0) assert "2 replayed from the cache and not counted" in capsys.readouterr().out