feat(novelty): usage and cost block for routing --model

dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md

@@ -0,0 +1,3 @@
+### Added
+
+- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage.

dokimi/docs/skill-novelty.md

@@ -609,6 +609,14 @@ Two numbers, reported side by side and never merged:
   `selection_rate_among_answered` beside it; a failed call is excluded from both and
   counted, and one failed query does not lose the rest.
 
+A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and
+write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the
+same totals on stdout. Cache tokens are reported separately from input tokens, and depending on
+the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes
+no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a
+call that raised is not counted. A call replayed from the response cache is counted in
+`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it.
+
 `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3`
 and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given`
 next to `n_queries_scored`, since a query with no content words is dropped rather than

dokimi/dokimi/commands/novelty.py

@@ -336,17 +336,21 @@ def card(ref: str) -> tuple[Candidate, str]:
         "lexical": lexical,
     }
     if args.model:
+        from dokimi.novelty.client import roll_up_cost
+
         selector = _client(args.model, args.provider, "selector", args)
         payload["model"] = dict(
             model_routing(candidates, queries, client=selector, gold=0, seed=args.seed),
             queries_self_derived=self_derived,
         )
+        payload["cost"] = roll_up_cost([selector])
 
     out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
     print(f"OK Saved routing measurement: {out}")
 
     _print_routing_summary(payload, len(competitors), self_derived)
 
+
 def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None:
     """The lexical line, the model line and their caveats, printed after a routing run."""
     # Every rate is printed with the number of queries behind it, and a self-derived one
@@ -371,11 +375,22 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool
               f"[{row['model']}]  n_queries {row['n_queries']} given{tag}")
         print("The model figure is an upper bound on harness behaviour: a harness that "
               "truncates descriptions in its own listing selects on less text.")
+        cost = payload["cost"]
+        if not cost["llm_calls"]:
+            print("cost     0 calls")
+        else:
+            spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None
+                     else "USD not known: a call had no price or no usage")
+            replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted"
+                        if cost["replayed_calls"] else "")
+            print(f"cost     {cost['llm_calls']} calls, {cost['input_tokens']} input / "
+                  f"{cost['output_tokens']} output tokens, cache read "
+                  f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, "
+                  f"{spent}{replayed}")
     if self_derived:
         print("Self-derived queries: read as triage only.")
 
 
-
 def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool,
                        out: Path, n_candidates: int) -> NoReturn:
     """Write the ``no_queries`` record, then exit saying why no query was scored."""

dokimi/dokimi/novelty/client.py

@@ -161,13 +161,22 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]:
     def cost_summary(self) -> dict[str, Any]:
         """Roll up this client's usage.
 
-        ``usd`` stays ``None`` when any call had unknown pricing, rather than
-        silently reporting the sum of the priced subset as the total.
+        ``usd`` stays ``None`` when any call this run paid for had unknown pricing or
+        returned no usage (a replayed call is not paid for, so it cannot make it unknown),
+        rather than silently reporting the sum of the priced subset as the total.
+        Calls served from the response cache are counted in ``replayed_calls`` and
+        left out of the token and dollar totals.
         """
-        input_tokens = sum(u.prompt_tokens for u in self.usage)
-        output_tokens = sum(u.completion_tokens for u in self.usage)
-        costs = [u.cost_usd for u in self.usage]
-        priced = costs and None not in costs
+        # A replayed row is the response cache's copy of an earlier call: its tokens and
+        # price were not spent by this process, so it is counted apart, as the spend cap does.
+        billed = [u for u in self.usage if not u.replayed]
+        input_tokens = sum(u.prompt_tokens for u in billed)
+        output_tokens = sum(u.completion_tokens for u in billed)
+        cache_read = sum(u.cache_read_tokens for u in billed)
+        cache_write = sum(u.cache_write_tokens for u in billed)
+        costs = [u.cost_usd for u in billed]
+        # A call that returned no usage row has no price either, so it makes the total unknown.
+        priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs
         usd = round(sum(c for c in costs if c is not None), 6) if priced else None
         return {
             "model": self.model,
@@ -176,6 +185,9 @@ def cost_summary(self) -> dict[str, Any]:
             "llm_calls": self.calls,
             "input_tokens": input_tokens,
             "output_tokens": output_tokens,
+            "cache_read_tokens": cache_read,
+            "cache_write_tokens": cache_write,
+            "replayed_calls": len(self.usage) - len(billed),
             "usd": usd,
         }
 
@@ -230,6 +242,9 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]:
         "llm_calls": sum(row["llm_calls"] for row in per_role),
         "input_tokens": sum(row["input_tokens"] for row in per_role),
         "output_tokens": sum(row["output_tokens"] for row in per_role),
+        "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role),
+        "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role),
+        "replayed_calls": sum(row["replayed_calls"] for row in per_role),
         "usd": round(sum(p for p in usd_parts if p is not None), 6)
         if usd_parts and None not in usd_parts
         else None,

dokimi/dokimi/schemas/novelty.v1.json

@@ -447,9 +447,12 @@
         "llm_calls": {"type": "integer", "minimum": 0},
         "input_tokens": {"type": "integer", "minimum": 0},
         "output_tokens": {"type": "integer", "minimum": 0},
+        "cache_read_tokens": {"type": "integer", "minimum": 0},
+        "cache_write_tokens": {"type": "integer", "minimum": 0},
+        "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."},
         "usd": {
           "type": ["number", "null"],
-          "description": "null when any call had unknown pricing — distinct from a genuinely free run."
+          "description": "null when any call this run paid for had unknown pricing — distinct from a genuinely free run."
         },
         "by_role": {"type": "array", "items": {"type": "object"}}
       }

dokimi/scripts/growth_baseline.json

@@ -1,6 +1,6 @@
 {
   "files": {
-    "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line; file split tracked separately to avoid conflicts across the W-series"},
+    "dokimi/commands/novelty.py": {"lines": 879, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line, W1 routing cost output; file split tracked separately to avoid conflicts across the W-series"},
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"},

dokimi/tests/test_novelty_measure.py

@@ -221,6 +221,31 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced():
     assert by_role["judge"]["usd"] is None
 
 
+def test_cost_rolls_up_cache_tokens_and_replays_across_every_role():
+    def client(role, records):
+        rows = iter(records)
+        return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows)))
+
+    extractor = client("extractor", [
+        UsageRecord(prompt_tokens=10, cache_read_tokens=4, cache_write_tokens=1, cost_usd=0.1),
+        UsageRecord(prompt_tokens=10, cost_usd=0.1, replayed=True),
+    ])
+    judge = client("judge", [
+        UsageRecord(prompt_tokens=20, cache_read_tokens=8, cache_write_tokens=4, cost_usd=0.2),
+        UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True),
+    ])
+    for c in (extractor, judge):
+        c.text("p")
+        c.text("p")
+    total = roll_up_cost([extractor, judge])
+    # analyze rolls up several roles: a total taken from one of them would pass a
+    # single-client test, so each new field is summed over two.
+    assert total["llm_calls"] == 4
+    assert total["cache_read_tokens"] == 12
+    assert total["cache_write_tokens"] == 5
+    assert total["replayed_calls"] == 2
+
+
 def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer():
     def answer(prompt, system):
         if prompt.startswith("Reference material"):

dokimi/tests/test_novelty_routing.py

@@ -509,6 +509,110 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t
     assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}"
 
 
+def _stub_provider(monkeypatch, usage):
+    """Replace the provider call the --model path makes; every reply picks card 1."""
+    from dokimi import llm
+
+    def stub(prompt, model, **_kwargs):
+        return '{"pick": 1}', usage
+
+    monkeypatch.setattr(llm, "llm_call_with_usage", stub)
+
+
+def _model_run(tmp_path):
+    queries = tmp_path / "q.json"
+    queries.write_text(json.dumps(["profile a binary", "tune a slow query"]), encoding="utf-8")
+    out = tmp_path / "routing.json"
+    args = _parse("novelty", "routing",
+                  "--artifact", _skill(tmp_path, "a", "profile a binary with perf"),
+                  "--competitors", _skill(tmp_path, "b", "tune a database"),
+                  "--queries", str(queries), "--model", "selector-m", "--out", str(out))
+    args.func(args)
+    return json.loads(out.read_text(encoding="utf-8"))
+
+
+def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, capsys):
+    from dokimi.metrics.usage import UsageRecord
+
+    _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5,
+                                            cache_read_tokens=60, cache_write_tokens=30,
+                                            cost_usd=0.002))
+    payload = _model_run(tmp_path)
+    assert payload["cost"]["llm_calls"] == 2
+    assert payload["cost"]["input_tokens"] == 200
+    assert payload["cost"]["output_tokens"] == 10
+    assert payload["cost"]["cache_read_tokens"] == 120
+    assert payload["cost"]["cache_write_tokens"] == 60
+    assert payload["cost"]["usd"] == 0.004
+    assert ("cost     2 calls, 200 input / 10 output tokens, cache read 120 / write 60, "
+            "$0.0040") in (out := capsys.readouterr().out)
+    assert "replayed" not in out
+
+
+def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys):
+    from dokimi.metrics.usage import UsageRecord
+
+    _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1))
+    payload = _model_run(tmp_path)
+    assert payload["cost"]["usd"] is None   # one unpriced call must not be read as free
+    assert "USD not known: a call had no price or no usage" in capsys.readouterr().out
+
+
+def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path):
+    queries = tmp_path / "q.json"
+    queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8")
+    out = tmp_path / "routing.json"
+    args = _parse("novelty", "routing",
+                  "--artifact", _skill(tmp_path, "a", "profile a binary with perf"),
+                  "--competitors", _skill(tmp_path, "b", "tune a database"),
+                  "--queries", str(queries), "--out", str(out))
+    args.func(args)
+    assert "cost" not in json.loads(out.read_text(encoding="utf-8"))
+
+
+def test_a_model_run_whose_every_call_fails_prints_no_price_blame(tmp_path, monkeypatch, capsys):
+    from dokimi import llm
+
+    def failing(prompt, model, **_kwargs):
+        raise RuntimeError("provider refused")
+
+    monkeypatch.setattr(llm, "llm_call_with_usage", failing)
+    payload = _model_run(tmp_path)
+    assert payload["cost"]["llm_calls"] == 0
+    assert payload["cost"]["usd"] is None   # no calls: nothing known, not "free"
+    out = capsys.readouterr().out
+    assert "cost     0 calls\n" in out
+    assert "USD not known" not in out
+
+
+def test_a_call_with_no_usage_makes_the_total_unknown_not_the_priced_subset(
+        tmp_path, monkeypatch, capsys):
+    from dokimi import llm
+    from dokimi.metrics.usage import UsageRecord
+
+    replies = iter([None, UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001)])
+    monkeypatch.setattr(llm, "llm_call_with_usage",
+                        lambda prompt, model, **_kw: ('{"pick": 1}', next(replies)))
+    payload = _model_run(tmp_path)
+    assert payload["cost"]["llm_calls"] == 2
+    assert payload["cost"]["usd"] is None
+    assert "$0.0010" not in capsys.readouterr().out
+
+
+def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, capsys):
+    from dokimi.metrics.usage import UsageRecord
+
+    _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5,
+                                            cache_read_tokens=60, cache_write_tokens=30,
+                                            cost_usd=0.002, replayed=True))
+    payload = _model_run(tmp_path)
+    cost = payload["cost"]
+    assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2)
+    assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0)
+    assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0)
+    assert "2 replayed from the cache and not counted" in capsys.readouterr().out
+
+
 def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path):
     out = tmp_path / "routing.json"
     args = _parse("novelty", "routing",