diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md new file mode 100644 index 00000000..77ee46b5 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -0,0 +1,3 @@ +### Added + +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index f0ef48df..c11569ab 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -609,6 +609,14 @@ Two numbers, reported side by side and never merged: `selection_rate_among_answered` beside it; a failed call is excluded from both and counted, and one failed query does not lose the rest. +A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and +write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the +same totals on stdout. Cache tokens are reported separately from input tokens, and depending on +the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes +no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a +call that raised is not counted. A call replayed from the response cache is counted in +`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it. + `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` next to `n_queries_scored`, since a query with no content words is dropped rather than diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 6afb318f..5dd5d56d 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -336,17 +336,21 @@ def card(ref: str) -> tuple[Candidate, str]: "lexical": lexical, } if args.model: + from dokimi.novelty.client import roll_up_cost + selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( model_routing(candidates, queries, client=selector, gold=0, seed=args.seed), queries_self_derived=self_derived, ) + payload["cost"] = roll_up_cost([selector]) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") _print_routing_summary(payload, len(competitors), self_derived) + def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: """The lexical line, the model line and their caveats, printed after a routing run.""" # Every rate is printed with the number of queries behind it, and a self-derived one @@ -371,11 +375,22 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool f"[{row['model']}] n_queries {row['n_queries']} given{tag}") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") + cost = payload["cost"] + if not cost["llm_calls"]: + print("cost 0 calls") + else: + spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None + else "USD not known: a call had no price or no usage") + replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost["replayed_calls"] else "") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read " + f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " + f"{spent}{replayed}") if self_derived: print("Self-derived queries: read as triage only.") - def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, out: Path, n_candidates: int) -> NoReturn: """Write the ``no_queries`` record, then exit saying why no query was scored.""" diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index b8579cc8..4f53b46b 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -161,13 +161,22 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing, rather than - silently reporting the sum of the priced subset as the total. + ``usd`` stays ``None`` when any call this run paid for had unknown pricing or + returned no usage (a replayed call is not paid for, so it cannot make it unknown), + rather than silently reporting the sum of the priced subset as the total. + Calls served from the response cache are counted in ``replayed_calls`` and + left out of the token and dollar totals. """ - input_tokens = sum(u.prompt_tokens for u in self.usage) - output_tokens = sum(u.completion_tokens for u in self.usage) - costs = [u.cost_usd for u in self.usage] - priced = costs and None not in costs + # A replayed row is the response cache's copy of an earlier call: its tokens and + # price were not spent by this process, so it is counted apart, as the spend cap does. + billed = [u for u in self.usage if not u.replayed] + input_tokens = sum(u.prompt_tokens for u in billed) + output_tokens = sum(u.completion_tokens for u in billed) + cache_read = sum(u.cache_read_tokens for u in billed) + cache_write = sum(u.cache_write_tokens for u in billed) + costs = [u.cost_usd for u in billed] + # A call that returned no usage row has no price either, so it makes the total unknown. + priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None return { "model": self.model, @@ -176,6 +185,9 @@ def cost_summary(self) -> dict[str, Any]: "llm_calls": self.calls, "input_tokens": input_tokens, "output_tokens": output_tokens, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, + "replayed_calls": len(self.usage) - len(billed), "usd": usd, } @@ -230,6 +242,9 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "llm_calls": sum(row["llm_calls"] for row in per_role), "input_tokens": sum(row["input_tokens"] for row in per_role), "output_tokens": sum(row["output_tokens"] for row in per_role), + "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), + "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), + "replayed_calls": sum(row["replayed_calls"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index dc41beec..5c7df79d 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -447,9 +447,12 @@ "llm_calls": {"type": "integer", "minimum": 0}, "input_tokens": {"type": "integer", "minimum": 0}, "output_tokens": {"type": "integer", "minimum": 0}, + "cache_read_tokens": {"type": "integer", "minimum": 0}, + "cache_write_tokens": {"type": "integer", "minimum": 0}, + "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], - "description": "null when any call had unknown pricing — distinct from a genuinely free run." + "description": "null when any call this run paid for had unknown pricing — distinct from a genuinely free run." }, "by_role": {"type": "array", "items": {"type": "object"}} } diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index c0456236..e2f73ebd 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 864, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line; file split tracked separately to avoid conflicts across the W-series"}, + "dokimi/commands/novelty.py": {"lines": 879, "rationale": "W3 adds routing no_queries handling, W10 a --catalog help line, W1 routing cost output; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index a94373a8..7191db32 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -221,6 +221,31 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced(): assert by_role["judge"]["usd"] is None +def test_cost_rolls_up_cache_tokens_and_replays_across_every_role(): + def client(role, records): + rows = iter(records) + return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows))) + + extractor = client("extractor", [ + UsageRecord(prompt_tokens=10, cache_read_tokens=4, cache_write_tokens=1, cost_usd=0.1), + UsageRecord(prompt_tokens=10, cost_usd=0.1, replayed=True), + ]) + judge = client("judge", [ + UsageRecord(prompt_tokens=20, cache_read_tokens=8, cache_write_tokens=4, cost_usd=0.2), + UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True), + ]) + for c in (extractor, judge): + c.text("p") + c.text("p") + total = roll_up_cost([extractor, judge]) + # analyze rolls up several roles: a total taken from one of them would pass a + # single-client test, so each new field is summed over two. + assert total["llm_calls"] == 4 + assert total["cache_read_tokens"] == 12 + assert total["cache_write_tokens"] == 5 + assert total["replayed_calls"] == 2 + + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 18c6ac60..4c4702a2 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -509,6 +509,110 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" +def _stub_provider(monkeypatch, usage): + """Replace the provider call the --model path makes; every reply picks card 1.""" + from dokimi import llm + + def stub(prompt, model, **_kwargs): + return '{"pick": 1}', usage + + monkeypatch.setattr(llm, "llm_call_with_usage", stub) + + +def _model_run(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002)) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["input_tokens"] == 200 + assert payload["cost"]["output_tokens"] == 10 + assert payload["cost"]["cache_read_tokens"] == 120 + assert payload["cost"]["cache_write_tokens"] == 60 + assert payload["cost"]["usd"] == 0.004 + assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " + "$0.0040") in (out := capsys.readouterr().out) + assert "replayed" not in out + + +def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] is None # one unpriced call must not be read as free + assert "USD not known: a call had no price or no usage" in capsys.readouterr().out + + +def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(out)) + args.func(args) + assert "cost" not in json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_whose_every_call_fails_prints_no_price_blame(tmp_path, monkeypatch, capsys): + from dokimi import llm + + def failing(prompt, model, **_kwargs): + raise RuntimeError("provider refused") + + monkeypatch.setattr(llm, "llm_call_with_usage", failing) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 0 + assert payload["cost"]["usd"] is None # no calls: nothing known, not "free" + out = capsys.readouterr().out + assert "cost 0 calls\n" in out + assert "USD not known" not in out + + +def test_a_call_with_no_usage_makes_the_total_unknown_not_the_priced_subset( + tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + replies = iter([None, UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001)]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["usd"] is None + assert "$0.0010" not in capsys.readouterr().out + + +def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002, replayed=True)) + payload = _model_run(tmp_path) + cost = payload["cost"] + assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) + assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0) + assert "2 replayed from the cache and not counted" in capsys.readouterr().out + + def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path): out = tmp_path / "routing.json" args = _parse("novelty", "routing",