From 133b87b1c18209286b13d8b47d73c7a8c20c6f33 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:03:18 -0700 Subject: [PATCH 01/14] feat(novelty): project the calls and dollars --estimate-only will spend The estimate counted one atom per two spans, and left out the verify, shorten, carrier and classify calls and the retention oracle's escalations. Against the five standard runs in the skill scan it came in 37-69% under. - The atom count is the larger of spans/2 and tokens/25, capped as before. - expected_calls counts the stages, every probe, and half the oracle pairs, split by role in by_role. total_calls and the every-pair-escalates total are unchanged. - expected_usd prices those calls at litellm list prices with a token profile fitted on the five runs; null when a role's model has no price. dokimi.llm.list_cost_usd is the price lookup the estimate uses. On the six measured runs the calls land within 20% of the counted total, the standard-run dollars within 25%. Triage dollars run low: its extraction writes about 1,700 tokens per call, which the profile does not model. The parity golden keeps main's output; the estimate cases compare the parts a tier or flag decides, and the bill is the intended change. Signed-off-by: Nikolay Petrov --- ...1_120000_napetrov_estimate-expected-usd.md | 10 ++ dokimi/docs/skill-novelty.md | 6 +- dokimi/dokimi/commands/novelty.py | 43 +++++++-- dokimi/dokimi/llm.py | 21 +++++ dokimi/dokimi/novelty/tiers.py | 91 +++++++++++++++++-- dokimi/tests/test_novelty_settings_parity.py | 27 +++++- dokimi/tests/test_novelty_tiers.py | 89 ++++++++++++++++++ 7 files changed, 269 insertions(+), 18 deletions(-) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md diff --git a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md new file mode 100644 index 00000000..a71a09b2 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md @@ -0,0 +1,10 @@ +### Changed + +- **`novelty analyze --estimate-only` projects the bill it will actually run.** The atom + count is the larger of one per two spans and one per 25 artifact tokens, capped as + before; the new `expected_calls` counts the verify, shorten, carrier and classify calls + and half the retention-oracle pairs that escalate to a model, and `by_role` splits it + into extractor, answerer and judge. `expected_usd` prices those calls at list prices, + and is `null` when a model has no price. Against five measured standard runs the calls + landed within 20% of the counted total on all six, where the old span-only figure was + 37-69% under. `total_calls` and `total_calls_if_every_pair_escalates` are unchanged. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bb..5758a0ef 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -784,7 +784,11 @@ python cli.py novelty analyze --artifact skill:s --tier study --estimate-only ``` `--estimate-only` prints the projected bill and exits without constructing a client, so -the decision costs nothing. An explicit flag always beats the tier, and the override is +the decision costs nothing. Its payload has `expected_calls`, the point figure between +the two totals (stages, every probe, and half the oracle pairs, split by `by_role`), and +`expected_usd`: those calls priced at list prices by role, `null` when a model has no +price. Against five measured standard runs the calls landed within 25% and the dollars +within 25%; triage dollars run low, because its extraction output is not modelled. An explicit flag always beats the tier, and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get to be read as one. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc943..3e72a432 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -62,16 +62,20 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: print(f" overridden by flags: {sorted(record['overrides'])}", file=sys.stderr) if args.estimate_only: - estimate = settings.estimate(len(segment(artifact_bundle.text)), + spans = segment(artifact_bundle.text) + n_tokens = sum(span.tokens for span in spans) + estimate = settings.estimate(len(spans), n_tokens=n_tokens, budgets=tuple(args.budgets), n_arms=len(_arms(args.arms))) + estimate["expected_usd"] = _estimate_usd(args, n_tokens, estimate["estimate"]["by_role"]) print(json.dumps(estimate, indent=2)) print("Estimate only, nothing was measured. ~4 calls per paired probe is the " "measured shape; batch sizes and retries move the real figure.", file=sys.stderr) if args.max_usd is None: - print("The estimate counts calls, not dollars: what a call costs depends on the " - "artifact's length and the models' prices. Pass --max-usd to stop the run " - "at a dollar figure.", file=sys.stderr) + print("expected_usd prices the calls by role at list prices, with a token profile " + "measured on five runs: a longer artifact or longer answers cost more, and " + "a model with no price gives null. Pass --max-usd to stop the run at a " + "dollar figure.", file=sys.stderr) return extractor, answerer, judge = _clients(args) @@ -714,8 +718,8 @@ def _add_model_args(parser: argparse.ArgumentParser, positive_int) -> None: _add_spend_cap(parser) -def _clients(args: argparse.Namespace): - """Build the three role clients, defaulting so roles never silently collapse.""" +def _role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: + """The (model, provider) of each role, defaulting so roles never silently collapse.""" from dokimi.novelty.client import default_judge if args.judge_model: @@ -727,10 +731,31 @@ def _clients(args: argparse.Namespace): raise SystemExit(str(exc)) from None extract_model = args.extract_model or judge_model extract_provider = args.extract_provider or judge_provider + return { + "extractor": (extract_model, extract_provider), + "answerer": (args.model, args.provider), + "judge": (judge_model, judge_provider), + } + + +def _estimate_usd(args: argparse.Namespace, n_tokens: int, by_role: dict[str, int]) -> float | None: + """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced.""" + from dokimi.novelty.tiers import estimate_usd + + try: + models = _role_models(args) + except SystemExit: + return None + return estimate_usd(by_role, n_tokens, models) + + +def _clients(args: argparse.Namespace): + """Build the three role clients, defaulting so roles never silently collapse.""" + models = _role_models(args) return ( - _client(extract_model, extract_provider, "extractor", args), - _client(args.model, args.provider, "answerer", args), - _client(judge_model, judge_provider, "judge", args), + _client(*models["extractor"], "extractor", args), + _client(*models["answerer"], "answerer", args), + _client(*models["judge"], "judge", args), ) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 2de6e523..92e23e00 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -335,6 +335,27 @@ def _safe_completion_cost(resp: Any, model: str, litellm_model: Optional[str] = return None +def list_cost_usd(model: str, provider: str, prompt_tokens: int, + completion_tokens: int) -> Optional[float]: + """Dollar cost of one call of these token counts at list price, or ``None``. + + The price map :func:`_safe_completion_cost` reads, for a call that has not been + made: an estimate prices a token profile rather than a response. ``None`` when the + model has no entry, for the same reason as there. + """ + from litellm import cost_per_token + + litellm_model = resolve_model(model, provider) + try: + prompt, completion = cost_per_token(model=litellm_model, prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens) + except Exception: + return None + if prompt + completion == 0.0 and not _has_price_entry(litellm_model, model): + return None + return float(prompt + completion) + + def _extract_usage( resp: Any, model: str, diff --git a/dokimi/dokimi/novelty/tiers.py b/dokimi/dokimi/novelty/tiers.py index 0210a333..0690d25f 100644 --- a/dokimi/dokimi/novelty/tiers.py +++ b/dokimi/dokimi/novelty/tiers.py @@ -208,20 +208,25 @@ def warnings(self, n_atoms: int | None = None) -> list[str]: ) return out - def estimate(self, n_spans: int, *, budgets: tuple[float, ...] = (), - n_arms: int = 2) -> dict: + def estimate(self, n_spans: int, *, n_tokens: int | None = None, + budgets: tuple[float, ...] = (), n_arms: int = 2) -> dict: """The ``--estimate-only`` bill for an artifact of ``n_spans`` spans. - One atom per two spans is the density the study's extractor produced, capped at - ``max_atoms``. The forfeits assume the cap is reached, since the atom count is a - guess until extraction runs: an artifact with fewer atoms than the cap drops the + The atom count is the larger of one atom per two spans and one per + :data:`TOKENS_PER_ATOM` tokens (spans alone undercount: short spans, such as table + rows, yield an atom each), capped at ``max_atoms``. Without ``n_tokens`` only the + span rule applies. The forfeits assume the cap is reached, since the atom count is + a guess until extraction runs: an artifact with fewer atoms than the cap drops the sampled-rate forfeits. """ cap = self.values["max_atoms"] expected = max(1, n_spans // 2) + if n_tokens is not None: + expected = max(expected, n_tokens // TOKENS_PER_ATOM) n_atoms = expected if cap is None else min(cap, expected) return { "spans": n_spans, + "tokens": n_tokens, "atom_cap": cap, # The full resolved settings, so the projection is the model `estimate_calls` # implements rather than a subset of it. @@ -273,6 +278,61 @@ def resolve_settings(tier: str | None, flags: dict) -> RunSettings: return RunSettings(values=values, tier=tier, overrides=overrides) +#: Atoms per artifact token for the atom count when the artifact's size is known. Measured +#: on five standard runs (atoms per token: 1/17 to 1/40, median 1/25); the span rule alone +#: gave 7 atoms for a 198-token skill that yielded 8, and 11 for one that yielded 18. +TOKENS_PER_ATOM = 25 + +#: The share of oracle pairs (carrier confirmation and shortening candidates) that the +#: oracle escalated to a model. Measured on five standard runs: 0.32-0.70, median 0.51. +ESCALATION_SHARE = 0.5 + +#: Per call, by role: (input tokens besides the artifact, input tokens per artifact token, +#: output tokens). Measured on the same five runs: the extractor and the answerer send the +#: artifact with every call, the judge sends the answer and the question only. +CALL_TOKENS: dict[str, tuple[int, int, int]] = { + "extractor": (130, 1, 190), + "answerer": (20, 1, 240), + "judge": (460, 0, 85), +} + + +def call_tokens(role: str, n_tokens: int) -> tuple[int, int]: + """Input and output tokens of one call of ``role`` on an artifact of ``n_tokens``. + + The extractor's output is a flat figure: on a triage run its extraction calls write + about 1,700 tokens each, which this does not model, so the dollar figure runs low there. + """ + base, per_token, out = CALL_TOKENS[role] + return base + per_token * n_tokens, out + + +def estimate_usd(by_role: dict[str, int], n_tokens: int, + models: dict[str, tuple[str, str]]) -> float | None: + """The expected dollar cost of ``by_role`` calls at list prices. + + Prices come from litellm's price map (:func:`dokimi.llm.list_cost_usd`), the same map a + run's own ``cost_usd`` uses, so the figure is the run's price applied to a token profile. + ``None`` when any role's model has no price: an unpriced call is unknown, not free. + + Args: + by_role: Calls per role, as :func:`estimate_calls` reports them. + n_tokens: Tokens in the artifact. + models: ``(model, provider)`` per role. + """ + from dokimi.llm import list_cost_usd + + total = 0.0 + for role, calls in by_role.items(): + model, provider = models[role] + tokens_in, tokens_out = call_tokens(role, n_tokens) + cost = list_cost_usd(model, provider, tokens_in, tokens_out) + if cost is None: + return None + total += calls * cost + return round(total, 6) + + def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: """Project the model-call bill before spending it. @@ -283,6 +343,10 @@ def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: counting client over three real skills at all three tiers: the projection came within 1-3% of the counted total at every point (see the PR that introduced tiers). + The oracle's pairs are the other term. Five measured standard runs escalated 36-73% of + them (median 51%) and the extractor's count equals the stage calls plus those + escalations, so ``expected_calls`` counts :data:`ESCALATION_SHARE` of the oracle pairs. + Args: n_atoms: Atoms expected after the cap. settings: Resolved tier settings (``paraphrases``, ``reversals``, stage switches). @@ -290,10 +354,13 @@ def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: Returns: ``{"n_atoms", "paired_probes", "reversal_probes", "probe_calls", "stage_calls", - "oracle_pairs", "total_calls", "total_calls_if_every_pair_escalates"}``. + "oracle_pairs", "total_calls", "total_calls_if_every_pair_escalates", + "by_role", "expected_calls"}``. ``total_calls`` assumes the retention oracle resolves every pair locally, which is what the validated 1-3% figure was measured on; the second total is the other end - of the range. The real figure also moves with batch sizes and retries. + of the range. ``expected_calls`` is the point figure between them, with ``by_role`` + splitting it into the extractor (stages and oracle escalations), answerer and judge. + The real figure also moves with batch sizes and retries. """ paired = n_atoms * (1 + (1 if settings.get("paraphrases") else 0)) reversals = n_atoms if settings.get("reversals") else 0 @@ -318,6 +385,14 @@ def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: (n_atoms if settings.get("confirm_carriers") else 0) + (n_atoms * budgets * strategies if settings.get("shorten_artifact") else 0) ) + escalated = round(ESCALATION_SHARE * oracle_pairs) + # Answer and grade are each one call per probe and arm; the extractor takes the stages + # and the escalations, which is how the measured runs split their calls. + by_role = { + "extractor": stage_calls + escalated, + "answerer": (paired + reversals) * n_arms, + "judge": (paired + reversals) * n_arms, + } return { "n_atoms": n_atoms, "paired_probes": paired, @@ -327,4 +402,6 @@ def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: "oracle_pairs": oracle_pairs, "total_calls": probe_calls + stage_calls, "total_calls_if_every_pair_escalates": probe_calls + stage_calls + oracle_pairs, + "by_role": by_role, + "expected_calls": sum(by_role.values()), } diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index d42e6c36..42742462 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -14,6 +14,10 @@ JSON copied in:: python -m tests.test_novelty_settings_parity --write + +The one intended change to an estimate is its bill (``estimate``, ``expected_usd``, the atom +count it projects) and the stderr note that explains it; ``_kept_estimate`` compares +everything else an estimate prints, which is the part a tier or a flag decides. """ from __future__ import annotations @@ -154,10 +158,31 @@ def scripted(model, provider, role, args): return steps +# The stderr note the estimate prints, which W2 rewrote; everything before it is main's. +_ESTIMATE_NOTES = ("The estimate counts calls", "expected_usd prices") + + +def _kept_estimate(step: dict) -> dict: + """An estimate step without the bill and the note W2 changed; a refusal is kept whole.""" + if not step["stdout"]: + return step + payload = json.loads(step["stdout"]) + stderr = step["stderr"] + for note in _ESTIMATE_NOTES: + stderr = stderr.split(note)[0] + return {"status": step["status"], "calls": step["calls"], "stderr": stderr, + **{key: payload[key] for key in ("spans", "atom_cap", "tier", "caveat")}} + + @pytest.mark.parametrize("name", list(CASES)) def test_no_setting_bill_or_tier_record_differs_from_main(name, tmp_path): golden = json.loads(GOLDEN.read_text(encoding="utf-8")) - assert _capture(name, tmp_path) == golden[name] + captured = _capture(name, tmp_path) + if name.startswith("estimate/"): + assert [_kept_estimate(step) for step in captured] == [ + _kept_estimate(step) for step in golden[name]] + else: + assert captured == golden[name] def test_the_golden_cases_are_the_ones_this_module_runs(): diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index fa8b86bb..c96b7661 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -351,3 +351,92 @@ def test_a_tier_on_the_config_and_the_older_tier_argument_record_the_same(tmp_pa assert on_config["warnings"] == as_argument["warnings"] assert any("tier `triage` was overridden by flags (max_atoms)" in w for w in on_config["warnings"]) + + +#: Six measured runs of the 2026-10-21 skill scan (``out/*/L2`` and ``out/*/L3``): the +#: artifact's spans and tokens, its tier, the atoms its extraction found, and what it counted +#: by role and dollars. Numbers only; no third-party skill text. +MEASURED_RUNS = [ + # name, spans, tokens, tier, atoms, calls by role, usd + ("brand-guidelines", 44, 458, "standard", 27, + {"extractor": 156, "answerer": 104, "judge": 104}, 0.666236), + ("caveman-review", 42, 567, "standard", 23, + {"extractor": 172, "answerer": 90, "judge": 90}, 0.820205), + ("slides", 26, 355, "standard", 13, + {"extractor": 86, "answerer": 52, "judge": 52}, 0.375282), + ("ponytail-debt", 23, 371, "standard", 18, + {"extractor": 96, "answerer": 70, "judge": 70}, 0.470331), + ("rea-tool-design", 7, 198, "standard", 8, + {"extractor": 34, "answerer": 32, "judge": 32}, 0.164377), + ("tdd", 27, 846, "triage", 21, + {"extractor": 3, "answerer": 42, "judge": 42}, 0.163192), +] +_MODELS = {"extractor": ("us.anthropic.claude-sonnet-5-5", "amazon-bedrock"), + "answerer": ("us.anthropic.claude-haiku-5-5", "amazon-bedrock"), + "judge": ("us.anthropic.claude-sonnet-5-5", "amazon-bedrock")} + + +def _measured_estimate(tier, spans, tokens): + return resolve_settings(tier, {}).estimate( + spans, n_tokens=tokens, budgets=(0.75, 0.5), n_arms=2)["estimate"] + + +def test_the_expected_calls_land_within_a_quarter_of_each_measured_run(): + # The old projection (one atom per two spans, no oracle escalations) was 37-69% under all + # six of these runs; the token rule and the escalation share bring every one within 20%. + for name, spans, tokens, tier, atoms, by_role, _ in MEASURED_RUNS: + estimate = _measured_estimate(tier, spans, tokens) + counted = sum(by_role.values()) + assert abs(estimate["expected_calls"] - counted) <= 0.25 * counted, name + assert abs(estimate["n_atoms"] - atoms) <= 0.25 * atoms, name + + +def test_the_expected_calls_sit_between_the_no_escalation_and_the_full_escalation_bounds(): + for name, spans, tokens, tier, _, _, _ in MEASURED_RUNS: + estimate = _measured_estimate(tier, spans, tokens) + assert (estimate["total_calls"] <= estimate["expected_calls"] + <= estimate["total_calls_if_every_pair_escalates"]), name + assert estimate["expected_calls"] == sum(estimate["by_role"].values()), name + + +def test_the_atom_count_takes_the_token_rule_when_the_artifact_has_more_tokens_than_spans(): + everything = resolve_settings(None, {"all_atoms": True}) + assert everything.estimate(7)["estimate"]["n_atoms"] == 3 # spans only + assert everything.estimate(7, n_tokens=198)["estimate"]["n_atoms"] == 7 + # Short spans with many tokens: the span rule was the smaller of the two. + assert everything.estimate(20, n_tokens=900)["estimate"]["n_atoms"] == 36 + + +def test_the_dollar_profile_reproduces_the_measured_standard_runs(): + # The profile priced at each run's own call counts lands within a quarter of what the + # run's `cost.usd` recorded. Triage is left out: its extraction calls write whole atom + # lists, which the flat profile does not model (see `call_tokens`). + from dokimi.novelty.tiers import estimate_usd + + for name, _, tokens, tier, _, by_role, usd in MEASURED_RUNS: + if tier != "standard": + continue + priced = estimate_usd(by_role, tokens, _MODELS) + assert priced is not None, name + assert abs(priced - usd) <= 0.25 * usd, name + + +def test_an_unpriced_model_gives_no_dollar_figure_rather_than_zero(): + from dokimi.novelty.tiers import estimate_usd + + unpriced = {"answerer": ("no-such-model-for-dokimi", "openai")} + assert estimate_usd({"answerer": 10}, 100, unpriced) is None + + +def test_estimate_only_prints_the_expected_calls_and_a_dollar_figure(tmp_path, capsys): + root = tmp_path / "s" + root.mkdir() + (root / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "standard", + "--estimate-only") + args.func(args) + payload = json.loads(capsys.readouterr().out) + assert payload["estimate"]["expected_calls"] == sum(payload["estimate"]["by_role"].values()) + assert payload["expected_usd"] > 0 # default models are priced; null would be a regression + assert payload["tokens"] > 0 From bab6874840c456a6f145e129d4580dcae1633db2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:17:24 -0700 Subject: [PATCH 02/14] fix(novelty): hold out half the measured runs for the estimate constants - Constants fitted on 13 of the 25 measured runs; the other 12 are MEASURED_RUNS. Held-out worst error: calls 13%, atoms 11%, dollars 33%, all within TOLERANCE (35%). One tolerance figure now in tests, comments, docs and changelog. - Extractor output per call priced per tier (triage 2,359 tokens, standard 195), with a bounded triage-USD test. - Parity test compares the whole estimate payload, dropping only the bill keys. - _estimate_usd no longer swallows SystemExit: it prints to stderr why USD is unavailable. - Drop the task id from comments; re-wrap the long paragraph in docs/skill-novelty.md. Signed-off-by: Nikolay Petrov --- ...1_120000_napetrov_estimate-expected-usd.md | 10 +- dokimi/docs/skill-novelty.md | 8 +- dokimi/dokimi/commands/novelty.py | 48 +++++-- dokimi/dokimi/novelty/tiers.py | 69 ++++++---- dokimi/tests/test_novelty_settings_parity.py | 32 ++++- dokimi/tests/test_novelty_tiers.py | 121 +++++++++++------- 6 files changed, 196 insertions(+), 92 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md index a71a09b2..b82827e4 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md @@ -1,10 +1,12 @@ ### Changed - **`novelty analyze --estimate-only` projects the bill it will actually run.** The atom - count is the larger of one per two spans and one per 25 artifact tokens, capped as + count is the larger of one per two spans and one per 23 artifact tokens, capped as before; the new `expected_calls` counts the verify, shorten, carrier and classify calls and half the retention-oracle pairs that escalate to a model, and `by_role` splits it into extractor, answerer and judge. `expected_usd` prices those calls at list prices, - and is `null` when a model has no price. Against five measured standard runs the calls - landed within 20% of the counted total on all six, where the old span-only figure was - 37-69% under. `total_calls` and `total_calls_if_every_pair_escalates` are unchanged. + with the extractor's output per call set per tier, and is `null` when a model has no + price. The constants were fitted on 13 of 25 measured runs; on the other 12 (held out) + calls landed within 13%, atoms within 11% and dollars within 33%, all within the 35% + tolerance, and dollars came out high, never low. `total_calls` and + `total_calls_if_every_pair_escalates` are unchanged. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5758a0ef..5d1a681e 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -787,9 +787,11 @@ python cli.py novelty analyze --artifact skill:s --tier study --estimate-only the decision costs nothing. Its payload has `expected_calls`, the point figure between the two totals (stages, every probe, and half the oracle pairs, split by `by_role`), and `expected_usd`: those calls priced at list prices by role, `null` when a model has no -price. Against five measured standard runs the calls landed within 25% and the dollars -within 25%; triage dollars run low, because its extraction output is not modelled. An explicit flag always beats the tier, and the override is -recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get +price. The constants were fitted on 13 of the 25 measured runs; the other 12 are held out. +On those 12 every projection landed within 35%: calls within 13%, atoms within 11%, dollars +within 33%. Every dollar projection came out high, not low. Only three of the 12 are standard +runs, so the standard profile rests on little data. An explicit flag always beats the tier, +and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get to be read as one. Each tier carries the **claims it forfeits**, and those strings are written into diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 3e72a432..fffb125e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -64,18 +64,24 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: if args.estimate_only: spans = segment(artifact_bundle.text) n_tokens = sum(span.tokens for span in spans) - estimate = settings.estimate(len(spans), n_tokens=n_tokens, - budgets=tuple(args.budgets), n_arms=len(_arms(args.arms))) - estimate["expected_usd"] = _estimate_usd(args, n_tokens, estimate["estimate"]["by_role"]) + estimate = settings.estimate( + len(spans), n_tokens=n_tokens, budgets=tuple(args.budgets), n_arms=len(_arms(args.arms)) + ) + estimate["expected_usd"] = _estimate_usd( + args, n_tokens, estimate["estimate"]["by_role"], settings.tier + ) print(json.dumps(estimate, indent=2)) print("Estimate only, nothing was measured. ~4 calls per paired probe is the " "measured shape; batch sizes and retries move the real figure.", file=sys.stderr) if args.max_usd is None: - print("expected_usd prices the calls by role at list prices, with a token profile " - "measured on five runs: a longer artifact or longer answers cost more, and " - "a model with no price gives null. Pass --max-usd to stop the run at a " - "dollar figure.", file=sys.stderr) + print( + "expected_usd prices the calls by role at list prices, from a token profile " + "fitted on measured runs: a longer artifact or longer answers cost more, and " + "a model with no price gives null. Pass --max-usd to stop the run at a " + "dollar figure.", + file=sys.stderr, + ) return extractor, answerer, judge = _clients(args) @@ -738,15 +744,35 @@ def _role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: } -def _estimate_usd(args: argparse.Namespace, n_tokens: int, by_role: dict[str, int]) -> float | None: - """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced.""" +def _estimate_usd( + args: argparse.Namespace, n_tokens: int, by_role: dict[str, int], tier: str | None +) -> float | None: + """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced. + + The reason goes to stderr: a role whose model does not resolve, or a role with no list + price, makes the figure unavailable, and the note names it. + """ + from dokimi.llm import list_cost_usd from dokimi.novelty.tiers import estimate_usd try: models = _role_models(args) - except SystemExit: + except SystemExit as exc: + print(f"The USD estimate is unavailable: {exc}", file=sys.stderr) return None - return estimate_usd(by_role, n_tokens, models) + usd = estimate_usd(by_role, n_tokens, models, tier) + if usd is None: + unpriced = sorted( + f"{role} ({model})" + for role, (model, provider) in models.items() + if list_cost_usd(model, provider, 1, 1) is None + ) + print( + "The USD estimate is unavailable: no list price in the litellm price map for " + + ", ".join(unpriced), + file=sys.stderr, + ) + return usd def _clients(args: argparse.Namespace): diff --git a/dokimi/dokimi/novelty/tiers.py b/dokimi/dokimi/novelty/tiers.py index 0690d25f..77cd2a95 100644 --- a/dokimi/dokimi/novelty/tiers.py +++ b/dokimi/dokimi/novelty/tiers.py @@ -278,37 +278,57 @@ def resolve_settings(tier: str | None, flags: dict) -> RunSettings: return RunSettings(values=values, tier=tier, overrides=overrides) -#: Atoms per artifact token for the atom count when the artifact's size is known. Measured -#: on five standard runs (atoms per token: 1/17 to 1/40, median 1/25); the span rule alone -#: gave 7 atoms for a 198-token skill that yielded 8, and 11 for one that yielded 18. -TOKENS_PER_ATOM = 25 +#: Atoms per artifact token for the atom count when the artifact's size is known. Fitted on +#: the fit half of the 25 measured skill-scan runs (the uncapped runs of that half: 9 to 40 +#: tokens per atom, median 22.8); the span rule alone undercounts short-span artifacts. +TOKENS_PER_ATOM = 23 #: The share of oracle pairs (carrier confirmation and shortening candidates) that the -#: oracle escalated to a model. Measured on five standard runs: 0.32-0.70, median 0.51. +#: oracle escalated to a model. Fitted on the fit half's standard runs: 0.11 (a one-atom +#: artifact) to 0.60, median 0.505. ESCALATION_SHARE = 0.5 -#: Per call, by role: (input tokens besides the artifact, input tokens per artifact token, -#: output tokens). Measured on the same five runs: the extractor and the answerer send the -#: artifact with every call, the judge sends the answer and the question only. -CALL_TOKENS: dict[str, tuple[int, int, int]] = { - "extractor": (130, 1, 190), - "answerer": (20, 1, 240), - "judge": (460, 0, 85), +#: Per call, for the roles whose calls do not depend on the tier: (input tokens besides the +#: artifact, input tokens per artifact token, output tokens). The answerer's input carries the +#: artifact on the paired arm; the judge sees the answer and the question only. Pooled over the +#: fit half. +CALL_TOKENS: dict[str, tuple[float, float, int]] = { + "answerer": (-38, 0.789, 284), + "judge": (468, 0.004, 89), +} + +#: The extractor's calls, by tier, in the same shape. A triage extraction packs the capped +#: atom set into each of its calls and writes about 2,360 output tokens per call, where a +#: standard extraction writes about 195; the input follows the same split. Study and an +#: untiered run have no measurement of their own and take the standard profile. +EXTRACTOR_TOKENS: dict[str, tuple[float, float, int]] = { + "triage": (971, 0.045, 2359), + "standard": (267, 0.651, 195), } -def call_tokens(role: str, n_tokens: int) -> tuple[int, int]: +def call_tokens(role: str, n_tokens: int, tier: str | None = None) -> tuple[int, int]: """Input and output tokens of one call of ``role`` on an artifact of ``n_tokens``. - The extractor's output is a flat figure: on a triage run its extraction calls write - about 1,700 tokens each, which this does not model, so the dollar figure runs low there. + Args: + role: ``extractor``, ``answerer`` or ``judge``. + n_tokens: Tokens in the artifact. + tier: The run's tier, which picks the extractor's profile (see + :data:`EXTRACTOR_TOKENS`). """ - base, per_token, out = CALL_TOKENS[role] - return base + per_token * n_tokens, out - - -def estimate_usd(by_role: dict[str, int], n_tokens: int, - models: dict[str, tuple[str, str]]) -> float | None: + if role == "extractor": + base, per_token, out = EXTRACTOR_TOKENS.get(tier or "", EXTRACTOR_TOKENS["standard"]) + else: + base, per_token, out = CALL_TOKENS[role] + return max(0, round(base + per_token * n_tokens)), out + + +def estimate_usd( + by_role: dict[str, int], + n_tokens: int, + models: dict[str, tuple[str, str]], + tier: str | None = None, +) -> float | None: """The expected dollar cost of ``by_role`` calls at list prices. Prices come from litellm's price map (:func:`dokimi.llm.list_cost_usd`), the same map a @@ -319,13 +339,14 @@ def estimate_usd(by_role: dict[str, int], n_tokens: int, by_role: Calls per role, as :func:`estimate_calls` reports them. n_tokens: Tokens in the artifact. models: ``(model, provider)`` per role. + tier: The run's tier; it sets the extractor's output (see :func:`call_tokens`). """ from dokimi.llm import list_cost_usd total = 0.0 for role, calls in by_role.items(): model, provider = models[role] - tokens_in, tokens_out = call_tokens(role, n_tokens) + tokens_in, tokens_out = call_tokens(role, n_tokens, tier) cost = list_cost_usd(model, provider, tokens_in, tokens_out) if cost is None: return None @@ -343,8 +364,8 @@ def estimate_calls(n_atoms: int, settings: dict, *, n_arms: int = 2) -> dict: counting client over three real skills at all three tiers: the projection came within 1-3% of the counted total at every point (see the PR that introduced tiers). - The oracle's pairs are the other term. Five measured standard runs escalated 36-73% of - them (median 51%) and the extractor's count equals the stage calls plus those + The oracle's pairs are the other term. The fit half's standard runs escalated 0.11 to + 0.60 of them (median 0.505), and the extractor's count equals the stage calls plus those escalations, so ``expected_calls`` counts :data:`ESCALATION_SHARE` of the oracle pairs. Args: diff --git a/dokimi/tests/test_novelty_settings_parity.py b/dokimi/tests/test_novelty_settings_parity.py index 42742462..cd59c799 100644 --- a/dokimi/tests/test_novelty_settings_parity.py +++ b/dokimi/tests/test_novelty_settings_parity.py @@ -15,9 +15,10 @@ python -m tests.test_novelty_settings_parity --write -The one intended change to an estimate is its bill (``estimate``, ``expected_usd``, the atom -count it projects) and the stderr note that explains it; ``_kept_estimate`` compares -everything else an estimate prints, which is the part a tier or a flag decides. +The one intended change to an estimate is its bill: ``tokens``, ``expected_usd``, ``by_role``, +``expected_calls``, and the atom count and totals that the projected atoms move. ``_kept_estimate`` +drops exactly those and compares the rest of the payload, the settings echo on stderr and the +tier record with its overrides included. """ from __future__ import annotations @@ -158,20 +159,37 @@ def scripted(model, provider, role, args): return steps -# The stderr note the estimate prints, which W2 rewrote; everything before it is main's. +# The stderr note the estimate prints: the one whose wording changed is cut from both sides. _ESTIMATE_NOTES = ("The estimate counts calls", "expected_usd prices") +# The bill the estimate now projects, and the atom count and totals that move with it. +_BILL_KEYS = ("tokens", "expected_usd") +_ESTIMATE_BILL_KEYS = ( + "by_role", + "expected_calls", + "n_atoms", + "total_calls", + "total_calls_if_every_pair_escalates", +) def _kept_estimate(step: dict) -> dict: - """An estimate step without the bill and the note W2 changed; a refusal is kept whole.""" + """An estimate step with only the bill removed; the rest of the payload is compared whole. + + The settings echo on stderr, the tier record with its overrides, the forfeits and the + caveat all stay. A refusal has no payload and is kept whole. + """ if not step["stdout"]: return step payload = json.loads(step["stdout"]) + for key in _BILL_KEYS: + payload.pop(key, None) + payload["estimate"] = { + key: value for key, value in payload["estimate"].items() if key not in _ESTIMATE_BILL_KEYS + } stderr = step["stderr"] for note in _ESTIMATE_NOTES: stderr = stderr.split(note)[0] - return {"status": step["status"], "calls": step["calls"], "stderr": stderr, - **{key: payload[key] for key in ("spans", "atom_cap", "tier", "caveat")}} + return {**step, "stdout": payload, "stderr": stderr} @pytest.mark.parametrize("name", list(CASES)) diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index c96b7661..4763b325 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -12,13 +12,19 @@ from dokimi.cli import build_parser from dokimi.novelty.tiers import ( + CALL_TOKENS, DEFAULT_SETTINGS, + ESCALATION_SHARE, + EXTRACTOR_TOKENS, FLAG_DEFAULTS, FORFEITS, SETTING_NAMES, TIERS, + TOKENS_PER_ATOM, RunSettings, + call_tokens, estimate_calls, + estimate_usd, resolve_settings, ) @@ -353,47 +359,85 @@ def test_a_tier_on_the_config_and_the_older_tier_argument_record_the_same(tmp_pa for w in on_config["warnings"]) -#: Six measured runs of the 2026-10-21 skill scan (``out/*/L2`` and ``out/*/L3``): the -#: artifact's spans and tokens, its tier, the atoms its extraction found, and what it counted -#: by role and dollars. Numbers only; no third-party skill text. +#: The 25 measured runs of the 2026-10-21 skill scan, split by artifact into a fit half (13 +#: runs), which the constants in tiers.py were fitted on, and the held-out half below (12 runs). +#: Each row: the artifact's spans and tokens, its tier, the atoms its extraction found, and the +#: calls by role and the dollars the run recorded. Numbers only: no skill text, no names. MEASURED_RUNS = [ - # name, spans, tokens, tier, atoms, calls by role, usd - ("brand-guidelines", 44, 458, "standard", 27, - {"extractor": 156, "answerer": 104, "judge": 104}, 0.666236), - ("caveman-review", 42, 567, "standard", 23, - {"extractor": 172, "answerer": 90, "judge": 90}, 0.820205), - ("slides", 26, 355, "standard", 13, - {"extractor": 86, "answerer": 52, "judge": 52}, 0.375282), - ("ponytail-debt", 23, 371, "standard", 18, - {"extractor": 96, "answerer": 70, "judge": 70}, 0.470331), - ("rea-tool-design", 7, 198, "standard", 8, - {"extractor": 34, "answerer": 32, "judge": 32}, 0.164377), - ("tdd", 27, 846, "triage", 21, - {"extractor": 3, "answerer": 42, "judge": 42}, 0.163192), + ("held-01", 7, 198, "standard", 8, {"extractor": 34, "answerer": 32, "judge": 32}, 0.164377), + ("held-02", 23, 371, "standard", 18, {"extractor": 96, "answerer": 70, "judge": 70}, 0.470331), + ("held-03", 42, 567, "standard", 23, {"extractor": 172, "answerer": 90, "judge": 90}, 0.820205), + ("held-04", 42, 567, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.180702), + ("held-05", 38, 745, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.183736), + ("held-06", 61, 883, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.158418), + ("held-07", 53, 1153, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.206459), + ("held-08", 66, 1768, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.225759), + ("held-09", 125, 2156, "triage", 25, {"extractor": 3, "answerer": 50, "judge": 50}, 0.182036), + ("held-10", 103, 2517, "triage", 25, {"extractor": 4, "answerer": 50, "judge": 50}, 0.245361), + ("held-11", 274, 3802, "triage", 25, {"extractor": 4, "answerer": 50, "judge": 50}, 0.246153), + ("held-12", 140, 6723, "triage", 25, {"extractor": 5, "answerer": 50, "judge": 50}, 0.338129), ] + +#: The error every projection is held to on the held-out runs, for calls, atoms and dollars. +#: The worst held-out dollar error is 33%; 35% is that figure rounded up. +TOLERANCE = 0.35 + _MODELS = {"extractor": ("us.anthropic.claude-sonnet-5-5", "amazon-bedrock"), "answerer": ("us.anthropic.claude-haiku-5-5", "amazon-bedrock"), "judge": ("us.anthropic.claude-sonnet-5-5", "amazon-bedrock")} +_BUDGETS = (0.75, 0.5) + + +def _projection(tier, spans, tokens): + """The estimate a run at this tier on this artifact prints, and its dollar figure.""" + estimate = resolve_settings(tier, {}).estimate( + spans, n_tokens=tokens, budgets=_BUDGETS, n_arms=2 + )["estimate"] + return estimate, estimate_usd(estimate["by_role"], tokens, _MODELS, tier) -def _measured_estimate(tier, spans, tokens): - return resolve_settings(tier, {}).estimate( - spans, n_tokens=tokens, budgets=(0.75, 0.5), n_arms=2)["estimate"] +def test_the_projection_constants_are_the_fitted_ones(): + # Asserted as values. A change to any of them changes the calibration, and the held-out + # numbers in docs/skill-novelty.md have to be fitted and restated with it. + assert TOKENS_PER_ATOM == 23 + assert ESCALATION_SHARE == 0.5 + assert CALL_TOKENS == {"answerer": (-38, 0.789, 284), "judge": (468, 0.004, 89)} + assert EXTRACTOR_TOKENS == {"triage": (971, 0.045, 2359), "standard": (267, 0.651, 195)} -def test_the_expected_calls_land_within_a_quarter_of_each_measured_run(): - # The old projection (one atom per two spans, no oracle escalations) was 37-69% under all - # six of these runs; the token rule and the escalation share bring every one within 20%. - for name, spans, tokens, tier, atoms, by_role, _ in MEASURED_RUNS: - estimate = _measured_estimate(tier, spans, tokens) +def test_held_out_calls_within_35_percent(): + for name, spans, tokens, tier, _, by_role, _ in MEASURED_RUNS: + estimate, _ = _projection(tier, spans, tokens) counted = sum(by_role.values()) - assert abs(estimate["expected_calls"] - counted) <= 0.25 * counted, name - assert abs(estimate["n_atoms"] - atoms) <= 0.25 * atoms, name + assert abs(estimate["expected_calls"] - counted) <= TOLERANCE * counted, name + + +def test_held_out_atoms_within_35_percent(): + for name, spans, tokens, tier, atoms, _, _ in MEASURED_RUNS: + estimate, _ = _projection(tier, spans, tokens) + assert abs(estimate["n_atoms"] - atoms) <= TOLERANCE * atoms, name + + +def test_held_out_dollars_within_35_percent(): + for name, spans, tokens, tier, _, _, usd in MEASURED_RUNS: + _, priced = _projection(tier, spans, tokens) + assert priced is not None, name + assert abs(priced - usd) <= TOLERANCE * usd, name + + +def test_held_out_triage_dollars_within_35_percent(): + # The triage runs are the ones the extractor's output per call is for: a flat profile + # priced them 21-43% low before the per-tier extractor profile. + triage = [row for row in MEASURED_RUNS if row[3] == "triage"] + assert len(triage) == 9 + for name, spans, tokens, tier, _, _, usd in triage: + _, priced = _projection(tier, spans, tokens) + assert abs(priced - usd) <= TOLERANCE * usd, name def test_the_expected_calls_sit_between_the_no_escalation_and_the_full_escalation_bounds(): for name, spans, tokens, tier, _, _, _ in MEASURED_RUNS: - estimate = _measured_estimate(tier, spans, tokens) + estimate, _ = _projection(tier, spans, tokens) assert (estimate["total_calls"] <= estimate["expected_calls"] <= estimate["total_calls_if_every_pair_escalates"]), name assert estimate["expected_calls"] == sum(estimate["by_role"].values()), name @@ -402,28 +446,19 @@ def test_the_expected_calls_sit_between_the_no_escalation_and_the_full_escalatio def test_the_atom_count_takes_the_token_rule_when_the_artifact_has_more_tokens_than_spans(): everything = resolve_settings(None, {"all_atoms": True}) assert everything.estimate(7)["estimate"]["n_atoms"] == 3 # spans only - assert everything.estimate(7, n_tokens=198)["estimate"]["n_atoms"] == 7 + assert everything.estimate(7, n_tokens=198)["estimate"]["n_atoms"] == 8 # Short spans with many tokens: the span rule was the smaller of the two. - assert everything.estimate(20, n_tokens=900)["estimate"]["n_atoms"] == 36 + assert everything.estimate(20, n_tokens=900)["estimate"]["n_atoms"] == 39 -def test_the_dollar_profile_reproduces_the_measured_standard_runs(): - # The profile priced at each run's own call counts lands within a quarter of what the - # run's `cost.usd` recorded. Triage is left out: its extraction calls write whole atom - # lists, which the flat profile does not model (see `call_tokens`). - from dokimi.novelty.tiers import estimate_usd - - for name, _, tokens, tier, _, by_role, usd in MEASURED_RUNS: - if tier != "standard": - continue - priced = estimate_usd(by_role, tokens, _MODELS) - assert priced is not None, name - assert abs(priced - usd) <= 0.25 * usd, name +def test_the_extractor_profile_follows_the_tier_and_study_takes_the_standard_one(): + assert call_tokens("extractor", 1000, "triage") == (1016, 2359) + assert call_tokens("extractor", 1000, "standard") == (918, 195) + assert call_tokens("extractor", 1000, "study") == call_tokens("extractor", 1000, "standard") + assert call_tokens("extractor", 1000) == call_tokens("extractor", 1000, "standard") def test_an_unpriced_model_gives_no_dollar_figure_rather_than_zero(): - from dokimi.novelty.tiers import estimate_usd - unpriced = {"answerer": ("no-such-model-for-dokimi", "openai")} assert estimate_usd({"answerer": 10}, 100, unpriced) is None From 6e01b780bd47b3f81fd718b95694a07967c28f09 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 23:46:34 -0700 Subject: [PATCH 03/14] test(novelty): pin the list prices the held-out dollar checks read The held-out dollar checks priced the projection through litellm's bundled price map. The locked litellm (1.105.0) has no entry for us.anthropic.claude-haiku-5-5, so the answerer priced to None on every CI Python and the checks failed; a newer local litellm passed. conftest keeps litellm on its bundled map, so the figures depended on the installed release. Pin the two models' list prices in the test with monkeypatch. The pinned figures equal the bundled 1.106.2 entries, so the projection and the 35% tolerance are unchanged. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_novelty_tiers.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index 4763b325..3230f104 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -387,6 +387,28 @@ def test_a_tier_on_the_config_and_the_older_tier_argument_record_the_same(tmp_pa "judge": ("us.anthropic.claude-sonnet-5-5", "amazon-bedrock")} _BUDGETS = (0.75, 0.5) +#: The list prices of the two models in ``_MODELS``, as litellm's bundled price map has them. +#: The dollar checks read the price map, and conftest keeps litellm on its bundled copy, so +#: the map is whichever release is installed: the locked litellm 1.105.0 has no entry for +#: ``us.anthropic.claude-haiku-5-5`` and the answerer priced to None. Pinned here, the checks +#: give the same figures on every release. +_PINNED_LIST_PRICES = { + "us.anthropic.claude-sonnet-5-5": {"litellm_provider": "bedrock_converse", "mode": "chat", + "input_cost_per_token": 2.2e-06, + "output_cost_per_token": 1.1e-05}, + "us.anthropic.claude-haiku-5-5": {"litellm_provider": "bedrock_converse", "mode": "chat", + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 5.5e-07}, +} + + +@pytest.fixture +def pinned_list_prices(monkeypatch): + from litellm import model_cost + + for name, entry in _PINNED_LIST_PRICES.items(): + monkeypatch.setitem(model_cost, name, entry) + def _projection(tier, spans, tokens): """The estimate a run at this tier on this artifact prints, and its dollar figure.""" @@ -418,14 +440,14 @@ def test_held_out_atoms_within_35_percent(): assert abs(estimate["n_atoms"] - atoms) <= TOLERANCE * atoms, name -def test_held_out_dollars_within_35_percent(): +def test_held_out_dollars_within_35_percent(pinned_list_prices): for name, spans, tokens, tier, _, _, usd in MEASURED_RUNS: _, priced = _projection(tier, spans, tokens) assert priced is not None, name assert abs(priced - usd) <= TOLERANCE * usd, name -def test_held_out_triage_dollars_within_35_percent(): +def test_held_out_triage_dollars_within_35_percent(pinned_list_prices): # The triage runs are the ones the extractor's output per call is for: a flat profile # priced them 21-43% low before the per-tier extractor profile. triage = [row for row in MEASURED_RUNS if row[3] == "triage"] From 5dd705778233bffa21601b43bc9c0b84606908df Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 13:54:33 +0000 Subject: [PATCH 04/14] docs(novelty): re-wrap the tier override sentence Signed-off-by: Anton Volkov --- dokimi/docs/skill-novelty.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5d1a681e..41e3865f 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -791,8 +791,8 @@ price. The constants were fitted on 13 of the 25 measured runs; the other 12 are On those 12 every projection landed within 35%: calls within 13%, atoms within 11%, dollars within 33%. Every dollar projection came out high, not low. Only three of the 12 are standard runs, so the standard profile rests on little data. An explicit flag always beats the tier, -and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get -to be read as one. +and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage +run and does not get to be read as one. Each tier carries the **claims it forfeits**, and those strings are written into `run_config.tier` *and* appended to the artifact's warnings. They follow the settings the From de6c34f4a144d67de1532c64e012650a67f4b57c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 13:54:58 +0000 Subject: [PATCH 05/14] docs(novelty): note the litellm price-map fetch in the estimate Signed-off-by: Anton Volkov --- dokimi/docs/skill-novelty.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 41e3865f..8092351c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -784,7 +784,12 @@ python cli.py novelty analyze --artifact skill:s --tier study --estimate-only ``` `--estimate-only` prints the projected bill and exits without constructing a client, so -the decision costs nothing. Its payload has `expected_calls`, the point figure between +the decision costs nothing. Pricing imports litellm, which tries to fetch its price map +from the network before it falls back to the copy bundled with the package; on a host +with no network that is a pause of a few seconds and a warning. Setting +`LITELLM_LOCAL_MODEL_COST_MAP=True` skips the fetch and uses the bundled prices. + +The estimate payload has `expected_calls`, the point figure between the two totals (stages, every probe, and half the oracle pairs, split by `by_role`), and `expected_usd`: those calls priced at list prices by role, `null` when a model has no price. The constants were fitted on 13 of the 25 measured runs; the other 12 are held out. From 88d697afe5ea0ad49b184f437ba7f9768f03b8ef Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 08:06:56 -0700 Subject: [PATCH 06/14] fix(novelty): correct the dollar-projection sign claim; pin the tier and no-price branches The held-out dollar projections ran high on 5 runs and low on 7 (worst -33%). Update the walkthrough page for expected_calls, by_role and expected_usd. Signed-off-by: Nikolay Petrov --- ...1_120000_napetrov_estimate-expected-usd.md | 3 +- dokimi/docs/skill-novelty-walkthrough.md | 18 ++++++----- dokimi/docs/skill-novelty.md | 4 ++- dokimi/tests/test_novelty_tiers.py | 32 +++++++++++++++++++ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md index b82827e4..b000d0ee 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md @@ -8,5 +8,6 @@ with the extractor's output per call set per tier, and is `null` when a model has no price. The constants were fitted on 13 of 25 measured runs; on the other 12 (held out) calls landed within 13%, atoms within 11% and dollars within 33%, all within the 35% - tolerance, and dollars came out high, never low. `total_calls` and + tolerance. Dollars ran high on 5 runs and low on 7 (worst -33%), so `expected_usd` is not a + ceiling. `total_calls` and `total_calls_if_every_pair_escalates` are unchanged. diff --git a/dokimi/docs/skill-novelty-walkthrough.md b/dokimi/docs/skill-novelty-walkthrough.md index 6497f0f0..43400e09 100644 --- a/dokimi/docs/skill-novelty-walkthrough.md +++ b/dokimi/docs/skill-novelty-walkthrough.md @@ -62,16 +62,18 @@ python cli.py novelty analyze --artifact skill:skills/my-skill --tier study --es Pass the same `--include-resources` (and `--tier`) you will run with: the bill covers only the files the flags select, and a thin `SKILL.md` over `references/` is priced many times -lower without it. It prints the projected model-call bill and exits before building a client. The bill is -`probes × arms × (answer + grade)` plus the retention oracle, which escalates only its -ambiguous band — so the output carries both ends, `total_calls` and -`total_calls_if_every_pair_escalates`. The probe term was validated to within 1–3% of +lower without it. It prints the projected model-call bill and exits before building a client. The +bill is `probes × arms × (answer + grade)` plus the retention oracle, which escalates only its +ambiguous band. The output carries `expected_calls`, the point figure, with `by_role` splitting +it into extractor, answerer and judge calls; `total_calls` and +`total_calls_if_every_pair_escalates` are the two ends. `expected_usd` prices `by_role` at list +prices, and is `null` when a model has no price. The probe term was validated to within 1–3% of counted calls at nine (skill, tier) points, on content that did not escalate. -The bill is in calls, not dollars: a call's price follows the artifact's length and the -models. Pass `--max-usd ` to every step that calls a model: the run stops once its calls -across all roles have cost that much (calls already in flight can overshoot it), and a call -with no known price stops it too, since the cap cannot see that spend. One measured pair, a +`expected_usd` is a projection, not a ceiling (held-out runs ran up to 33% low): a call's price +follows the artifact's length and the models. Pass `--max-usd ` to every step that calls a +model: the run stops once its calls across all roles have cost that much (calls already in +flight can overshoot it), and a call with no known price stops it too, since the cap cannot see that spend. One measured pair, a 19k-token skill: triage 103 calls, about $1.80; `study` 1,901 calls, about $14.50. ## 3. Triage: is there anything in here? (~100 calls) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 8092351c..8a289cba 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -794,7 +794,9 @@ the two totals (stages, every probe, and half the oracle pairs, split by `by_rol `expected_usd`: those calls priced at list prices by role, `null` when a model has no price. The constants were fitted on 13 of the 25 measured runs; the other 12 are held out. On those 12 every projection landed within 35%: calls within 13%, atoms within 11%, dollars -within 33%. Every dollar projection came out high, not low. Only three of the 12 are standard +within 33%. The dollar projections ran high on 5 runs (up to +29%) and low on 7 (down to +-33%), so `expected_usd` is not a ceiling; the triage projection ran lower the larger the +artifact was (-4% at 1,153 tokens, -33% at 6,723). Only three of the 12 are standard runs, so the standard profile rests on little data. An explicit flag always beats the tier, and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get to be read as one. diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index 3230f104..3fa39d14 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -497,3 +497,35 @@ def test_estimate_only_prints_the_expected_calls_and_a_dollar_figure(tmp_path, c assert payload["estimate"]["expected_calls"] == sum(payload["estimate"]["by_role"].values()) assert payload["expected_usd"] > 0 # default models are priced; null would be a regression assert payload["tokens"] > 0 + + +def test_the_cli_prices_a_triage_estimate_with_the_triage_extractor_profile(tmp_path, capsys): + from dokimi.commands.novelty import _role_models + + root = tmp_path / "s" + root.mkdir() + (root / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "triage", + "--estimate-only") + args.func(args) + payload = json.loads(capsys.readouterr().out) + by_role, tokens = payload["estimate"]["by_role"], payload["tokens"] + models = _role_models(args) + triage = estimate_usd(by_role, tokens, models, "triage") + assert payload["expected_usd"] == triage + assert triage != estimate_usd(by_role, tokens, models, None) + + +def test_list_cost_usd_distinguishes_an_unpriced_model_from_a_free_one(monkeypatch): + import litellm + + from dokimi.llm import list_cost_usd + + # cost_per_token prices a model it has no entry for at 0.0 when the route is known. + monkeypatch.setattr(litellm, "cost_per_token", lambda **_: (0.0, 0.0)) + assert list_cost_usd("not-in-the-price-map-xyz", "openai", 100, 10) is None + monkeypatch.setitem(litellm.model_cost, "free-in-the-price-map-xyz", + {"litellm_provider": "openai", "mode": "chat", + "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}) + assert list_cost_usd("free-in-the-price-map-xyz", "openai", 100, 10) == 0.0 From 09f678a7efc847796bfdfa7b63f3a730e635abec Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 22 Oct 2026 17:28:30 +0000 Subject: [PATCH 07/14] docs(novelty): re-wrap the over-long line in the cost paragraph Signed-off-by: Anton Volkov --- dokimi/docs/skill-novelty-walkthrough.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/skill-novelty-walkthrough.md b/dokimi/docs/skill-novelty-walkthrough.md index 43400e09..93e79927 100644 --- a/dokimi/docs/skill-novelty-walkthrough.md +++ b/dokimi/docs/skill-novelty-walkthrough.md @@ -73,8 +73,9 @@ counted calls at nine (skill, tier) points, on content that did not escalate. `expected_usd` is a projection, not a ceiling (held-out runs ran up to 33% low): a call's price follows the artifact's length and the models. Pass `--max-usd ` to every step that calls a model: the run stops once its calls across all roles have cost that much (calls already in -flight can overshoot it), and a call with no known price stops it too, since the cap cannot see that spend. One measured pair, a -19k-token skill: triage 103 calls, about $1.80; `study` 1,901 calls, about $14.50. +flight can overshoot it), and a call with no known price stops it too, +since the cap cannot see that spend. One measured pair, a 19k-token skill: triage 103 calls, +about $1.80; `study` 1,901 calls, about $14.50. ## 3. Triage: is there anything in here? (~100 calls) From 5674d62cb526f03be76fa61d2705a2df1ef89e75 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:42:28 -0700 Subject: [PATCH 08/14] fix(novelty): keep litellm's provider banner off the --estimate-only JSON; pin the call_tokens floor Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm.py | 18 +++++++++++------- dokimi/tests/test_novelty_tiers.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 92e23e00..31756728 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -73,6 +73,7 @@ import logging import os import re +import sys import threading import time from dataclasses import dataclass, field, replace @@ -346,13 +347,16 @@ def list_cost_usd(model: str, provider: str, prompt_tokens: int, from litellm import cost_per_token litellm_model = resolve_model(model, provider) - try: - prompt, completion = cost_per_token(model=litellm_model, prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens) - except Exception: - return None - if prompt + completion == 0.0 and not _has_price_entry(litellm_model, model): - return None + # litellm prints a "Provider List" help banner to stdout before it raises for a model + # it cannot route. ``--estimate-only`` prints JSON on stdout, so keep that out of it. + with contextlib.redirect_stdout(sys.stderr): + try: + prompt, completion = cost_per_token(model=litellm_model, prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens) + except Exception: + return None + if prompt + completion == 0.0 and not _has_price_entry(litellm_model, model): + return None return float(prompt + completion) diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index 3fa39d14..d96df427 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -529,3 +529,25 @@ def test_list_cost_usd_distinguishes_an_unpriced_model_from_a_free_one(monkeypat {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}) assert list_cost_usd("free-in-the-price-map-xyz", "openai", 100, 10) == 0.0 + + +def test_estimate_only_stdout_stays_json_for_a_model_litellm_cannot_route(tmp_path, capsys): + # litellm prints a "Provider List" banner to stdout before it raises for such a model. + root = tmp_path / "s" + root.mkdir() + (root / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "standard", + "--estimate-only", "--model", "foo-bar-9", "--provider", "openai") + args.func(args) + payload = json.loads(capsys.readouterr().out) + assert payload["expected_usd"] is None + + +def test_call_tokens_never_goes_negative_for_a_tiny_artifact(): + # The answerer's base is negative (-38 + 0.789 per artifact token), so a 10-token + # artifact would price a negative prompt without the floor. + for role in ("extractor", "answerer", "judge"): + for tier in (None, "triage", "standard"): + assert min(call_tokens(role, 10, tier)) >= 0 + assert call_tokens("answerer", 10)[0] == 0 From 575819b2c3d9e7fafe23e792c9bb0aeab1d70652 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:05:40 -0700 Subject: [PATCH 09/14] refactor(llm): move list_cost_usd to llm_pricing to keep llm.py at its baseline; chore(growth): raise commands/novelty.py for W2 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 2 +- dokimi/dokimi/llm.py | 25 --------------------- dokimi/dokimi/llm_pricing.py | 35 +++++++++++++++++++++++++++++ dokimi/dokimi/novelty/tiers.py | 4 ++-- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_novelty_tiers.py | 2 +- 6 files changed, 40 insertions(+), 30 deletions(-) create mode 100644 dokimi/dokimi/llm_pricing.py diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index fffb125e..154bbe1e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -752,7 +752,7 @@ def _estimate_usd( The reason goes to stderr: a role whose model does not resolve, or a role with no list price, makes the figure unavailable, and the note names it. """ - from dokimi.llm import list_cost_usd + from dokimi.llm_pricing import list_cost_usd from dokimi.novelty.tiers import estimate_usd try: diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index af596d45..8cf5fe85 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -73,7 +73,6 @@ import logging import os import re -import sys import threading import time from dataclasses import dataclass, field, replace @@ -336,30 +335,6 @@ def _safe_completion_cost(resp: Any, model: str, litellm_model: Optional[str] = return None -def list_cost_usd(model: str, provider: str, prompt_tokens: int, - completion_tokens: int) -> Optional[float]: - """Dollar cost of one call of these token counts at list price, or ``None``. - - The price map :func:`_safe_completion_cost` reads, for a call that has not been - made: an estimate prices a token profile rather than a response. ``None`` when the - model has no entry, for the same reason as there. - """ - from litellm import cost_per_token - - litellm_model = resolve_model(model, provider) - # litellm prints a "Provider List" help banner to stdout before it raises for a model - # it cannot route. ``--estimate-only`` prints JSON on stdout, so keep that out of it. - with contextlib.redirect_stdout(sys.stderr): - try: - prompt, completion = cost_per_token(model=litellm_model, prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens) - except Exception: - return None - if prompt + completion == 0.0 and not _has_price_entry(litellm_model, model): - return None - return float(prompt + completion) - - def _extract_usage( resp: Any, model: str, diff --git a/dokimi/dokimi/llm_pricing.py b/dokimi/dokimi/llm_pricing.py new file mode 100644 index 00000000..1f86ee0f --- /dev/null +++ b/dokimi/dokimi/llm_pricing.py @@ -0,0 +1,35 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""List-price estimates for a call that has not been made (kept out of ``dokimi.llm``).""" + +from __future__ import annotations + +import contextlib +import sys + + +def list_cost_usd(model: str, provider: str, prompt_tokens: int, + completion_tokens: int) -> float | None: + """Dollar cost of one call of these token counts at list price, or ``None``. + + The price map :func:`dokimi.llm._safe_completion_cost` reads, for a call that has not + been made: an estimate prices a token profile rather than a response. ``None`` when the + model has no entry, for the same reason as there. + """ + from litellm import cost_per_token + + from dokimi.llm import _has_price_entry, resolve_model + + litellm_model = resolve_model(model, provider) + # litellm prints a "Provider List" help banner to stdout before it raises for a model + # it cannot route. ``--estimate-only`` prints JSON on stdout, so keep that out of it. + with contextlib.redirect_stdout(sys.stderr): + try: + prompt, completion = cost_per_token(model=litellm_model, prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens) + except Exception: + return None + if prompt + completion == 0.0 and not _has_price_entry(litellm_model, model): + return None + return float(prompt + completion) diff --git a/dokimi/dokimi/novelty/tiers.py b/dokimi/dokimi/novelty/tiers.py index 77cd2a95..90c60b12 100644 --- a/dokimi/dokimi/novelty/tiers.py +++ b/dokimi/dokimi/novelty/tiers.py @@ -331,7 +331,7 @@ def estimate_usd( ) -> float | None: """The expected dollar cost of ``by_role`` calls at list prices. - Prices come from litellm's price map (:func:`dokimi.llm.list_cost_usd`), the same map a + Prices come from litellm's price map (:func:`dokimi.llm_pricing.list_cost_usd`), the same map a run's own ``cost_usd`` uses, so the figure is the run's price applied to a token profile. ``None`` when any role's model has no price: an unpriced call is unknown, not free. @@ -341,7 +341,7 @@ def estimate_usd( models: ``(model, provider)`` per role. tier: The run's tier; it sets the extractor's output (see :func:`call_tokens`). """ - from dokimi.llm import list_cost_usd + from dokimi.llm_pricing import list_cost_usd total = 0.0 for role, calls in by_role.items(): diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8..51c4db6f 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 878, "rationale": "W2 adds the estimate flags; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index d96df427..42cdf0bd 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -520,7 +520,7 @@ def test_the_cli_prices_a_triage_estimate_with_the_triage_extractor_profile(tmp_ def test_list_cost_usd_distinguishes_an_unpriced_model_from_a_free_one(monkeypatch): import litellm - from dokimi.llm import list_cost_usd + from dokimi.llm_pricing import list_cost_usd # cost_per_token prices a model it has no entry for at 0.0 when the route is known. monkeypatch.setattr(litellm, "cost_per_token", lambda **_: (0.0, 0.0)) From a926010b1609f243b9551cd5d1d51c9b40030baa Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:11:01 -0700 Subject: [PATCH 10/14] fix(llm): reach litellm's cost_per_token through dokimi.llm; place llm_pricing in the foundation layer Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm.py | 18 +++++++++--------- dokimi/dokimi/llm_pricing.py | 4 +--- dokimi/tests/test_import_architecture.py | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 8cf5fe85..4d462cea 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -285,20 +285,20 @@ def is_retryable(exc: BaseException) -> bool: def _has_price_entry(litellm_model: str, model: str) -> bool: - """Whether litellm's price map has an entry for this model. - - Looks up the routed name (``bedrock/us.anthropic.x``), the name without its - route prefixes (``us.anthropic.x``; ``bedrock/converse/openai.x`` -> - ``openai.x``) and the caller's bare model name. - """ + """Whether litellm's price map has an entry for the routed name, the name without its + route prefixes (``bedrock/converse/openai.x`` -> ``openai.x``) or the bare model name.""" from litellm import model_cost - candidates = {litellm_model, model, litellm_model.rsplit("/", 1)[-1]} - if "/" in litellm_model: - candidates.add(litellm_model.split("/", 1)[1]) + candidates = {litellm_model, model, *litellm_model.split("/", 1)[1:], + litellm_model.rsplit("/", 1)[-1]} return any(c in model_cost for c in candidates) +def cost_per_token(**kwargs: Any) -> tuple[float, float]: # litellm's, for llm_pricing + from litellm import cost_per_token as price + return price(**kwargs) + + def _safe_completion_cost(resp: Any, model: str, litellm_model: Optional[str] = None) -> Optional[float]: """Dollar cost of a completion via litellm, or ``None`` if no pricing. diff --git a/dokimi/dokimi/llm_pricing.py b/dokimi/dokimi/llm_pricing.py index 1f86ee0f..b99f2179 100644 --- a/dokimi/dokimi/llm_pricing.py +++ b/dokimi/dokimi/llm_pricing.py @@ -17,9 +17,7 @@ def list_cost_usd(model: str, provider: str, prompt_tokens: int, been made: an estimate prices a token profile rather than a response. ``None`` when the model has no entry, for the same reason as there. """ - from litellm import cost_per_token - - from dokimi.llm import _has_price_entry, resolve_model + from dokimi.llm import _has_price_entry, cost_per_token, resolve_model litellm_model = resolve_model(model, provider) # litellm prints a "Provider List" help banner to stdout before it raises for a model diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 5df0aac9..4733ee32 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -69,7 +69,7 @@ # process chokepoints, console output. ("foundation", ( "dokimi", "dokimi._version", "dokimi.artifacts", "dokimi.caching", "dokimi.config", - "dokimi.console", "dokimi.defaults", "dokimi.intents", "dokimi.llm", + "dokimi.console", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_pricing", "dokimi.process", "dokimi.records", "dokimi.registry", "dokimi.utils", # Token/cost record the LLM chokepoint fills; it lives under metrics/ but # depends on nothing and is read by every LLM caller. From 96daabd0dcd9a84741458b5467004b2bdb849a30 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:57:25 -0700 Subject: [PATCH 11/14] docs(changelog): say the legacy estimate totals follow the new atom count total_calls and total_calls_if_every_pair_escalates keep their formula, but n_atoms now takes the token rule, so both rise wherever the span rule left the count under the tier cap and the token rule gives more. Signed-off-by: Alexander Rybkin --- .../20261021_120000_napetrov_estimate-expected-usd.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md index b000d0ee..7fd25455 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md @@ -9,5 +9,6 @@ price. The constants were fitted on 13 of 25 measured runs; on the other 12 (held out) calls landed within 13%, atoms within 11% and dollars within 33%, all within the 35% tolerance. Dollars ran high on 5 runs and low on 7 (worst -33%), so `expected_usd` is not a - ceiling. `total_calls` and - `total_calls_if_every_pair_escalates` are unchanged. + ceiling. `total_calls` and `total_calls_if_every_pair_escalates` keep their formula but + follow the new atom count, so they rise wherever the span rule left the count under the + cap and the token rule gives more (a 588-token skill at `--tier standard`: 129 -> 229). From bdf8d85225c9b78ee9840473b29971a474c803a7 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:53:33 -0700 Subject: [PATCH 12/14] test(novelty): pin the null dollar figure when no judge resolves --estimate-only catches the SystemExit from _role_models and prints expected_usd null with a note; nothing reached that branch, so catching the wrong exception aborted the preview with every test green. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_tiers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index 42cdf0bd..2db9e0ff 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -551,3 +551,22 @@ def test_call_tokens_never_goes_negative_for_a_tiny_artifact(): for tier in (None, "triage", "standard"): assert min(call_tokens(role, 10, tier)) >= 0 assert call_tokens("answerer", 10)[0] == 0 + + +def _one_line_skill(root): + root.mkdir(parents=True) + (root / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + return root + + +def test_estimate_only_gives_a_null_dollar_figure_when_no_judge_resolves(tmp_path, capsys): + # A real run exits here (openrouter has no default judge); the preview still prints. + args = _parse("novelty", "analyze", "--artifact", f"skill:{_one_line_skill(tmp_path / 's')}", + "--tier", "triage", "--estimate-only", "--judge-provider", "openrouter") + args.func(args) + out = capsys.readouterr() + payload = json.loads(out.out) + assert payload["expected_usd"] is None + assert payload["estimate"]["expected_calls"] > 0 + assert "USD estimate is unavailable: --judge-provider openrouter" in out.err From 76a567c2533b7fed978b10a6076d59b72e845ca9 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 00:53:44 -0700 Subject: [PATCH 13/14] test(novelty): pin that by_role scales answer and grade calls with --arms Every fit row and test used two arms, so replacing n_arms with 2 in estimate_calls' by_role passed the suite while --arms with a third arm under-counted the answerer and judge bill by a third. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_tiers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index 2db9e0ff..bfeb4f31 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -570,3 +570,17 @@ def test_estimate_only_gives_a_null_dollar_figure_when_no_judge_resolves(tmp_pat assert payload["expected_usd"] is None assert payload["estimate"]["expected_calls"] > 0 assert "USD estimate is unavailable: --judge-provider openrouter" in out.err + + +def test_every_extra_arm_adds_an_answer_and_a_grade_per_probe_to_the_bill(tmp_path, capsys): + by_role = {} + for arms in ("no_artifact,full", "no_artifact,full,shuffled"): + skill = _one_line_skill(tmp_path / str(len(by_role))) + args = _parse("novelty", "analyze", "--artifact", f"skill:{skill}", "--tier", "standard", + "--estimate-only", "--arms", arms) + args.func(args) + by_role[arms] = json.loads(capsys.readouterr().out)["estimate"]["by_role"] + two, three = by_role["no_artifact,full"], by_role["no_artifact,full,shuffled"] + assert three["answerer"] * 2 == two["answerer"] * 3 > 0 + assert three["judge"] * 2 == two["judge"] * 3 > 0 + assert three["extractor"] == two["extractor"] From ee809eccb7014d06c8068a1ccbb458279a0e7f35 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:54:18 -0700 Subject: [PATCH 14/14] fix(novelty): price the answerer per context arm, flag extrapolated estimates, pin the estimate inputs The answerer's artifact term now scales with the arms that carry the artifact, and an artifact above the largest measured one sets usd_extrapolated. Tests pin the reversal term of by_role and the artifact token count passed to estimate_usd. Signed-off-by: Nikolay Petrov --- ...1_120000_napetrov_estimate-expected-usd.md | 3 + dokimi/docs/skill-novelty.md | 8 ++- dokimi/dokimi/commands/novelty.py | 17 ++++- dokimi/dokimi/novelty/tiers.py | 24 ++++++- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_novelty_tiers.py | 66 ++++++++++++++++++- 6 files changed, 111 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md index 7fd25455..f0316ef6 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_estimate-expected-usd.md @@ -12,3 +12,6 @@ ceiling. `total_calls` and `total_calls_if_every_pair_escalates` keep their formula but follow the new atom count, so they rise wherever the span rule left the count under the cap and the token rule gives more (a 588-token skill at `--tier standard`: 129 -> 229). + The answerer's artifact term is priced per arm that carries the artifact, so + `--arms no_artifact,full,shuffled` projects higher than the two-arm default; an artifact + above the largest one measured (6,723 tokens) sets `usd_extrapolated: true`. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 8ae799a8..27eb022c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -803,8 +803,12 @@ price. The constants were fitted on 13 of the 25 measured runs; the other 12 are On those 12 every projection landed within 35%: calls within 13%, atoms within 11%, dollars within 33%. The dollar projections ran high on 5 runs (up to +29%) and low on 7 (down to -33%), so `expected_usd` is not a ceiling; the triage projection ran lower the larger the -artifact was (-4% at 1,153 tokens, -33% at 6,723). Only three of the 12 are standard -runs, so the standard profile rests on little data. An explicit flag always beats the tier, +artifact was (-4% at 1,153 tokens, -33% at 6,723). The largest artifact measured has 6,723 +tokens: above that the dollar figure extrapolates, and the payload says so +(`usd_extrapolated: true`, with a note on stderr). The answerer's artifact term is priced per +arm that carries the artifact (every arm but `no_artifact`), so a three-arm run costs more +per answerer call than a two-arm one; a `--placebo-text` of another length is not read. Only +three of the 12 are standard runs, so the standard profile rests on little data. An explicit flag always beats the tier, and the override is recorded — a `--tier triage --max-atoms 90` run is not a stock triage run and does not get to be read as one. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 773bddfc..fccabb1c 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -49,6 +49,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: ) from dokimi.novelty.spans import segment + from dokimi.novelty.tiers import FITTED_TOKENS_MAX artifact_bundle = _bundle_from_args(args) settings = _settings(args) @@ -67,9 +68,18 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: estimate = settings.estimate( len(spans), n_tokens=n_tokens, budgets=tuple(args.budgets), n_arms=len(_arms(args.arms)) ) + n_arms = len(_arms(args.arms)) estimate["expected_usd"] = _estimate_usd( - args, n_tokens, estimate["estimate"]["by_role"], settings.tier + args, n_tokens, estimate["estimate"]["by_role"], settings.tier, + context_share=(n_arms - 1) / n_arms, ) + # Every arm but the baseline is priced as carrying the artifact; a placebo of another + # length is not read. + if n_tokens > FITTED_TOKENS_MAX: + estimate["usd_extrapolated"] = True + print(f"expected_usd extrapolates: the artifact has {n_tokens} tokens and the " + f"profiles were fitted on artifacts up to {FITTED_TOKENS_MAX}; the projection " + "ran low as the artifact grew", file=sys.stderr) print(json.dumps(estimate, indent=2)) print("Estimate only, nothing was measured. ~4 calls per paired probe is the " "measured shape; batch sizes and retries move the real figure.", @@ -781,7 +791,8 @@ def _role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: def _estimate_usd( - args: argparse.Namespace, n_tokens: int, by_role: dict[str, int], tier: str | None + args: argparse.Namespace, n_tokens: int, by_role: dict[str, int], tier: str | None, + context_share: float = 0.5, ) -> float | None: """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced. @@ -796,7 +807,7 @@ def _estimate_usd( except SystemExit as exc: print(f"The USD estimate is unavailable: {exc}", file=sys.stderr) return None - usd = estimate_usd(by_role, n_tokens, models, tier) + usd = estimate_usd(by_role, n_tokens, models, tier, context_share) if usd is None: unpriced = sorted( f"{role} ({model})" diff --git a/dokimi/dokimi/novelty/tiers.py b/dokimi/dokimi/novelty/tiers.py index 90c60b12..35bf3fc4 100644 --- a/dokimi/dokimi/novelty/tiers.py +++ b/dokimi/dokimi/novelty/tiers.py @@ -307,7 +307,19 @@ def resolve_settings(tier: str | None, flags: dict) -> RunSettings: } -def call_tokens(role: str, n_tokens: int, tier: str | None = None) -> tuple[int, int]: +#: The largest artifact (in tokens) among the measured runs the profiles above were fitted +#: and checked on. A projection for a bigger artifact extrapolates: the triage figure ran +#: lower as the artifact grew (-4% at 1,153 tokens, -33% at 6,723). +FITTED_TOKENS_MAX = 6723 + +#: The share of the answerer's calls whose context carries the artifact in the measured +#: runs, which had two arms (the baseline carries none, ``full`` carries it). The answerer's +#: per-token input is a mean over its arms, so it scales with this share. +MEASURED_CONTEXT_SHARE = 0.5 + + +def call_tokens(role: str, n_tokens: int, tier: str | None = None, + context_share: float = MEASURED_CONTEXT_SHARE) -> tuple[int, int]: """Input and output tokens of one call of ``role`` on an artifact of ``n_tokens``. Args: @@ -315,11 +327,17 @@ def call_tokens(role: str, n_tokens: int, tier: str | None = None) -> tuple[int, n_tokens: Tokens in the artifact. tier: The run's tier, which picks the extractor's profile (see :data:`EXTRACTOR_TOKENS`). + context_share: The share of the answerer's calls whose context carries the artifact + (arms that carry it over arms run). The answerer's artifact term is a mean over + its arms, so it scales with ``context_share / MEASURED_CONTEXT_SHARE``: 1 at the + default two arms, 1.5 for ``no_artifact,full,placebo,shuffled``. """ if role == "extractor": base, per_token, out = EXTRACTOR_TOKENS.get(tier or "", EXTRACTOR_TOKENS["standard"]) else: base, per_token, out = CALL_TOKENS[role] + if role == "answerer": + per_token *= context_share / MEASURED_CONTEXT_SHARE return max(0, round(base + per_token * n_tokens)), out @@ -328,6 +346,7 @@ def estimate_usd( n_tokens: int, models: dict[str, tuple[str, str]], tier: str | None = None, + context_share: float = MEASURED_CONTEXT_SHARE, ) -> float | None: """The expected dollar cost of ``by_role`` calls at list prices. @@ -340,13 +359,14 @@ def estimate_usd( n_tokens: Tokens in the artifact. models: ``(model, provider)`` per role. tier: The run's tier; it sets the extractor's output (see :func:`call_tokens`). + context_share: Arms that carry the artifact over arms run (see :func:`call_tokens`). """ from dokimi.llm_pricing import list_cost_usd total = 0.0 for role, calls in by_role.items(): model, provider = models[role] - tokens_in, tokens_out = call_tokens(role, n_tokens, tier) + tokens_in, tokens_out = call_tokens(role, n_tokens, tier, context_share) cost = list_cost_usd(model, provider, tokens_in, tokens_out) if cost is None: return None diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 1dde9158..bb953476 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 914, "rationale": "estimate-only projection in novelty CLI; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 925, "rationale": "estimate-only projection in novelty CLI; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_tiers.py b/dokimi/tests/test_novelty_tiers.py index bfeb4f31..e1a3ffab 100644 --- a/dokimi/tests/test_novelty_tiers.py +++ b/dokimi/tests/test_novelty_tiers.py @@ -16,6 +16,7 @@ DEFAULT_SETTINGS, ESCALATION_SHARE, EXTRACTOR_TOKENS, + FITTED_TOKENS_MAX, FLAG_DEFAULTS, FORFEITS, SETTING_NAMES, @@ -504,7 +505,9 @@ def test_the_cli_prices_a_triage_estimate_with_the_triage_extractor_profile(tmp_ root = tmp_path / "s" root.mkdir() - (root / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + body = "\n\n".join(f"Section {i}. Do not enable the legacy driver number {i} on a " + "production host without a rollback plan." for i in range(60)) + (root / "SKILL.md").write_text(f"---\nname: s\ndescription: d\n---\n\n# S\n\n{body}\n", encoding="utf-8") args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "triage", "--estimate-only") @@ -515,6 +518,9 @@ def test_the_cli_prices_a_triage_estimate_with_the_triage_extractor_profile(tmp_ triage = estimate_usd(by_role, tokens, models, "triage") assert payload["expected_usd"] == triage assert triage != estimate_usd(by_role, tokens, models, None) + # The artifact's token count is an input: a figure at 0 tokens is a different one. + assert tokens > 100 and triage != estimate_usd(by_role, 0, models, "triage") + assert "usd_extrapolated" not in payload def test_list_cost_usd_distinguishes_an_unpriced_model_from_a_free_one(monkeypatch): @@ -584,3 +590,61 @@ def test_every_extra_arm_adds_an_answer_and_a_grade_per_probe_to_the_bill(tmp_pa assert three["answerer"] * 2 == two["answerer"] * 3 > 0 assert three["judge"] * 2 == two["judge"] * 3 > 0 assert three["extractor"] == two["extractor"] + + +def test_a_study_run_counts_its_reversal_probes_in_the_answerer_and_judge_calls(): + study = resolve_settings("study", {}) + by_role = study.estimate(40, n_tokens=2000, n_arms=2)["estimate"] + assert by_role["reversal_probes"] > 0 + expected = (by_role["paired_probes"] + by_role["reversal_probes"]) * 2 + assert by_role["by_role"]["answerer"] == by_role["by_role"]["judge"] == expected + + +def test_the_answerer_artifact_term_scales_with_the_arms_that_carry_it(): + two = call_tokens("answerer", 6000)[0] + assert call_tokens("answerer", 6000, None, 0.5) == call_tokens("answerer", 6000) + three = call_tokens("answerer", 6000, None, 2 / 3)[0] + four = call_tokens("answerer", 6000, None, 3 / 4)[0] + assert two < three < four + assert three == round(-38 + 0.789 * (2 / 3) / 0.5 * 6000) + # The judge never sees the artifact, so its input does not move with the arms. + assert call_tokens("judge", 6000, None, 3 / 4) == call_tokens("judge", 6000) + + +def test_the_cli_prices_three_context_arms_above_two(tmp_path, capsys): + root = tmp_path / "s" + root.mkdir() + body = "\n\n".join(f"Section {i}. Do not enable legacy driver {i} on a production host." + for i in range(60)) + (root / "SKILL.md").write_text(f"---\nname: s\ndescription: d\n---\n\n# S\n\n{body}\n", + encoding="utf-8") + figures = [] + for arms in ("no_artifact,full", "no_artifact,full,shuffled"): + args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "triage", + "--arms", arms, "--estimate-only") + args.func(args) + figures.append(json.loads(capsys.readouterr().out)) + two, three = figures + # Three arms run 1.5x the answerer and judge calls, and the answerer's calls carry more of + # the artifact each, so the dollars rise by more than the calls alone. + calls = (three["estimate"]["by_role"]["answerer"] / two["estimate"]["by_role"]["answerer"]) + assert calls == 1.5 + assert three["expected_usd"] > two["expected_usd"] + + +def test_an_estimate_for_an_artifact_beyond_the_fitted_range_says_it_extrapolates( + tmp_path, capsys): + root = tmp_path / "s" + root.mkdir() + body = "\n\n".join(f"Section {i}. " + "Do not enable the legacy driver on a host. " * 12 + for i in range(120)) + (root / "SKILL.md").write_text(f"---\nname: s\ndescription: d\n---\n\n# S\n\n{body}\n", + encoding="utf-8") + args = _parse("novelty", "analyze", "--artifact", f"skill:{root}", "--tier", "triage", + "--estimate-only") + args.func(args) + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert payload["tokens"] > FITTED_TOKENS_MAX + assert payload["usd_extrapolated"] is True + assert "extrapolates" in captured.err