From 654b935874053975591987df206863c5f541e090 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 08:59:30 -0400 Subject: [PATCH 1/5] Give a model with no recorded tokens no cost, not $0 model_cost_latency filled a missing total_cost_usd from PRICE_OVERRIDES_PER_1M with unrecorded token counts counted as 0, so Claude Fable 5, whose batch rows record no usage, came out at costUsd 0.0. The fill now needs recorded prompt and completion counts; otherwise cost is omitted. The paper's cost table looped only over models model_cost_latency returns, so it now seeds every board model, and Fable 5 keeps its modelStats fallback. Co-Authored-By: Claude Opus 5.5 --- paper/index.qmd | 5 +- policybench/analysis.py | 22 +++-- tests/test_cost_latency.py | 197 ++++++++++++++++++++++++++++++++++++- 3 files changed, 214 insertions(+), 10 deletions(-) diff --git a/paper/index.qmd b/paper/index.qmd index cfd9e3e0..63680c56 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -748,7 +748,8 @@ def load_us_cost_latency() -> dict: # through the same model_cost_latency function as the leaderboard pipeline, # then are reconciled against the frozen # modelStats: Claude Fable 5's usage is recorded at the run level (its - # per-row usage columns are empty), and batch or asynchronous waves log + # per-row usage columns are empty, so model_cost_latency returns nothing + # for it), and batch or asynchronous waves log # submission times rather than generation time, so a zero or sub-second # median is treated as unmeasured and falls back to modelStats or "—". from policybench.analysis import model_cost_latency @@ -759,6 +760,8 @@ def load_us_cost_latency() -> dict: ) computed = model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M) stats = us_model.set_index("model") + for model_id in stats.index: + computed.setdefault(model_id, {}) for model_id, entry in computed.items(): row = stats.loc[model_id] if model_id in stats.index else None cost = entry.get("costPerHousehold") diff --git a/policybench/analysis.py b/policybench/analysis.py index 0ae3b331..88b07f40 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -1422,7 +1422,9 @@ def model_cost_latency( each household's calls summed, then the median across households — which is robust to the occasional rate-limit retry that inflates the mean. When a model's reconstructed cost is missing (a provider preview model not yet in - litellm's price map), it is filled from ``price_overrides``. + litellm's price map), it is filled from ``price_overrides``, but only from + recorded prompt and completion token counts. A model whose rows record + neither (Claude Fable 5's batch rows carry no usage) gets no cost, not $0. """ if predictions.empty or "model" not in predictions.columns: return {} @@ -1441,15 +1443,19 @@ def model_cost_latency( for _, row in usage.iterrows(): model = str(row["model"]) cost = row.get("total_cost_usd") - if (cost is None or pd.isna(cost)) and model in overrides: + prompt = row.get("prompt_tokens") + completion = row.get("completion_tokens") + if ( + (cost is None or pd.isna(cost)) + and model in overrides + and not (prompt is None or pd.isna(prompt)) + and not (completion is None or pd.isna(completion)) + ): price = overrides[model] - prompt = row.get("prompt_tokens") - completion = row.get("completion_tokens") - prompt = 0.0 if prompt is None or pd.isna(prompt) else float(prompt) - completion = ( - 0.0 if completion is None or pd.isna(completion) else float(completion) + cost = ( + float(prompt) * price["input"] / 1e6 + + float(completion) * price["output"] / 1e6 ) - cost = prompt * price["input"] / 1e6 + completion * price["output"] / 1e6 entry: dict[str, float] = {} if cost is not None and not pd.isna(cost): entry["costUsd"] = _clean_json_number(float(cost)) diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index df3b8dfb..865b7f27 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -1,10 +1,36 @@ -"""Tests for analysis.model_cost_latency (leaderboard cost + latency).""" +"""Tests for analysis.model_cost_latency (leaderboard cost + latency). + +Cost invariants, for every input (the property test executes them): +- costUsd and costPerHousehold are both present or both absent; +- a model has a cost exactly when one of its rows recorded total_cost_usd, or + it has a price override and its rows recorded prompt and completion token + counts; a cost is never built from a count nobody recorded, so a model + with no usage at all gets no cost rather than $0; +- a recorded cost is the sum of the recorded row costs and the override is + not consulted; a filled cost is the recorded token sums at the override's + prices; +- costPerHousehold is costUsd over the distinct households in the input. +The frozen-snapshot tests check the published figures: every model's usage +fields but Claude Fable 5's reproduce from the frozen predictions, and Fable +5's, which those rows do not record, are not computed at all. +""" + +import json +import math +from pathlib import Path import numpy as np import pandas as pd import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from policybench.analysis import model_cost_latency +from policybench.config import PRICE_OVERRIDES_PER_1M +from policybench.snapshot_payload import read_run_payload + +ROOT = Path(__file__).resolve().parents[1] +SNAPSHOT_DIR = ROOT / "paper/snapshot/20260501" def _row(model, scenario, cost, elapsed, prompt=100, completion=50): @@ -92,3 +118,172 @@ def test_all_null_latency_is_omitted_not_zero(): def test_empty_predictions_returns_empty(): assert model_cost_latency(pd.DataFrame()) == {} + + +def test_price_override_without_recorded_tokens_gives_no_cost(): + """Claude Fable 5's batch rows record no cost, tokens or latency. Its + price override has nothing to price, so it gets no cost, not $0.""" + preds = pd.DataFrame( + [ + _row("claude-fable-5", s, np.nan, np.nan, prompt=np.nan, completion=np.nan) + for s in ("s1", "s2") + ] + ) + overrides = {"claude-fable-5": PRICE_OVERRIDES_PER_1M["claude-fable-5"]} + assert model_cost_latency(preds, overrides) == {} + + +def test_price_override_without_recorded_tokens_keeps_the_rest(): + preds = pd.DataFrame( + [_row("m", "s1", np.nan, 5.0, prompt=np.nan, completion=np.nan)] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert out == {"m": {"latencySeconds": 5.0}} + + +@pytest.mark.parametrize( + "prompt, completion", [(np.nan, 2_000_000), (1_000_000, np.nan)] +) +def test_price_override_needs_both_token_counts(prompt, completion): + """Pricing one side would report part of the spend as all of it.""" + preds = pd.DataFrame( + [_row("m", "s1", np.nan, 5.0, prompt=prompt, completion=completion)] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert "costUsd" not in out["m"] + assert "costPerHousehold" not in out["m"] + + +def test_recorded_zero_cost_and_recorded_zero_tokens_stay_zero(): + """A $0 the rows record (a free model, or zero tokens) is a real $0.""" + preds = pd.DataFrame( + [ + _row("free", "s1", 0.0, 5.0), + _row("zero-tokens", "s1", np.nan, 5.0, prompt=0, completion=0), + ] + ) + out = model_cost_latency(preds, {"zero-tokens": {"input": 1.0, "output": 2.0}}) + assert out["free"]["costUsd"] == 0.0 + assert out["zero-tokens"]["costUsd"] == 0.0 + + +MODEL_NAMES = ("a", "b", "c") +SCENARIOS = ("s1", "s2", "s3", "s4") + +missing = st.just(float("nan")) +usage_row = st.fixed_dictionaries( + { + "model": st.sampled_from(MODEL_NAMES), + "scenario_id": st.sampled_from(SCENARIOS), + "total_cost_usd": st.one_of( + missing, + st.just(0.0), + st.floats(min_value=0, max_value=50, allow_subnormal=False), + ), + "prompt_tokens": st.one_of(missing, st.integers(0, 5_000_000).map(float)), + "completion_tokens": st.one_of(missing, st.integers(0, 5_000_000).map(float)), + "elapsed_seconds": st.one_of(missing, st.floats(min_value=0, max_value=600)), + } +) +price = st.fixed_dictionaries( + { + "input": st.floats(min_value=0, max_value=100, allow_subnormal=False), + "output": st.floats(min_value=0, max_value=100, allow_subnormal=False), + } +) + + +def _reference_cost(rows: list[dict], override: dict | None) -> float | None: + """The cost the invariants define, from the rows in plain Python.""" + recorded = [ + r["total_cost_usd"] for r in rows if not math.isnan(r["total_cost_usd"]) + ] + if recorded: + return math.fsum(recorded) + prompt = [r["prompt_tokens"] for r in rows if not math.isnan(r["prompt_tokens"])] + completion = [ + r["completion_tokens"] for r in rows if not math.isnan(r["completion_tokens"]) + ] + if override is None or not prompt or not completion: + return None + return ( + math.fsum(prompt) * override["input"] / 1e6 + + math.fsum(completion) * override["output"] / 1e6 + ) + + +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(usage_row, min_size=1, max_size=12), + overrides=st.dictionaries(st.sampled_from(MODEL_NAMES), price, max_size=3), +) +def test_cost_is_present_exactly_when_recorded_usage_supports_it(rows, overrides): + frame = pd.DataFrame(rows) + frame["variable"] = "v" + frame["prediction"] = 1.0 + frame["total_tokens"] = frame["prompt_tokens"] + frame["completion_tokens"] + out = model_cost_latency(frame, overrides) + households = frame["scenario_id"].nunique() + for model in sorted(set(frame["model"])): + entry = out.get(model, {}) + expected = _reference_cost( + [r for r in rows if r["model"] == model], overrides.get(model) + ) + assert ("costUsd" in entry) == ("costPerHousehold" in entry) + if expected is None: + assert "costUsd" not in entry + continue + assert entry["costUsd"] == pytest.approx(expected, rel=1e-12, abs=1e-12) + assert entry["costPerHousehold"] == pytest.approx( + entry["costUsd"] / households, rel=1e-12, abs=1e-12 + ) + + +# --- The frozen snapshot: published figures -------------------------------- + +USAGE_FIELDS = ("costUsd", "costPerHousehold", "latencySeconds", "totalTokens") + + +@pytest.fixture(scope="module") +def frozen(): + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + run = SNAPSHOT_DIR / "runs" / manifest["source_run_labels"]["us"] + predictions = pd.read_csv( + run / "predictions.csv.gz", + usecols=[ + "model", + "scenario_id", + "variable", + "prediction", + "error", + "total_cost_usd", + "elapsed_seconds", + "prompt_tokens", + "completion_tokens", + "total_tokens", + ], + low_memory=False, + ) + published = {row["model"]: row for row in read_run_payload(run)["modelStats"]} + return model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M), published + + +def test_frozen_usage_reproduces_every_published_model_but_fable_5(frozen): + """The published cost, latency and tokens of every model but Fable 5 are + model_cost_latency's on the frozen predictions, to the bit and with the + same fields present.""" + computed, published = frozen + assert len(published) > 1 + for model, row in published.items(): + if model == "claude-fable-5": + continue + assert {k: row[k] for k in USAGE_FIELDS if k in row} == computed[model], model + + +def test_frozen_fable_5_usage_is_carried_not_computed(frozen): + """Fable 5's rows record no usage, so nothing is computed for it (no $0 + cost); the published figures are the four the release drivers carry.""" + computed, published = frozen + assert "claude-fable-5" not in computed + fable = published["claude-fable-5"] + assert all(fable[k] > 0 for k in USAGE_FIELDS) From 75855c0b786d9593dadfcd909f89e4f2b6a6e3d4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:11:00 -0400 Subject: [PATCH 2/5] Test the release drivers' Fable 5 carry with no usage in the export The export now reports none of Fable 5's usage, so every driver test's fake exporter drops all four fields instead of reporting $0. The finish tests check that the carried row keeps the published key order. test_carried_usage.py pins the carry's precondition (the published row ends with the four fields, in CARRIED_USAGE's order). A property test runs release_20261006's carry on both exporter behaviours. A slow local test rebuilds the committed payload byte for byte through the real export and the carry. Co-Authored-By: Claude Opus 5.5 --- tests/test_carried_usage.py | 153 ++++++++++++++++++++++++++++++++ tests/test_finish_adds0928.py | 2 +- tests/test_finish_argon.py | 8 +- tests/test_finish_gpt61sol.py | 5 +- tests/test_finish_haiku55.py | 8 +- tests/test_finish_musewonder.py | 8 +- tests/test_freeze_argon.py | 5 +- tests/test_freeze_gpt61sol.py | 5 +- tests/test_freeze_haiku55.py | 5 +- tests/test_freeze_musewonder.py | 5 +- 10 files changed, 187 insertions(+), 17 deletions(-) create mode 100644 tests/test_carried_usage.py diff --git a/tests/test_carried_usage.py b/tests/test_carried_usage.py new file mode 100644 index 00000000..0ef36a1f --- /dev/null +++ b/tests/test_carried_usage.py @@ -0,0 +1,153 @@ +"""Fable 5's carried usage: the export reports none of it, the carry restores it. + +Claude Fable 5 ran through the Anthropic batch adapter, and its rows record no +cost, token or latency fields. So analysis.model_cost_latency computes none of +its usage, and export_full_run reports none (it reported a $0 cost before +model_cost_latency stopped pricing unrecorded tokens). Every release driver +carries the four published fields from its base release (CARRIED_USAGE). It +then gates on serialized bytes, so key order counts as well as values. + +Invariants: +- every driver carries the same four fields, in one order; +- the published Fable 5 row ends with those four fields in that order, so + assigning them to an exported row that lacks them reproduces the published + key order; +- release_20261006.export_payload's carry restores the published row byte + for byte whether the export omits Fable 5's usage or reports a $0 cost, and + it touches no other model; +- (slow, local) the committed snapshot's payload rebuilds byte for byte from + its committed inputs through export_full_run and that carry. +""" + +from __future__ import annotations + +import gzip +import importlib +import json +import shutil +import sys +import tempfile +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import release_20261006 as release # noqa: E402 + +from policybench.snapshot_payload import ( # noqa: E402 + read_run_payload, + read_run_payload_text, +) + +FABLE = "claude-fable-5" +CARRIED = ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") +FROZEN = ROOT / release.SNAPSHOT_RUN +DRIVERS = ( + "release_20261006", + "release_20261013b", + "finish_gpt61sol", + "finish_haiku55", + "finish_musewonder", + "finish_argon", +) + + +@pytest.mark.parametrize("name", DRIVERS) +def test_every_driver_carries_fable_5s_four_fields_in_one_order(name): + assert importlib.import_module(name).CARRIED_USAGE == {FABLE: CARRIED} + + +def test_the_published_fable_5_row_ends_with_the_carried_fields(): + """The precondition for the carry to reproduce the published bytes when + the export reports none of Fable 5's usage.""" + rows = read_run_payload(FROZEN)["modelStats"] + fable = next(row for row in rows if row["model"] == FABLE) + assert tuple(fable)[-len(CARRIED) :] == CARRIED + assert all(fable[key] > 0 for key in CARRIED) + + +STAT_KEYS = ("condition", "score", "exact", "n", "accuracy", "aggregateScore") + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + leading=st.lists(st.sampled_from(STAT_KEYS), unique=True), + usage=st.tuples( + st.floats(min_value=0.01, max_value=1e4), + st.floats(min_value=1e-4, max_value=100), + st.integers(min_value=1, max_value=10**9), + st.floats(min_value=0.5, max_value=3600), + ), + exporter=st.sampled_from(["omits", "zero"]), +) +def test_the_carry_restores_the_published_row_whichever_way_the_export_reports_it( + monkeypatch, leading, usage, exporter +): + """release_20261006.export_payload's carry, run on a stubbed export: the + carried Fable 5 row serializes as the published one, and the other model's + row is the export's.""" + import policybench.dashboard_schema + import policybench.full_run_export + + published = {"model": FABLE, **{key: 1.5 for key in leading}} + published.update(zip(CARRIED, usage)) + exported_fable = {"model": FABLE, **{key: 1.5 for key in leading}} + if exporter == "zero": + exported_fable.update(costUsd=0.0, costPerHousehold=0.0) + other = {"model": "other", "exact": 50.0, "costUsd": 0.0, "costPerHousehold": 0.0} + base = {"countries": {"us": {"modelStats": [published, dict(other)]}}} + export = {"countries": {"us": {"modelStats": [exported_fable, dict(other)]}}} + monkeypatch.setattr( + policybench.full_run_export, "export_full_run", lambda *a, **k: export + ) + monkeypatch.setattr( + policybench.dashboard_schema, "validate_dashboard_payload", lambda *a, **k: [] + ) + with tempfile.TemporaryDirectory() as scratch: + bundle = Path(scratch) / "bundle" + for rel in release.EXPORT_INPUTS: + (bundle / rel).parent.mkdir(parents=True, exist_ok=True) + (bundle / rel).write_bytes(b"") + payload = release.export_payload(bundle, base) + fable, rest = payload["countries"]["us"]["modelStats"] + assert json.dumps(fable) == json.dumps(published) + assert json.dumps(rest) == json.dumps(other) + + +@pytest.mark.slow +def test_the_committed_payload_rebuilds_through_export_and_the_carry(monkeypatch): + """Local only, about a minute: export_full_run on the committed snapshot's + inputs reports none of Fable 5's usage, and with release_20261006's carry + the US payload is the committed one byte for byte.""" + from policybench.full_run_export import export_full_run + + monkeypatch.chdir(ROOT) + committed = read_run_payload_text(FROZEN) + base = {"countries": {"us": json.loads(committed)}} + with tempfile.TemporaryDirectory() as scratch: + bundle = Path(scratch) / release.RUN + (bundle / "us").mkdir(parents=True) + (bundle / "annotations").mkdir() + for name in (*release.REFERENCE_FILES, release.EXCLUSIONS): + shutil.copyfile(FROZEN / name, bundle / "us" / name) + with gzip.open(FROZEN / "predictions.csv.gz", "rb") as source: + (bundle / "us" / "predictions.csv").write_bytes(source.read()) + for name in release.ANNOTATION_CSVS: + shutil.copyfile( + ROOT / release.ANNOTATIONS / name, bundle / "annotations" / name + ) + raw = export_full_run(bundle, countries=["us"], skip_app_data=True) + payload = release.export_payload(bundle, base) + raw_fable = next( + row for row in raw["countries"]["us"]["modelStats"] if row["model"] == FABLE + ) + assert not set(CARRIED) & set(raw_fable) + assert json.dumps(payload["countries"]["us"], allow_nan=False) == committed diff --git a/tests/test_finish_adds0928.py b/tests/test_finish_adds0928.py index dbc22a0d..d3b89328 100644 --- a/tests/test_finish_adds0928.py +++ b/tests/test_finish_adds0928.py @@ -546,7 +546,7 @@ def test_strict_export_recombines_to_freeze_bytes_and_binds_evidence( for model in driver.MODELS.values() ] for field in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): - stats[41][field] = None + del stats[41][field] payload = {"countries": {"us": {"modelStats": stats}}} live = {"countries": {"us": {"modelStats": incumbents}}} gate_calls = [] diff --git a/tests/test_finish_argon.py b/tests/test_finish_argon.py index 3cff93f9..80640d06 100644 --- a/tests/test_finish_argon.py +++ b/tests/test_finish_argon.py @@ -2829,8 +2829,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [ _stat(NEW, 65.0, costUsd=0.13), _stat(OTHER, 61.0, costUsd=0.11), @@ -2924,6 +2925,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_base_record_explains( assert {row["n"] for row in rows} == {1935} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index 2ed39158..1fdfa34a 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -1893,8 +1893,9 @@ def _exported(incumbents): """What export_full_run returns: Fable's usage is gone, the addition is new.""" stats = copy.deepcopy(incumbents) fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=2.0)] diff --git a/tests/test_finish_haiku55.py b/tests/test_finish_haiku55.py index 4f798b1f..3c65865e 100644 --- a/tests/test_finish_haiku55.py +++ b/tests/test_finish_haiku55.py @@ -3474,8 +3474,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=0.13)] @@ -3567,6 +3568,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_ten_records_explain(s assert {row["n"] for row in rows} == {1910} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_finish_musewonder.py b/tests/test_finish_musewonder.py index b0b2e0d2..aff002e3 100644 --- a/tests/test_finish_musewonder.py +++ b/tests/test_finish_musewonder.py @@ -2788,8 +2788,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=0.13)] @@ -2881,6 +2882,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_base_record_explains( assert {row["n"] for row in rows} == {1926} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_freeze_argon.py b/tests/test_freeze_argon.py index 54191b8a..514bdb96 100644 --- a/tests/test_freeze_argon.py +++ b/tests/test_freeze_argon.py @@ -1000,8 +1000,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board diff --git a/tests/test_freeze_gpt61sol.py b/tests/test_freeze_gpt61sol.py index a2f6efa2..d5ed8ae7 100644 --- a/tests/test_freeze_gpt61sol.py +++ b/tests/test_freeze_gpt61sol.py @@ -444,8 +444,9 @@ def staged_board(freeze_preflight, monkeypatch, rebuilds, exports): for row in exported["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] exports["board"] = exported def export_full_run(run_dir, *, countries, skip_app_data): diff --git a/tests/test_freeze_haiku55.py b/tests/test_freeze_haiku55.py index c899945d..dab82af1 100644 --- a/tests/test_freeze_haiku55.py +++ b/tests/test_freeze_haiku55.py @@ -910,8 +910,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board diff --git a/tests/test_freeze_musewonder.py b/tests/test_freeze_musewonder.py index 6c30eda9..44d1df31 100644 --- a/tests/test_freeze_musewonder.py +++ b/tests/test_freeze_musewonder.py @@ -965,8 +965,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board From c4d2b79a05df880e4375149075282e4010f3cf40 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:16:46 -0400 Subject: [PATCH 3/5] State only what Fable 5's rows record, and stop saying the export reports $0 The finish drivers' comments said export_full_run reports $0 for Fable 5; it now reports none of its usage. The new docstrings no longer say why Fable 5's rows lack usage (PR #228's audit finds the batch-adapter account wrong); they say only that the frozen board rows record none. The paper's existing comment block is left as it was, and the seeding loop carries its own comment. Co-Authored-By: Claude Opus 5.5 --- paper/index.qmd | 5 +++-- policybench/analysis.py | 3 ++- scripts/finish_argon.py | 2 +- scripts/finish_gpt61sol.py | 2 +- scripts/finish_haiku55.py | 2 +- scripts/finish_musewonder.py | 2 +- tests/test_carried_usage.py | 6 +++--- tests/test_cost_latency.py | 2 +- 8 files changed, 13 insertions(+), 11 deletions(-) diff --git a/paper/index.qmd b/paper/index.qmd index 63680c56..1ea29073 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -748,8 +748,7 @@ def load_us_cost_latency() -> dict: # through the same model_cost_latency function as the leaderboard pipeline, # then are reconciled against the frozen # modelStats: Claude Fable 5's usage is recorded at the run level (its - # per-row usage columns are empty, so model_cost_latency returns nothing - # for it), and batch or asynchronous waves log + # per-row usage columns are empty), and batch or asynchronous waves log # submission times rather than generation time, so a zero or sub-second # median is treated as unmeasured and falls back to modelStats or "—". from policybench.analysis import model_cost_latency @@ -760,6 +759,8 @@ def load_us_cost_latency() -> dict: ) computed = model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M) stats = us_model.set_index("model") + # model_cost_latency gives no entry to a model whose rows record no usage + # (Claude Fable 5), so seed every board model to reach its fallback. for model_id in stats.index: computed.setdefault(model_id, {}) for model_id, entry in computed.items(): diff --git a/policybench/analysis.py b/policybench/analysis.py index 88b07f40..53db746a 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -1424,7 +1424,8 @@ def model_cost_latency( model's reconstructed cost is missing (a provider preview model not yet in litellm's price map), it is filled from ``price_overrides``, but only from recorded prompt and completion token counts. A model whose rows record - neither (Claude Fable 5's batch rows carry no usage) gets no cost, not $0. + neither (Claude Fable 5's frozen board rows record no usage) gets no cost, + not $0. """ if predictions.empty or "model" not in predictions.columns: return {} diff --git a/scripts/finish_argon.py b/scripts/finish_argon.py index 3c5027df..0aff6028 100644 --- a/scripts/finish_argon.py +++ b/scripts/finish_argon.py @@ -213,7 +213,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py index 982ae6f6..505ad8a4 100644 --- a/scripts/finish_gpt61sol.py +++ b/scripts/finish_gpt61sol.py @@ -175,7 +175,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_haiku55.py b/scripts/finish_haiku55.py index 233f09ec..f62872f3 100644 --- a/scripts/finish_haiku55.py +++ b/scripts/finish_haiku55.py @@ -193,7 +193,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py index 16fe4e99..0e2e234b 100644 --- a/scripts/finish_musewonder.py +++ b/scripts/finish_musewonder.py @@ -203,7 +203,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/tests/test_carried_usage.py b/tests/test_carried_usage.py index 0ef36a1f..df3e4e6b 100644 --- a/tests/test_carried_usage.py +++ b/tests/test_carried_usage.py @@ -1,8 +1,8 @@ """Fable 5's carried usage: the export reports none of it, the carry restores it. -Claude Fable 5 ran through the Anthropic batch adapter, and its rows record no -cost, token or latency fields. So analysis.model_cost_latency computes none of -its usage, and export_full_run reports none (it reported a $0 cost before +Claude Fable 5's frozen board rows record no cost, token or latency fields. +So analysis.model_cost_latency computes none of its usage, and +export_full_run reports none (it reported a $0 cost before model_cost_latency stopped pricing unrecorded tokens). Every release driver carries the four published fields from its base release (CARRIED_USAGE). It then gates on serialized bytes, so key order counts as well as values. diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index 865b7f27..11be90b9 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -121,7 +121,7 @@ def test_empty_predictions_returns_empty(): def test_price_override_without_recorded_tokens_gives_no_cost(): - """Claude Fable 5's batch rows record no cost, tokens or latency. Its + """Claude Fable 5's frozen board rows record no cost, tokens or latency. Its price override has nothing to price, so it gets no cost, not $0.""" preds = pd.DataFrame( [ From 460cad157695f5fadc2aa3dbdb30402a20465a72 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:24:41 -0400 Subject: [PATCH 4/5] Give gpt61sol's drift-gate property Fable 5 cases that fit the new export Its two cost mutations popped or negated the exported Fable 5 row's costUsd, which the export no longer reports, so at target 44 they raised KeyError. There they now model an export that reports a cost after all (the old $0, or -0.0), which the carry must overwrite with no drift. @example pins both cases, plus a reordered Fable row, so the property no longer depends on Hypothesis sampling them. Co-Authored-By: Claude Opus 5.5 --- tests/test_finish_gpt61sol.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index 1fdfa34a..f54c8858 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -21,7 +21,7 @@ import pandas as pd import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) @@ -2713,6 +2713,9 @@ def test_export_writes_no_receipt_when_the_record_disagrees(exporting, monkeypat target=st.integers(min_value=0, max_value=44), mutation=st.sampled_from(sorted(MUTATIONS)), ) +@example(target=44, mutation="zero_cost_to_missing") +@example(target=44, mutation="zero_to_negative_zero") +@example(target=44, mutation="key_order") def test_the_no_drift_gate_refuses_any_change_to_any_incumbent( exporting, target, mutation ): @@ -2720,15 +2723,19 @@ def test_the_no_drift_gate_refuses_any_change_to_any_incumbent( stage, _, run, _ = exporting incumbents = _incumbents() stats = _exported(incumbents) - MUTATIONS[mutation](stats[target]) (stage / "data-board46.json").unlink(missing_ok=True) - # Fable's usage values are carried over from the base (the exporter cannot - # recompute them), so a changed value there is replaced by the released - # one; a missing key comes back out of order and is still refused. - if target == 44 and mutation == "zero_to_negative_zero": + # Fable's usage is carried over from the base (the exporter reports none + # of it), so its export row has no cost for these two to change. Their + # analogue there is an export that reports a cost after all, as the + # exporter did ($0) before model_cost_latency stopped pricing unrecorded + # tokens: the carry replaces it with the released value, so no drift. + if target == 44 and mutation in ("zero_cost_to_missing", "zero_to_negative_zero"): + zero = -0.0 if mutation == "zero_to_negative_zero" else 0.0 + stats[target].update(costUsd=zero, costPerHousehold=zero) run(stats, incumbents, early=True) assert json.dumps(stats[target]) == json.dumps(incumbents[target]) return + MUTATIONS[mutation](stats[target]) with pytest.raises(SystemExit, match="incumbent modelStats drift"): run(stats, incumbents, early=True) assert not (stage / "data-board46.json").exists() From 473ebb3d865ebf0158249c6677880113a1591145 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:41:07 -0400 Subject: [PATCH 5/5] Price only rows that record both counts, and guard the paper's fallback Review r1 executed a counterexample: one row records only a prompt count and another only a completion count. The model-level sums were then both recorded, so the fill still priced each missing count as $0. The fill now sums only rows that record both counts, as eval_no_tools prices no row missing either. Frozen output is bit-identical: every row of the two filled models records both counts. Also: - tests for split and mixed rows; - the property generator draws whole uncosted and untokened models, so the fill and the no-usage branches each get tens of cases per run; - a render-time assertion that every model the board prices shows a cost in the paper (it fails with ['claude-fable-5'] without the seeding loop); - docs/gpt61sol/design.md's "$0" sentence now in the past tense. Co-Authored-By: Claude Opus 5.5 --- docs/gpt61sol/design.md | 3 +- paper/index.qmd | 9 +++++ policybench/analysis.py | 23 +++++++----- tests/test_cost_latency.py | 74 +++++++++++++++++++++++++++++++------- 4 files changed, 87 insertions(+), 22 deletions(-) diff --git a/docs/gpt61sol/design.md b/docs/gpt61sol/design.md index 72607196..0ef81078 100644 --- a/docs/gpt61sol/design.md +++ b/docs/gpt61sol/design.md @@ -103,7 +103,8 @@ the stage's binding against it. The exporter still needs the Fable 5 usage carry-over. Fable 5 ran through the Anthropic batch adapter, and its committed rows carry no cost, token or latency -fields, so `export_full_run` reports `costUsd` $0 and omits the rest. The live +fields, so `export_full_run` reported `costUsd` $0 and omitted the rest (since +#230 it reports none of them). The live payload has $54.11, 3,850,174 tokens and 97.68 s. The driver copies those four values from the base, and the drift gate then checks every field and the key order. Ox Alpha's `costUsd` of $0.0 comes out of the exporter unchanged, so it diff --git a/paper/index.qmd b/paper/index.qmd index 1ea29073..f7ef1de0 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -783,6 +783,15 @@ def load_us_cost_latency() -> dict: us_cost_latency = load_us_cost_latency() +# Every model the board prices shows a cost. One whose frozen rows record no +# usage (Claude Fable 5) takes its modelStats figure, not a dash. +_unpriced = [ + row["model"] + for _, row in us_model.iterrows() + if (row.get("costPerHousehold") or 0) > 0 + and us_cost_latency.get(row["model"], {}).get("costPerHousehold") is None +] +assert not _unpriced, _unpriced def cost_latency_table() -> pd.DataFrame: diff --git a/policybench/analysis.py b/policybench/analysis.py index 53db746a..e0398c63 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -1422,10 +1422,11 @@ def model_cost_latency( each household's calls summed, then the median across households — which is robust to the occasional rate-limit retry that inflates the mean. When a model's reconstructed cost is missing (a provider preview model not yet in - litellm's price map), it is filled from ``price_overrides``, but only from - recorded prompt and completion token counts. A model whose rows record - neither (Claude Fable 5's frozen board rows record no usage) gets no cost, - not $0. + litellm's price map), it is filled from ``price_overrides`` by pricing the + rows that recorded both prompt and completion token counts. A row missing + either count adds nothing, as ``eval_no_tools._reconstruct_token_cost`` + prices no such row, and a model with no row to price (Claude Fable 5's + frozen board rows record no usage) gets no cost, not $0. """ if predictions.empty or "model" not in predictions.columns: return {} @@ -1440,19 +1441,25 @@ def model_cost_latency( .median() ) overrides = price_overrides or {} + token_columns = ["prompt_tokens", "completion_tokens"] + priceable = pd.DataFrame(columns=token_columns) + if set(token_columns) <= set(predictions.columns): + priceable = ( + predictions.dropna(subset=token_columns) + .groupby("model")[token_columns] + .sum() + ) out: dict[str, dict[str, float]] = {} for _, row in usage.iterrows(): model = str(row["model"]) cost = row.get("total_cost_usd") - prompt = row.get("prompt_tokens") - completion = row.get("completion_tokens") if ( (cost is None or pd.isna(cost)) and model in overrides - and not (prompt is None or pd.isna(prompt)) - and not (completion is None or pd.isna(completion)) + and row["model"] in priceable.index ): price = overrides[model] + prompt, completion = priceable.loc[row["model"], token_columns] cost = ( float(prompt) * price["input"] / 1e6 + float(completion) * price["output"] / 1e6 diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index 11be90b9..45c6eec4 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -3,12 +3,13 @@ Cost invariants, for every input (the property test executes them): - costUsd and costPerHousehold are both present or both absent; - a model has a cost exactly when one of its rows recorded total_cost_usd, or - it has a price override and its rows recorded prompt and completion token - counts; a cost is never built from a count nobody recorded, so a model - with no usage at all gets no cost rather than $0; + it has a price override and one of its rows recorded both prompt and + completion token counts; a row missing either count is not priced, so no + cost rests on a count nobody recorded, and a model with no usage at all + gets no cost rather than $0; - a recorded cost is the sum of the recorded row costs and the override is - not consulted; a filled cost is the recorded token sums at the override's - prices; + not consulted; a filled cost is the override's prices applied to the token + sums of the rows that recorded both counts; - costPerHousehold is costUsd over the distinct households in the input. The frozen-snapshot tests check the published figures: every model's usage fields but Claude Fable 5's reproduce from the frozen predictions, and Fable @@ -154,6 +155,33 @@ def test_price_override_needs_both_token_counts(prompt, completion): assert "costPerHousehold" not in out["m"] +def test_price_override_needs_both_counts_on_one_row(): + """Counts split across rows price no row: each row is missing one side.""" + preds = pd.DataFrame( + [ + _row("m", "s1", np.nan, 5.0, prompt=1_000_000, completion=np.nan), + _row("m", "s2", np.nan, 5.0, prompt=np.nan, completion=1_000_000), + ] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert "costUsd" not in out["m"] + + +def test_price_override_prices_only_rows_with_both_counts(): + """A row missing a count adds nothing; it is not priced at $0 for it.""" + preds = pd.DataFrame( + [ + _row("m", "s1", np.nan, 5.0, prompt=1_000_000, completion=2_000_000), + _row("m", "s2", np.nan, 5.0, prompt=3_000_000, completion=np.nan), + _row("m", "s3", np.nan, 5.0, prompt=np.nan, completion=np.nan), + ] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + # s1 only: 1M * $1/1M + 2M * $2/1M + assert out["m"]["costUsd"] == pytest.approx(5.0) + assert out["m"]["costPerHousehold"] == pytest.approx(5.0 / 3) + + def test_recorded_zero_cost_and_recorded_zero_tokens_stay_zero(): """A $0 the rows record (a free model, or zero tokens) is a real $0.""" preds = pd.DataFrame( @@ -200,24 +228,44 @@ def _reference_cost(rows: list[dict], override: dict | None) -> float | None: ] if recorded: return math.fsum(recorded) - prompt = [r["prompt_tokens"] for r in rows if not math.isnan(r["prompt_tokens"])] - completion = [ - r["completion_tokens"] for r in rows if not math.isnan(r["completion_tokens"]) + priced = [ + r + for r in rows + if not math.isnan(r["prompt_tokens"]) and not math.isnan(r["completion_tokens"]) ] - if override is None or not prompt or not completion: + if override is None or not priced: return None return ( - math.fsum(prompt) * override["input"] / 1e6 - + math.fsum(completion) * override["output"] / 1e6 + math.fsum(r["prompt_tokens"] for r in priced) * override["input"] / 1e6 + + math.fsum(r["completion_tokens"] for r in priced) * override["output"] / 1e6 ) @settings(max_examples=300, deadline=None) @given( - rows=st.lists(usage_row, min_size=1, max_size=12), + drawn=st.lists(usage_row, min_size=1, max_size=12), overrides=st.dictionaries(st.sampled_from(MODEL_NAMES), price, max_size=3), + uncosted=st.sets(st.sampled_from(MODEL_NAMES)), + untokened=st.sets(st.sampled_from(MODEL_NAMES)), ) -def test_cost_is_present_exactly_when_recorded_usage_supports_it(rows, overrides): +def test_cost_is_present_exactly_when_recorded_usage_supports_it( + drawn, overrides, uncosted, untokened +): + # Whole models with no recorded cost, or (like Fable 5) no token counts, + # so the fill and the no-usage cases are drawn often. + nan = float("nan") + rows = [ + { + **row, + **({"total_cost_usd": nan} if row["model"] in uncosted else {}), + **( + {"prompt_tokens": nan, "completion_tokens": nan} + if row["model"] in untokened + else {} + ), + } + for row in drawn + ] frame = pd.DataFrame(rows) frame["variable"] = "v" frame["prediction"] = 1.0