Give a model with no recorded tokens no cost, not $0

docs/gpt61sol/design.md

@@ -103,7 +103,8 @@ the stage's binding against it.
 
 The exporter still needs the Fable 5 usage carry-over. Fable 5 ran through the
 Anthropic batch adapter, and its committed rows carry no cost, token or latency
-fields, so `export_full_run` reports `costUsd` $0 and omits the rest. The live
+fields, so `export_full_run` reported `costUsd` $0 and omitted the rest (since
+#230 it reports none of them). The live
 payload has $54.11, 3,850,174 tokens and 97.68 s. The driver copies those four
 values from the base, and the drift gate then checks every field and the key
 order. Ox Alpha's `costUsd` of $0.0 comes out of the exporter unchanged, so it

paper/index.qmd

@@ -759,6 +759,10 @@ def load_us_cost_latency() -> dict:
     )
     computed = model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M)
     stats = us_model.set_index("model")
+    # model_cost_latency gives no entry to a model whose rows record no usage
+    # (Claude Fable 5), so seed every board model to reach its fallback.
+    for model_id in stats.index:
+        computed.setdefault(model_id, {})
     for model_id, entry in computed.items():
         row = stats.loc[model_id] if model_id in stats.index else None
         cost = entry.get("costPerHousehold")
@@ -779,6 +783,15 @@ def load_us_cost_latency() -> dict:
 
 
 us_cost_latency = load_us_cost_latency()
+# Every model the board prices shows a cost. One whose frozen rows record no
+# usage (Claude Fable 5) takes its modelStats figure, not a dash.
+_unpriced = [
+    row["model"]
+    for _, row in us_model.iterrows()
+    if (row.get("costPerHousehold") or 0) > 0
+    and us_cost_latency.get(row["model"], {}).get("costPerHousehold") is None
+]
+assert not _unpriced, _unpriced
 
 
 def cost_latency_table() -> pd.DataFrame:

policybench/analysis.py

@@ -1422,7 +1422,11 @@ def model_cost_latency(
     each household's calls summed, then the median across households — which is
     robust to the occasional rate-limit retry that inflates the mean. When a
     model's reconstructed cost is missing (a provider preview model not yet in
-    litellm's price map), it is filled from ``price_overrides``.
+    litellm's price map), it is filled from ``price_overrides`` by pricing the
+    rows that recorded both prompt and completion token counts. A row missing
+    either count adds nothing, as ``eval_no_tools._reconstruct_token_cost``
+    prices no such row, and a model with no row to price (Claude Fable 5's
+    frozen board rows record no usage) gets no cost, not $0.
     """
     if predictions.empty or "model" not in predictions.columns:
         return {}
@@ -1437,19 +1441,29 @@ def model_cost_latency(
             .median()
         )
     overrides = price_overrides or {}
+    token_columns = ["prompt_tokens", "completion_tokens"]
+    priceable = pd.DataFrame(columns=token_columns)
+    if set(token_columns) <= set(predictions.columns):
+        priceable = (
+            predictions.dropna(subset=token_columns)
+            .groupby("model")[token_columns]
+            .sum()
+        )
     out: dict[str, dict[str, float]] = {}
     for _, row in usage.iterrows():
         model = str(row["model"])
         cost = row.get("total_cost_usd")
-        if (cost is None or pd.isna(cost)) and model in overrides:
+        if (
+            (cost is None or pd.isna(cost))
+            and model in overrides
+            and row["model"] in priceable.index
+        ):
             price = overrides[model]
-            prompt = row.get("prompt_tokens")
-            completion = row.get("completion_tokens")
-            prompt = 0.0 if prompt is None or pd.isna(prompt) else float(prompt)
-            completion = (
-                0.0 if completion is None or pd.isna(completion) else float(completion)
+            prompt, completion = priceable.loc[row["model"], token_columns]
+            cost = (
+                float(prompt) * price["input"] / 1e6
+                + float(completion) * price["output"] / 1e6
             )
-            cost = prompt * price["input"] / 1e6 + completion * price["output"] / 1e6
         entry: dict[str, float] = {}
         if cost is not None and not pd.isna(cost):
             entry["costUsd"] = _clean_json_number(float(cost))

scripts/finish_argon.py

@@ -213,7 +213,7 @@
 PINNED_INPUTS = ("predictions.csv", "run_state.json")
 # Incumbent usage the exporter cannot recompute from committed predictions:
 # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost,
-# token or latency fields, so export_full_run reports $0 and omits the rest.
+# token or latency fields, so export_full_run reports none of its usage.
 # The released values are carried over; the drift gate then checks every field.
 CARRIED_USAGE = {
     "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds")

scripts/finish_gpt61sol.py

@@ -175,7 +175,7 @@
 PINNED_INPUTS = ("predictions.csv", "run_state.json")
 # Incumbent usage the exporter cannot recompute from committed predictions:
 # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost,
-# token or latency fields, so export_full_run reports $0 and omits the rest.
+# token or latency fields, so export_full_run reports none of its usage.
 # The released values are carried over; the drift gate then checks every field.
 CARRIED_USAGE = {
     "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds")

scripts/finish_haiku55.py

@@ -193,7 +193,7 @@
 PINNED_INPUTS = ("predictions.csv", "run_state.json")
 # Incumbent usage the exporter cannot recompute from committed predictions:
 # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost,
-# token or latency fields, so export_full_run reports $0 and omits the rest.
+# token or latency fields, so export_full_run reports none of its usage.
 # The released values are carried over; the drift gate then checks every field.
 CARRIED_USAGE = {
     "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds")

scripts/finish_musewonder.py

@@ -203,7 +203,7 @@
 PINNED_INPUTS = ("predictions.csv", "run_state.json")
 # Incumbent usage the exporter cannot recompute from committed predictions:
 # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost,
-# token or latency fields, so export_full_run reports $0 and omits the rest.
+# token or latency fields, so export_full_run reports none of its usage.
 # The released values are carried over; the drift gate then checks every field.
 CARRIED_USAGE = {
     "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds")

tests/test_carried_usage.py

@@ -0,0 +1,153 @@
+"""Fable 5's carried usage: the export reports none of it, the carry restores it.
+
+Claude Fable 5's frozen board rows record no cost, token or latency fields.
+So analysis.model_cost_latency computes none of its usage, and
+export_full_run reports none (it reported a $0 cost before
+model_cost_latency stopped pricing unrecorded tokens). Every release driver
+carries the four published fields from its base release (CARRIED_USAGE). It
+then gates on serialized bytes, so key order counts as well as values.
+
+Invariants:
+- every driver carries the same four fields, in one order;
+- the published Fable 5 row ends with those four fields in that order, so
+  assigning them to an exported row that lacks them reproduces the published
+  key order;
+- release_20261006.export_payload's carry restores the published row byte
+  for byte whether the export omits Fable 5's usage or reports a $0 cost, and
+  it touches no other model;
+- (slow, local) the committed snapshot's payload rebuilds byte for byte from
+  its committed inputs through export_full_run and that carry.
+"""
+
+from __future__ import annotations
+
+import gzip
+import importlib
+import json
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+from hypothesis import HealthCheck, given, settings
+from hypothesis import strategies as st
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "scripts"))
+
+import release_20261006 as release  # noqa: E402
+
+from policybench.snapshot_payload import (  # noqa: E402
+    read_run_payload,
+    read_run_payload_text,
+)
+
+FABLE = "claude-fable-5"
+CARRIED = ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds")
+FROZEN = ROOT / release.SNAPSHOT_RUN
+DRIVERS = (
+    "release_20261006",
+    "release_20261013b",
+    "finish_gpt61sol",
+    "finish_haiku55",
+    "finish_musewonder",
+    "finish_argon",
+)
+
+
+@pytest.mark.parametrize("name", DRIVERS)
+def test_every_driver_carries_fable_5s_four_fields_in_one_order(name):
+    assert importlib.import_module(name).CARRIED_USAGE == {FABLE: CARRIED}
+
+
+def test_the_published_fable_5_row_ends_with_the_carried_fields():
+    """The precondition for the carry to reproduce the published bytes when
+    the export reports none of Fable 5's usage."""
+    rows = read_run_payload(FROZEN)["modelStats"]
+    fable = next(row for row in rows if row["model"] == FABLE)
+    assert tuple(fable)[-len(CARRIED) :] == CARRIED
+    assert all(fable[key] > 0 for key in CARRIED)
+
+
+STAT_KEYS = ("condition", "score", "exact", "n", "accuracy", "aggregateScore")
+
+
+@settings(
+    max_examples=60,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(
+    leading=st.lists(st.sampled_from(STAT_KEYS), unique=True),
+    usage=st.tuples(
+        st.floats(min_value=0.01, max_value=1e4),
+        st.floats(min_value=1e-4, max_value=100),
+        st.integers(min_value=1, max_value=10**9),
+        st.floats(min_value=0.5, max_value=3600),
+    ),
+    exporter=st.sampled_from(["omits", "zero"]),
+)
+def test_the_carry_restores_the_published_row_whichever_way_the_export_reports_it(
+    monkeypatch, leading, usage, exporter
+):
+    """release_20261006.export_payload's carry, run on a stubbed export: the
+    carried Fable 5 row serializes as the published one, and the other model's
+    row is the export's."""
+    import policybench.dashboard_schema
+    import policybench.full_run_export
+
+    published = {"model": FABLE, **{key: 1.5 for key in leading}}
+    published.update(zip(CARRIED, usage))
+    exported_fable = {"model": FABLE, **{key: 1.5 for key in leading}}
+    if exporter == "zero":
+        exported_fable.update(costUsd=0.0, costPerHousehold=0.0)
+    other = {"model": "other", "exact": 50.0, "costUsd": 0.0, "costPerHousehold": 0.0}
+    base = {"countries": {"us": {"modelStats": [published, dict(other)]}}}
+    export = {"countries": {"us": {"modelStats": [exported_fable, dict(other)]}}}
+    monkeypatch.setattr(
+        policybench.full_run_export, "export_full_run", lambda *a, **k: export
+    )
+    monkeypatch.setattr(
+        policybench.dashboard_schema, "validate_dashboard_payload", lambda *a, **k: []
+    )
+    with tempfile.TemporaryDirectory() as scratch:
+        bundle = Path(scratch) / "bundle"
+        for rel in release.EXPORT_INPUTS:
+            (bundle / rel).parent.mkdir(parents=True, exist_ok=True)
+            (bundle / rel).write_bytes(b"")
+        payload = release.export_payload(bundle, base)
+    fable, rest = payload["countries"]["us"]["modelStats"]
+    assert json.dumps(fable) == json.dumps(published)
+    assert json.dumps(rest) == json.dumps(other)
+
+
+@pytest.mark.slow
+def test_the_committed_payload_rebuilds_through_export_and_the_carry(monkeypatch):
+    """Local only, about a minute: export_full_run on the committed snapshot's
+    inputs reports none of Fable 5's usage, and with release_20261006's carry
+    the US payload is the committed one byte for byte."""
+    from policybench.full_run_export import export_full_run
+
+    monkeypatch.chdir(ROOT)
+    committed = read_run_payload_text(FROZEN)
+    base = {"countries": {"us": json.loads(committed)}}
+    with tempfile.TemporaryDirectory() as scratch:
+        bundle = Path(scratch) / release.RUN
+        (bundle / "us").mkdir(parents=True)
+        (bundle / "annotations").mkdir()
+        for name in (*release.REFERENCE_FILES, release.EXCLUSIONS):
+            shutil.copyfile(FROZEN / name, bundle / "us" / name)
+        with gzip.open(FROZEN / "predictions.csv.gz", "rb") as source:
+            (bundle / "us" / "predictions.csv").write_bytes(source.read())
+        for name in release.ANNOTATION_CSVS:
+            shutil.copyfile(
+                ROOT / release.ANNOTATIONS / name, bundle / "annotations" / name
+            )
+        raw = export_full_run(bundle, countries=["us"], skip_app_data=True)
+        payload = release.export_payload(bundle, base)
+    raw_fable = next(
+        row for row in raw["countries"]["us"]["modelStats"] if row["model"] == FABLE
+    )
+    assert not set(CARRIED) & set(raw_fable)
+    assert json.dumps(payload["countries"]["us"], allow_nan=False) == committed