diff --git a/docs/gpt61sol/design.md b/docs/gpt61sol/design.md index 72607196..0ef81078 100644 --- a/docs/gpt61sol/design.md +++ b/docs/gpt61sol/design.md @@ -103,7 +103,8 @@ the stage's binding against it. The exporter still needs the Fable 5 usage carry-over. Fable 5 ran through the Anthropic batch adapter, and its committed rows carry no cost, token or latency -fields, so `export_full_run` reports `costUsd` $0 and omits the rest. The live +fields, so `export_full_run` reported `costUsd` $0 and omitted the rest (since +#230 it reports none of them). The live payload has $54.11, 3,850,174 tokens and 97.68 s. The driver copies those four values from the base, and the drift gate then checks every field and the key order. Ox Alpha's `costUsd` of $0.0 comes out of the exporter unchanged, so it diff --git a/paper/index.qmd b/paper/index.qmd index cfd9e3e0..f7ef1de0 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -759,6 +759,10 @@ def load_us_cost_latency() -> dict: ) computed = model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M) stats = us_model.set_index("model") + # model_cost_latency gives no entry to a model whose rows record no usage + # (Claude Fable 5), so seed every board model to reach its fallback. + for model_id in stats.index: + computed.setdefault(model_id, {}) for model_id, entry in computed.items(): row = stats.loc[model_id] if model_id in stats.index else None cost = entry.get("costPerHousehold") @@ -779,6 +783,15 @@ def load_us_cost_latency() -> dict: us_cost_latency = load_us_cost_latency() +# Every model the board prices shows a cost. One whose frozen rows record no +# usage (Claude Fable 5) takes its modelStats figure, not a dash. +_unpriced = [ + row["model"] + for _, row in us_model.iterrows() + if (row.get("costPerHousehold") or 0) > 0 + and us_cost_latency.get(row["model"], {}).get("costPerHousehold") is None +] +assert not _unpriced, _unpriced def cost_latency_table() -> pd.DataFrame: diff --git a/policybench/analysis.py b/policybench/analysis.py index 0ae3b331..e0398c63 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -1422,7 +1422,11 @@ def model_cost_latency( each household's calls summed, then the median across households — which is robust to the occasional rate-limit retry that inflates the mean. When a model's reconstructed cost is missing (a provider preview model not yet in - litellm's price map), it is filled from ``price_overrides``. + litellm's price map), it is filled from ``price_overrides`` by pricing the + rows that recorded both prompt and completion token counts. A row missing + either count adds nothing, as ``eval_no_tools._reconstruct_token_cost`` + prices no such row, and a model with no row to price (Claude Fable 5's + frozen board rows record no usage) gets no cost, not $0. """ if predictions.empty or "model" not in predictions.columns: return {} @@ -1437,19 +1441,29 @@ def model_cost_latency( .median() ) overrides = price_overrides or {} + token_columns = ["prompt_tokens", "completion_tokens"] + priceable = pd.DataFrame(columns=token_columns) + if set(token_columns) <= set(predictions.columns): + priceable = ( + predictions.dropna(subset=token_columns) + .groupby("model")[token_columns] + .sum() + ) out: dict[str, dict[str, float]] = {} for _, row in usage.iterrows(): model = str(row["model"]) cost = row.get("total_cost_usd") - if (cost is None or pd.isna(cost)) and model in overrides: + if ( + (cost is None or pd.isna(cost)) + and model in overrides + and row["model"] in priceable.index + ): price = overrides[model] - prompt = row.get("prompt_tokens") - completion = row.get("completion_tokens") - prompt = 0.0 if prompt is None or pd.isna(prompt) else float(prompt) - completion = ( - 0.0 if completion is None or pd.isna(completion) else float(completion) + prompt, completion = priceable.loc[row["model"], token_columns] + cost = ( + float(prompt) * price["input"] / 1e6 + + float(completion) * price["output"] / 1e6 ) - cost = prompt * price["input"] / 1e6 + completion * price["output"] / 1e6 entry: dict[str, float] = {} if cost is not None and not pd.isna(cost): entry["costUsd"] = _clean_json_number(float(cost)) diff --git a/scripts/finish_argon.py b/scripts/finish_argon.py index 3c5027df..0aff6028 100644 --- a/scripts/finish_argon.py +++ b/scripts/finish_argon.py @@ -213,7 +213,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py index 982ae6f6..505ad8a4 100644 --- a/scripts/finish_gpt61sol.py +++ b/scripts/finish_gpt61sol.py @@ -175,7 +175,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_haiku55.py b/scripts/finish_haiku55.py index 233f09ec..f62872f3 100644 --- a/scripts/finish_haiku55.py +++ b/scripts/finish_haiku55.py @@ -193,7 +193,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py index 16fe4e99..0e2e234b 100644 --- a/scripts/finish_musewonder.py +++ b/scripts/finish_musewonder.py @@ -203,7 +203,7 @@ PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: # Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# token or latency fields, so export_full_run reports none of its usage. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/tests/test_carried_usage.py b/tests/test_carried_usage.py new file mode 100644 index 00000000..df3e4e6b --- /dev/null +++ b/tests/test_carried_usage.py @@ -0,0 +1,153 @@ +"""Fable 5's carried usage: the export reports none of it, the carry restores it. + +Claude Fable 5's frozen board rows record no cost, token or latency fields. +So analysis.model_cost_latency computes none of its usage, and +export_full_run reports none (it reported a $0 cost before +model_cost_latency stopped pricing unrecorded tokens). Every release driver +carries the four published fields from its base release (CARRIED_USAGE). It +then gates on serialized bytes, so key order counts as well as values. + +Invariants: +- every driver carries the same four fields, in one order; +- the published Fable 5 row ends with those four fields in that order, so + assigning them to an exported row that lacks them reproduces the published + key order; +- release_20261006.export_payload's carry restores the published row byte + for byte whether the export omits Fable 5's usage or reports a $0 cost, and + it touches no other model; +- (slow, local) the committed snapshot's payload rebuilds byte for byte from + its committed inputs through export_full_run and that carry. +""" + +from __future__ import annotations + +import gzip +import importlib +import json +import shutil +import sys +import tempfile +from pathlib import Path + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import release_20261006 as release # noqa: E402 + +from policybench.snapshot_payload import ( # noqa: E402 + read_run_payload, + read_run_payload_text, +) + +FABLE = "claude-fable-5" +CARRIED = ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") +FROZEN = ROOT / release.SNAPSHOT_RUN +DRIVERS = ( + "release_20261006", + "release_20261013b", + "finish_gpt61sol", + "finish_haiku55", + "finish_musewonder", + "finish_argon", +) + + +@pytest.mark.parametrize("name", DRIVERS) +def test_every_driver_carries_fable_5s_four_fields_in_one_order(name): + assert importlib.import_module(name).CARRIED_USAGE == {FABLE: CARRIED} + + +def test_the_published_fable_5_row_ends_with_the_carried_fields(): + """The precondition for the carry to reproduce the published bytes when + the export reports none of Fable 5's usage.""" + rows = read_run_payload(FROZEN)["modelStats"] + fable = next(row for row in rows if row["model"] == FABLE) + assert tuple(fable)[-len(CARRIED) :] == CARRIED + assert all(fable[key] > 0 for key in CARRIED) + + +STAT_KEYS = ("condition", "score", "exact", "n", "accuracy", "aggregateScore") + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + leading=st.lists(st.sampled_from(STAT_KEYS), unique=True), + usage=st.tuples( + st.floats(min_value=0.01, max_value=1e4), + st.floats(min_value=1e-4, max_value=100), + st.integers(min_value=1, max_value=10**9), + st.floats(min_value=0.5, max_value=3600), + ), + exporter=st.sampled_from(["omits", "zero"]), +) +def test_the_carry_restores_the_published_row_whichever_way_the_export_reports_it( + monkeypatch, leading, usage, exporter +): + """release_20261006.export_payload's carry, run on a stubbed export: the + carried Fable 5 row serializes as the published one, and the other model's + row is the export's.""" + import policybench.dashboard_schema + import policybench.full_run_export + + published = {"model": FABLE, **{key: 1.5 for key in leading}} + published.update(zip(CARRIED, usage)) + exported_fable = {"model": FABLE, **{key: 1.5 for key in leading}} + if exporter == "zero": + exported_fable.update(costUsd=0.0, costPerHousehold=0.0) + other = {"model": "other", "exact": 50.0, "costUsd": 0.0, "costPerHousehold": 0.0} + base = {"countries": {"us": {"modelStats": [published, dict(other)]}}} + export = {"countries": {"us": {"modelStats": [exported_fable, dict(other)]}}} + monkeypatch.setattr( + policybench.full_run_export, "export_full_run", lambda *a, **k: export + ) + monkeypatch.setattr( + policybench.dashboard_schema, "validate_dashboard_payload", lambda *a, **k: [] + ) + with tempfile.TemporaryDirectory() as scratch: + bundle = Path(scratch) / "bundle" + for rel in release.EXPORT_INPUTS: + (bundle / rel).parent.mkdir(parents=True, exist_ok=True) + (bundle / rel).write_bytes(b"") + payload = release.export_payload(bundle, base) + fable, rest = payload["countries"]["us"]["modelStats"] + assert json.dumps(fable) == json.dumps(published) + assert json.dumps(rest) == json.dumps(other) + + +@pytest.mark.slow +def test_the_committed_payload_rebuilds_through_export_and_the_carry(monkeypatch): + """Local only, about a minute: export_full_run on the committed snapshot's + inputs reports none of Fable 5's usage, and with release_20261006's carry + the US payload is the committed one byte for byte.""" + from policybench.full_run_export import export_full_run + + monkeypatch.chdir(ROOT) + committed = read_run_payload_text(FROZEN) + base = {"countries": {"us": json.loads(committed)}} + with tempfile.TemporaryDirectory() as scratch: + bundle = Path(scratch) / release.RUN + (bundle / "us").mkdir(parents=True) + (bundle / "annotations").mkdir() + for name in (*release.REFERENCE_FILES, release.EXCLUSIONS): + shutil.copyfile(FROZEN / name, bundle / "us" / name) + with gzip.open(FROZEN / "predictions.csv.gz", "rb") as source: + (bundle / "us" / "predictions.csv").write_bytes(source.read()) + for name in release.ANNOTATION_CSVS: + shutil.copyfile( + ROOT / release.ANNOTATIONS / name, bundle / "annotations" / name + ) + raw = export_full_run(bundle, countries=["us"], skip_app_data=True) + payload = release.export_payload(bundle, base) + raw_fable = next( + row for row in raw["countries"]["us"]["modelStats"] if row["model"] == FABLE + ) + assert not set(CARRIED) & set(raw_fable) + assert json.dumps(payload["countries"]["us"], allow_nan=False) == committed diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index df3b8dfb..45c6eec4 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -1,10 +1,37 @@ -"""Tests for analysis.model_cost_latency (leaderboard cost + latency).""" +"""Tests for analysis.model_cost_latency (leaderboard cost + latency). + +Cost invariants, for every input (the property test executes them): +- costUsd and costPerHousehold are both present or both absent; +- a model has a cost exactly when one of its rows recorded total_cost_usd, or + it has a price override and one of its rows recorded both prompt and + completion token counts; a row missing either count is not priced, so no + cost rests on a count nobody recorded, and a model with no usage at all + gets no cost rather than $0; +- a recorded cost is the sum of the recorded row costs and the override is + not consulted; a filled cost is the override's prices applied to the token + sums of the rows that recorded both counts; +- costPerHousehold is costUsd over the distinct households in the input. +The frozen-snapshot tests check the published figures: every model's usage +fields but Claude Fable 5's reproduce from the frozen predictions, and Fable +5's, which those rows do not record, are not computed at all. +""" + +import json +import math +from pathlib import Path import numpy as np import pandas as pd import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from policybench.analysis import model_cost_latency +from policybench.config import PRICE_OVERRIDES_PER_1M +from policybench.snapshot_payload import read_run_payload + +ROOT = Path(__file__).resolve().parents[1] +SNAPSHOT_DIR = ROOT / "paper/snapshot/20260501" def _row(model, scenario, cost, elapsed, prompt=100, completion=50): @@ -92,3 +119,219 @@ def test_all_null_latency_is_omitted_not_zero(): def test_empty_predictions_returns_empty(): assert model_cost_latency(pd.DataFrame()) == {} + + +def test_price_override_without_recorded_tokens_gives_no_cost(): + """Claude Fable 5's frozen board rows record no cost, tokens or latency. Its + price override has nothing to price, so it gets no cost, not $0.""" + preds = pd.DataFrame( + [ + _row("claude-fable-5", s, np.nan, np.nan, prompt=np.nan, completion=np.nan) + for s in ("s1", "s2") + ] + ) + overrides = {"claude-fable-5": PRICE_OVERRIDES_PER_1M["claude-fable-5"]} + assert model_cost_latency(preds, overrides) == {} + + +def test_price_override_without_recorded_tokens_keeps_the_rest(): + preds = pd.DataFrame( + [_row("m", "s1", np.nan, 5.0, prompt=np.nan, completion=np.nan)] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert out == {"m": {"latencySeconds": 5.0}} + + +@pytest.mark.parametrize( + "prompt, completion", [(np.nan, 2_000_000), (1_000_000, np.nan)] +) +def test_price_override_needs_both_token_counts(prompt, completion): + """Pricing one side would report part of the spend as all of it.""" + preds = pd.DataFrame( + [_row("m", "s1", np.nan, 5.0, prompt=prompt, completion=completion)] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert "costUsd" not in out["m"] + assert "costPerHousehold" not in out["m"] + + +def test_price_override_needs_both_counts_on_one_row(): + """Counts split across rows price no row: each row is missing one side.""" + preds = pd.DataFrame( + [ + _row("m", "s1", np.nan, 5.0, prompt=1_000_000, completion=np.nan), + _row("m", "s2", np.nan, 5.0, prompt=np.nan, completion=1_000_000), + ] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + assert "costUsd" not in out["m"] + + +def test_price_override_prices_only_rows_with_both_counts(): + """A row missing a count adds nothing; it is not priced at $0 for it.""" + preds = pd.DataFrame( + [ + _row("m", "s1", np.nan, 5.0, prompt=1_000_000, completion=2_000_000), + _row("m", "s2", np.nan, 5.0, prompt=3_000_000, completion=np.nan), + _row("m", "s3", np.nan, 5.0, prompt=np.nan, completion=np.nan), + ] + ) + out = model_cost_latency(preds, {"m": {"input": 1.0, "output": 2.0}}) + # s1 only: 1M * $1/1M + 2M * $2/1M + assert out["m"]["costUsd"] == pytest.approx(5.0) + assert out["m"]["costPerHousehold"] == pytest.approx(5.0 / 3) + + +def test_recorded_zero_cost_and_recorded_zero_tokens_stay_zero(): + """A $0 the rows record (a free model, or zero tokens) is a real $0.""" + preds = pd.DataFrame( + [ + _row("free", "s1", 0.0, 5.0), + _row("zero-tokens", "s1", np.nan, 5.0, prompt=0, completion=0), + ] + ) + out = model_cost_latency(preds, {"zero-tokens": {"input": 1.0, "output": 2.0}}) + assert out["free"]["costUsd"] == 0.0 + assert out["zero-tokens"]["costUsd"] == 0.0 + + +MODEL_NAMES = ("a", "b", "c") +SCENARIOS = ("s1", "s2", "s3", "s4") + +missing = st.just(float("nan")) +usage_row = st.fixed_dictionaries( + { + "model": st.sampled_from(MODEL_NAMES), + "scenario_id": st.sampled_from(SCENARIOS), + "total_cost_usd": st.one_of( + missing, + st.just(0.0), + st.floats(min_value=0, max_value=50, allow_subnormal=False), + ), + "prompt_tokens": st.one_of(missing, st.integers(0, 5_000_000).map(float)), + "completion_tokens": st.one_of(missing, st.integers(0, 5_000_000).map(float)), + "elapsed_seconds": st.one_of(missing, st.floats(min_value=0, max_value=600)), + } +) +price = st.fixed_dictionaries( + { + "input": st.floats(min_value=0, max_value=100, allow_subnormal=False), + "output": st.floats(min_value=0, max_value=100, allow_subnormal=False), + } +) + + +def _reference_cost(rows: list[dict], override: dict | None) -> float | None: + """The cost the invariants define, from the rows in plain Python.""" + recorded = [ + r["total_cost_usd"] for r in rows if not math.isnan(r["total_cost_usd"]) + ] + if recorded: + return math.fsum(recorded) + priced = [ + r + for r in rows + if not math.isnan(r["prompt_tokens"]) and not math.isnan(r["completion_tokens"]) + ] + if override is None or not priced: + return None + return ( + math.fsum(r["prompt_tokens"] for r in priced) * override["input"] / 1e6 + + math.fsum(r["completion_tokens"] for r in priced) * override["output"] / 1e6 + ) + + +@settings(max_examples=300, deadline=None) +@given( + drawn=st.lists(usage_row, min_size=1, max_size=12), + overrides=st.dictionaries(st.sampled_from(MODEL_NAMES), price, max_size=3), + uncosted=st.sets(st.sampled_from(MODEL_NAMES)), + untokened=st.sets(st.sampled_from(MODEL_NAMES)), +) +def test_cost_is_present_exactly_when_recorded_usage_supports_it( + drawn, overrides, uncosted, untokened +): + # Whole models with no recorded cost, or (like Fable 5) no token counts, + # so the fill and the no-usage cases are drawn often. + nan = float("nan") + rows = [ + { + **row, + **({"total_cost_usd": nan} if row["model"] in uncosted else {}), + **( + {"prompt_tokens": nan, "completion_tokens": nan} + if row["model"] in untokened + else {} + ), + } + for row in drawn + ] + frame = pd.DataFrame(rows) + frame["variable"] = "v" + frame["prediction"] = 1.0 + frame["total_tokens"] = frame["prompt_tokens"] + frame["completion_tokens"] + out = model_cost_latency(frame, overrides) + households = frame["scenario_id"].nunique() + for model in sorted(set(frame["model"])): + entry = out.get(model, {}) + expected = _reference_cost( + [r for r in rows if r["model"] == model], overrides.get(model) + ) + assert ("costUsd" in entry) == ("costPerHousehold" in entry) + if expected is None: + assert "costUsd" not in entry + continue + assert entry["costUsd"] == pytest.approx(expected, rel=1e-12, abs=1e-12) + assert entry["costPerHousehold"] == pytest.approx( + entry["costUsd"] / households, rel=1e-12, abs=1e-12 + ) + + +# --- The frozen snapshot: published figures -------------------------------- + +USAGE_FIELDS = ("costUsd", "costPerHousehold", "latencySeconds", "totalTokens") + + +@pytest.fixture(scope="module") +def frozen(): + manifest = json.loads((SNAPSHOT_DIR / "manifest.json").read_text()) + run = SNAPSHOT_DIR / "runs" / manifest["source_run_labels"]["us"] + predictions = pd.read_csv( + run / "predictions.csv.gz", + usecols=[ + "model", + "scenario_id", + "variable", + "prediction", + "error", + "total_cost_usd", + "elapsed_seconds", + "prompt_tokens", + "completion_tokens", + "total_tokens", + ], + low_memory=False, + ) + published = {row["model"]: row for row in read_run_payload(run)["modelStats"]} + return model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M), published + + +def test_frozen_usage_reproduces_every_published_model_but_fable_5(frozen): + """The published cost, latency and tokens of every model but Fable 5 are + model_cost_latency's on the frozen predictions, to the bit and with the + same fields present.""" + computed, published = frozen + assert len(published) > 1 + for model, row in published.items(): + if model == "claude-fable-5": + continue + assert {k: row[k] for k in USAGE_FIELDS if k in row} == computed[model], model + + +def test_frozen_fable_5_usage_is_carried_not_computed(frozen): + """Fable 5's rows record no usage, so nothing is computed for it (no $0 + cost); the published figures are the four the release drivers carry.""" + computed, published = frozen + assert "claude-fable-5" not in computed + fable = published["claude-fable-5"] + assert all(fable[k] > 0 for k in USAGE_FIELDS) diff --git a/tests/test_finish_adds0928.py b/tests/test_finish_adds0928.py index dbc22a0d..d3b89328 100644 --- a/tests/test_finish_adds0928.py +++ b/tests/test_finish_adds0928.py @@ -546,7 +546,7 @@ def test_strict_export_recombines_to_freeze_bytes_and_binds_evidence( for model in driver.MODELS.values() ] for field in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): - stats[41][field] = None + del stats[41][field] payload = {"countries": {"us": {"modelStats": stats}}} live = {"countries": {"us": {"modelStats": incumbents}}} gate_calls = [] diff --git a/tests/test_finish_argon.py b/tests/test_finish_argon.py index 3cff93f9..80640d06 100644 --- a/tests/test_finish_argon.py +++ b/tests/test_finish_argon.py @@ -2829,8 +2829,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [ _stat(NEW, 65.0, costUsd=0.13), _stat(OTHER, 61.0, costUsd=0.11), @@ -2924,6 +2925,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_base_record_explains( assert {row["n"] for row in rows} == {1935} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index 2ed39158..f54c8858 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -21,7 +21,7 @@ import pandas as pd import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) @@ -1893,8 +1893,9 @@ def _exported(incumbents): """What export_full_run returns: Fable's usage is gone, the addition is new.""" stats = copy.deepcopy(incumbents) fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=2.0)] @@ -2712,6 +2713,9 @@ def test_export_writes_no_receipt_when_the_record_disagrees(exporting, monkeypat target=st.integers(min_value=0, max_value=44), mutation=st.sampled_from(sorted(MUTATIONS)), ) +@example(target=44, mutation="zero_cost_to_missing") +@example(target=44, mutation="zero_to_negative_zero") +@example(target=44, mutation="key_order") def test_the_no_drift_gate_refuses_any_change_to_any_incumbent( exporting, target, mutation ): @@ -2719,15 +2723,19 @@ def test_the_no_drift_gate_refuses_any_change_to_any_incumbent( stage, _, run, _ = exporting incumbents = _incumbents() stats = _exported(incumbents) - MUTATIONS[mutation](stats[target]) (stage / "data-board46.json").unlink(missing_ok=True) - # Fable's usage values are carried over from the base (the exporter cannot - # recompute them), so a changed value there is replaced by the released - # one; a missing key comes back out of order and is still refused. - if target == 44 and mutation == "zero_to_negative_zero": + # Fable's usage is carried over from the base (the exporter reports none + # of it), so its export row has no cost for these two to change. Their + # analogue there is an export that reports a cost after all, as the + # exporter did ($0) before model_cost_latency stopped pricing unrecorded + # tokens: the carry replaces it with the released value, so no drift. + if target == 44 and mutation in ("zero_cost_to_missing", "zero_to_negative_zero"): + zero = -0.0 if mutation == "zero_to_negative_zero" else 0.0 + stats[target].update(costUsd=zero, costPerHousehold=zero) run(stats, incumbents, early=True) assert json.dumps(stats[target]) == json.dumps(incumbents[target]) return + MUTATIONS[mutation](stats[target]) with pytest.raises(SystemExit, match="incumbent modelStats drift"): run(stats, incumbents, early=True) assert not (stage / "data-board46.json").exists() diff --git a/tests/test_finish_haiku55.py b/tests/test_finish_haiku55.py index 4f798b1f..3c65865e 100644 --- a/tests/test_finish_haiku55.py +++ b/tests/test_finish_haiku55.py @@ -3474,8 +3474,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=0.13)] @@ -3567,6 +3568,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_ten_records_explain(s assert {row["n"] for row in rows} == {1910} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_finish_musewonder.py b/tests/test_finish_musewonder.py index b0b2e0d2..aff002e3 100644 --- a/tests/test_finish_musewonder.py +++ b/tests/test_finish_musewonder.py @@ -2788,8 +2788,9 @@ def _exported(incumbents): row.update(n=driver.RELEASE_SCORED, exact=row["exact"] + 0.25) row["score"] = row["exact"] / 100 fable = stats[-1] - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return stats + [_stat(NEW, 65.0, costUsd=0.13)] @@ -2881,6 +2882,9 @@ def test_the_scope_check_allows_every_incumbent_change_the_base_record_explains( assert {row["n"] for row in rows} == {1926} fable = next(row for row in rows if row["model"] == "claude-fable-5") assert {key: fable[key] for key in FABLE_USAGE} == FABLE_USAGE + # The export reports none of Fable 5's usage; the carry puts it back where + # the published row has it, so the serialized key order is the release's. + assert list(fable) == list(incumbents[-1]) assert fable["exact"] == incumbents[-1]["exact"] + 0.25 (_, run_dir, kwargs), (_, bundle, live, record) = scope.calls assert run_dir == bundle == Path("b") diff --git a/tests/test_freeze_argon.py b/tests/test_freeze_argon.py index 54191b8a..514bdb96 100644 --- a/tests/test_freeze_argon.py +++ b/tests/test_freeze_argon.py @@ -1000,8 +1000,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board diff --git a/tests/test_freeze_gpt61sol.py b/tests/test_freeze_gpt61sol.py index a2f6efa2..d5ed8ae7 100644 --- a/tests/test_freeze_gpt61sol.py +++ b/tests/test_freeze_gpt61sol.py @@ -444,8 +444,9 @@ def staged_board(freeze_preflight, monkeypatch, rebuilds, exports): for row in exported["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] exports["board"] = exported def export_full_run(run_dir, *, countries, skip_app_data): diff --git a/tests/test_freeze_haiku55.py b/tests/test_freeze_haiku55.py index c899945d..dab82af1 100644 --- a/tests/test_freeze_haiku55.py +++ b/tests/test_freeze_haiku55.py @@ -910,8 +910,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board diff --git a/tests/test_freeze_musewonder.py b/tests/test_freeze_musewonder.py index 6c30eda9..44d1df31 100644 --- a/tests/test_freeze_musewonder.py +++ b/tests/test_freeze_musewonder.py @@ -965,8 +965,9 @@ def _without_carried_usage(board: dict) -> dict: for row in board["countries"]["us"]["modelStats"] if row["model"] == "claude-fable-5" ) - fable.update(costUsd=0.0, costPerHousehold=0.0) - del fable["totalTokens"], fable["latencySeconds"] + # Fable 5's rows record no usage, so export_full_run reports none of it. + for key in ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds"): + del fable[key] return board