diff --git a/reference_audit/2026-10-05-louisiana/README.md b/reference_audit/2026-10-05-louisiana/README.md index 53d8462e..5f55097b 100644 --- a/reference_audit/2026-10-05-louisiana/README.md +++ b/reference_audit/2026-10-05-louisiana/README.md @@ -90,7 +90,7 @@ If Max rules instead that the convention takes the amount published before the f ## Method 1. **Sweep.** `scripts/sweep_la_standard_deduction.py` rebuilds every household with `Scenario.to_pe_household` and computes every output on policyengine-us 2.15.17 with `latest_final`. Its baseline reproduces all 1,928 scored references. It then reruns every household with the Louisiana 2026 amounts set to each candidate, joint at 200%. Outputs: `verification/sweep_la_standard_deduction.csv`, `verification/sweep_la_households.json` (each Louisiana household's AGI, deduction, taxable income and tax under each candidate) and the log. -2. **Score.** `scripts/leaderboard_impact.py` copies the frozen run to scratch directories and never writes the snapshot. For each option it rewrites the two reference values (and the copy's reference digest) or appends the two outputs to the exclusion record. It scores each copy with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy reproduces the published payload's scoring field for field. Outputs: `verification/leaderboard_impact.json`, `leaderboard_impact_models.csv` and the log. +2. **Score.** `scripts/leaderboard_impact.py` copies the frozen run to scratch directories and never writes the snapshot. It reads the run and the sweep's values from git at the pass's commits, not from the working tree ([Pinned inputs](#pinned-inputs)). For each option it rewrites the two reference values (and the copy's reference digest) or appends the two outputs to the exclusion record. It scores each copy with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy reproduces the published payload's scoring field for field. Outputs: `verification/leaderboard_impact.json`, `leaderboard_impact_models.csv` and the log, which is its standard output. 3. **Module check.** `scripts/verify_convention_module.py` recomputes all 1,984 outputs with `fixes/latest_final_la.py` and compares each with the sweep's `ldr_published` column; they agree on every output. policyengine-us applies a reform set twice (before uprating and again with its structural reforms), so the module's guard accepts the engine's value or its own. 4. **Tests.** `tests/test_la_standard_deduction_audit.py` checks these records without loading the engine: - the baseline matches every scored reference, and the candidates move only the two outputs; @@ -103,3 +103,39 @@ If Max rules instead that the convention takes the amount published before the f Louisiana's withholding proxy (`la_withheld_income_tax`) subtracts the federal standard deduction, so the Louisiana amount does not reach the federal SALT deduction. The three other Louisiana households (038, 057, 074) have Louisiana AGI below the deduction under every candidate. Engine: `results/local/adds202609/triage/.venv-pe21517` (policyengine-us 2.15.17). + +## Pinned inputs + +This audit's passes read release dashboard-data-20260930's run, as #187 committed it at `8b4c0ca1`, and this directory's inputs as #202 merged them at `9ce4ade8`. Every committed file the scripts below write regenerates byte for byte from those inputs. Release dashboard-data-20261006 (#202) then excluded eight more outputs and rewrote the run's payload and exclusion record. On that working tree the unpinned impact script exited 0 after rewriting both evidence files. Every option scored eight fewer outputs than in the pass (1,920 instead of 1,928, and 1,918 instead of 1,926 for `exclude`), and the top-five lists and rank changes moved. + +Each script below now takes every file it reads from git with `git show :`. It stops before scoring or writing anything unless each file's sha256 matches its pin, and it never reads the working tree. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + +### `scripts/leaderboard_impact.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/data.json.gz` | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | +| `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | +| `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | +| `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | +| `verification/sweep_la_standard_deduction.csv` | `9ce4ade8` | `406bceb71581dfe676722740bd588896e2b72ef1baa74b4975d80b1ca851a8e0` | + +It stages each file under `/pass_inputs/` and reads only that copy. `verification/sweep_la_standard_deduction.csv` is the sweep's output (step 1); the score step reads the candidates' values from it. + +### Reproduce + +The scoring code is the checkout's. The impact script's check on its unchanged copy stops it if that code no longer scores the pinned run as the pinned payload records. The scripts run from any checkout that has both commits in its history; a shallow clone needs `git fetch --unshallow` first: + +```bash +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py --scratch \ + > reference_audit/2026-10-05-louisiana/verification/leaderboard_impact.log +``` + +`--out-dir ` writes the files elsewhere. `tests/test_reference_audit_pins.py` checks the pins, that a changed byte in any input is refused, and that no script reads the working tree. Its slow test regenerates `leaderboard_impact.json` and `leaderboard_impact_models.csv` from this checkout, whose run #202 rewrote, and requires the committed files, and the log, byte for byte. The slow test takes one to two minutes per audit, so CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_audit_pins.py +``` diff --git a/reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py index 8030792f..12f46ab7 100644 --- a/reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py @@ -1,6 +1,17 @@ """Score the frozen run under each option for the two Louisiana state income tax outputs. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as PASS_COMMIT (release +dashboard-data-20260930, #187) holds it, and the sweep's candidate values as +AUDIT_COMMIT (#202, which merged this audit) holds them. Later releases rewrite the +working tree's run: dashboard-data-20261006 (#202) excluded eight more outputs, and +scoring that run would silently regenerate different evidence (1,920 scored outputs +instead of the pass's 1,928). Each input in INPUTS is staged under +``/pass_inputs`` and must match its pinned sha256, or the script stops before +scoring anything. The scoring code is the checkout's; the ``keep`` check below stops +the script if that code no longer scores the pinned run as the pinned payload records. + +Copies the staged run (predictions, references, scenarios, exclusions) to scratch directories and never writes the snapshot. Each option is one copy: ``keep`` @@ -19,6 +30,12 @@ PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-louisiana/scripts/leaderboard_impact.py --scratch + +Its standard output is ``verification/leaderboard_impact.log``. The checkout needs +PASS_COMMIT and AUDIT_COMMIT in its history (a shallow clone needs +``git fetch --unshallow``). ``--out-dir`` writes the verification files elsewhere; +tests/test_reference_audit_pins.py regenerates them that way and requires the +committed ones, and the log, byte for byte. """ from __future__ import annotations @@ -37,12 +54,51 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -SWEEP = HERE / "verification/sweep_la_standard_deduction.csv" OUT_DIR = HERE / "verification" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_PATH = "reference_audit/2026-10-05-louisiana" +SWEEP_PATH = f"{AUDIT_PATH}/verification/sweep_la_standard_deduction.csv" +# Every file the script reads from the repository: path -> (commit, sha256). +# pass_inputs() stages each at the same path under its target. +INPUTS = { + f"{RUN_PATH}/data.json.gz": ( + PASS_COMMIT, + "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + ), + f"{RUN_PATH}/predictions.csv.gz": ( + PASS_COMMIT, + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + ), + f"{RUN_PATH}/reference_outputs.csv": ( + PASS_COMMIT, + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + ), + f"{RUN_PATH}/reference_outputs.csv.meta.json": ( + PASS_COMMIT, + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b", + ), + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + f"{RUN_PATH}/scenarios.csv": ( + PASS_COMMIT, + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + ), + f"{RUN_PATH}/scenarios.csv.meta.json": ( + PASS_COMMIT, + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb", + ), + SWEEP_PATH: ( + AUDIT_COMMIT, + "406bceb71581dfe676722740bd588896e2b72ef1baa74b4975d80b1ca851a8e0", + ), +} OUTPUTS = ( ("scenario_051", "state_income_tax_before_refundable_credits"), ("scenario_077", "state_income_tax_before_refundable_credits"), @@ -50,6 +106,7 @@ REGENERATE = ("ldr_published", "ldr_official", "hold_2025") OPTIONS = ("keep",) + REGENERATE + ("exclude",) TOLERANCE = 1.0 +# Copied into each copy and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -71,12 +128,71 @@ EXACT_PAYLOAD_KEYS = ("programStats", "heatmap", "globalWeights", "failureModes") +def overlaps(scratch: Path, root: Path) -> bool: + """Whether ``scratch`` and ``root`` are one directory or one holds the other. + + Judged by filesystem identity (device and inode), not spelling, so case on a + case-insensitive filesystem or a link cannot hide an overlap. ``scratch`` need + not exist yet; its nearest existing ancestors decide. + """ + + def identities(path: Path) -> set[tuple[int, int]]: + found = set() + for part in (path, *path.parents): + try: + status = part.stat() + except OSError: + continue + found.add((status.st_dev, status.st_ino)) + return found + + root_status = root.stat() + if (root_status.st_dev, root_status.st_ino) in identities(scratch): + return True + try: + scratch_status = scratch.stat() + except OSError: + return False + return (scratch_status.st_dev, scratch_status.st_ino) in identities(root) + + def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def candidate_values() -> dict[str, dict[tuple[str, str], float]]: - sweep = pd.read_csv(SWEEP).set_index(["scenario_id", "variable"]) +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target + + +def pass_inputs(target: Path) -> Path: + """Stage every input in INPUTS at its repository path under ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, (commit, pinned) in INPUTS.items(): + git_input(commit, path, pinned, target / path) + return target + + +def candidate_values(path: Path) -> dict[str, dict[tuple[str, str], float]]: + """Each candidate's two reference values, from the sweep CSV at ``path``.""" + sweep = pd.read_csv(path).set_index(["scenario_id", "variable"]) values = {} for candidate in ("engine",) + REGENERATE: values[candidate] = {key: float(sweep.loc[key, candidate]) for key in OUTPUTS} @@ -106,12 +222,13 @@ def exclusion_entries(values: dict) -> list[dict]: return entries -def stage(target: Path, option: str, values: dict) -> Path: +def stage(source: Path, target: Path, option: str, values: dict) -> Path: + """Copy the staged run at ``source`` to ``target`` and apply ``option``.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if option in REGENERATE: # Rewrite only the two rows' value field, so every other byte stays as frozen, # and record the new digest in the copy's sidecar, as a regeneration would. @@ -169,8 +286,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -207,9 +325,9 @@ def compare(base: dict, option: dict) -> pd.DataFrame: return rows.sort_values("exact_rank_published") -def exact_models(values: dict, option: str) -> dict[str, list[str]]: +def exact_models(values: dict, option: str, source: Path) -> dict[str, list[str]]: """Models within the $1 tolerance of each output's reference under an option.""" - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) predictions = published["scenarioPredictions"] result = {} for sid, variable in OUTPUTS: @@ -230,17 +348,29 @@ def exact_models(values: dict, option: str) -> dict[str, list[str]]: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--scratch", required=True) + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() - scratch = Path(args.scratch) - values = candidate_values() + scratch = Path(args.scratch).resolve() + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/, so scratch and + # the repository must not overlap: neither may lie inside the other. + if overlaps(scratch, ROOT): + parser.error(f"--scratch must not overlap the repository ({ROOT})") + inputs = pass_inputs(scratch / "pass_inputs") + source = inputs / RUN_PATH + values = candidate_values(inputs / SWEEP_PATH) payloads = {} for option in OPTIONS: - payloads[option] = analyze(stage(scratch / option, option, values)) + payloads[option] = analyze(stage(source, scratch / option, option, values)) if option == "keep": - check_reproduces_published(payloads["keep"]) + check_reproduces_published(payloads["keep"], source) - OUT_DIR.mkdir(parents=True, exist_ok=True) + out_dir.mkdir(parents=True, exist_ok=True) summary = { "outputs": [f"{sid}/{variable}" for sid, variable in OUTPUTS], "reference_values": { @@ -267,7 +397,7 @@ def main() -> None: frames.append(frame.reset_index()) summary["options"][option] = { "scored_outputs": int(payloads[option]["modelStats"][0]["n"]), - "exact_models": exact_models(values, option), + "exact_models": exact_models(values, option, source), "exact_delta_pp": { "min": float(models["exact_delta"].min()), "max": float(models["exact_delta"].max()), @@ -295,8 +425,8 @@ def main() -> None: "top5_exact": models.sort_values("exact_rank_option").head(5).index.tolist(), "top5_score": models.sort_values("score_rank_option").head(5).index.tolist(), } - pd.concat(frames).to_csv(OUT_DIR / "leaderboard_impact_models.csv", index=False) - (OUT_DIR / "leaderboard_impact.json").write_text( + pd.concat(frames).to_csv(out_dir / "leaderboard_impact_models.csv", index=False) + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(json.dumps(summary, indent=2)) diff --git a/reference_audit/2026-10-05-medicare-part-b/README.md b/reference_audit/2026-10-05-medicare-part-b/README.md index deb60345..78dd09b6 100644 --- a/reference_audit/2026-10-05-medicare-part-b/README.md +++ b/reference_audit/2026-10-05-medicare-part-b/README.md @@ -55,13 +55,13 @@ scenario_114's prompt has no premium or coverage line. It lists a single head, a For each household with a Medicare-eligible person it records each person's Medicare variables and a propagation trace. The trace lists every engine variable, at every 2026 period the simulation computed, whose value differs between the reference and the `no_part_b` or `not_enrolled` simulation. Households where an output moves also run on a grid: the Part B readings crossed with the four readings of the state income tax in SALT from #191 (withholding estimate, liability at a fixed point, none paid, none paid without the local sales tax estimate). Outputs: `verification/sweep_part_b.csv` (every output and impact weight under every reading), `verification/sweep_part_b_summary.json`, `verification/sweep_part_b_households.json` and `verification/sweep_part_b.log`. 2. **Explain.** `scripts/explain_households.py` writes `verification/part_b_households.csv`. Each of the 33 Medicare households gets a row with the premium, the federal itemization election and medical deduction with and without it, its federal, state and SNAP outputs, and the state variables its trace shows moving. -3. **Propose.** `scripts/propose_exclusions.py` writes `proposed_exclusions.json`. It holds a `reference_depends_on_unlisted_input` record, in the format of the frozen run's `reference_exclusions.json`, for every scored output that `no_part_b` moves by more than the $1 exact-match tolerance. An output #191 already proposes to exclude gets a standalone record under `conditional_on_salt_decision` instead, to install only if #191's record is not adopted. The script also writes `verification/model_answers.csv`, which tags every model's answer on the moved outputs with the readings it lands within $1 of. It reads #191's records from `verification/inputs/pr191_proposed_exclusions.json`, a copy of #191's `reference_audit/2026-10-05/proposed_exclusions.json` at head 8af912a0 that the script checks against its sha256, so this directory does not depend on #191's branch. -4. **Impact.** `scripts/leaderboard_impact.py` copies the frozen run to scratch directories and never writes the snapshot. It scores each copy with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy reproduces every compared value of the published payload. It then scores three cases: +3. **Propose.** `scripts/propose_exclusions.py` writes `proposed_exclusions.json`, from the run, the sweep and the dashboard's model labels as git holds them at the pass's commits ([Pinned inputs](#pinned-inputs)). It holds a `reference_depends_on_unlisted_input` record, in the format of the frozen run's `reference_exclusions.json`, for every scored output that `no_part_b` moves by more than the $1 exact-match tolerance. An output #191 already proposes to exclude gets a standalone record under `conditional_on_salt_decision` instead, to install only if #191's record is not adopted. The script also writes `verification/model_answers.csv`, which tags every model's answer on the moved outputs with the readings it lands within $1 of. It reads #191's records from `verification/inputs/pr191_proposed_exclusions.json`, a copy of #191's `reference_audit/2026-10-05/proposed_exclusions.json` at head 8af912a0 that the script checks against its sha256, so this directory does not depend on #191's branch. +4. **Impact.** `scripts/leaderboard_impact.py` copies the frozen run to scratch directories and never writes the snapshot. It reads the run, this audit's records and sweep, and #191's records from git at the pass's commits, not from the working tree ([Pinned inputs](#pinned-inputs)). It scores each copy with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy reproduces every compared value of the published payload. It then scores three cases: - this audit's two records alone, as if #191 is not adopted; - #191's three records alone, which reproduces #191's figures exactly; - #191's three plus this audit's Virginia record. - Each case also recomputes the legacy household impact summary, `impact_summary_by_model.csv`. The freeze writes it beside the analyze output with `scripts/freeze_snapshot.household_impact_summary_by_model`, and the snapshot manifest pins it. The unchanged copy reproduces the frozen file. Outputs: `verification/leaderboard_impact.json`, `leaderboard_impact_models.csv`, `leaderboard_impact_marginal.csv` and `leaderboard_impact.log`. + Each case also recomputes the legacy household impact summary, `impact_summary_by_model.csv`. The freeze writes it beside the analyze output with `scripts/freeze_snapshot.household_impact_summary_by_model`, and the snapshot manifest pins it. The unchanged copy reproduces the frozen file. Outputs: `verification/leaderboard_impact.json`, `leaderboard_impact_models.csv` and `leaderboard_impact_marginal.csv`. 5. **Verify.** Four independent reviewers checked the work in a Claude Code workflow on 2026-10-05, each in its own scratch directory: - **Mechanism.** One read the engine source, built a static graph of every reader of the premium, recomputed scenario_114 to the cent from engine intermediates, and reran all 100 households. - **Differential.** One reran the no-premium sweep two other ways: a parameter reform setting the 2026 base premium to 0, and a structural reform cutting `medical_expense_health_insurance_premiums` to its non-Medicare branch. Both were bit-identical to `no_part_b` on all 1,984 outputs. @@ -128,7 +128,7 @@ No model answered within $1 of either frozen reference. Model answers land on th ## Leaderboard impact -From `verification/leaderboard_impact.json`. Scored outputs per model are 1,928 today. +From `verification/leaderboard_impact.json`. Scored outputs per model are 1,928 in release dashboard-data-20260930, the release this audit scored. | Case | Records added | Scored outputs | Exact-rate change, all models | GPT-6 Sol exact | Always-zero exact | |---|---:|---:|---|---:|---:| @@ -196,3 +196,55 @@ Two sets of counts follow, one if d963 adopts #191's three records and one if it 4. **Paper.** Add the Medicare enrollment and Part B premium input to the paper's unlisted-input list and sweep narrative, and re-render the abstract and tables. 5. **Legacy impact summary.** The freeze regenerates `impact_summary_by_model.csv` from the scored reference; its pin in the snapshot manifest changes with the records (see "Leaderboard impact"). + +## Pinned inputs + +This audit's passes read release dashboard-data-20260930's run, as #187 committed it at `8b4c0ca1`, and this directory's inputs as #202 merged them at `9ce4ade8`. Every committed file the scripts below write regenerates byte for byte from those inputs. Release dashboard-data-20261006 (#202) then installed #191's three records and this audit's Virginia record. It also rewrote the run's payload, exclusion record and legacy impact summary. On that working tree the unpinned impact script stopped at the first case: its appended records duplicated #202's, and the analyze CLI refuses a duplicate exclusion. The unpinned `propose_exclusions.py` stopped too, because both scenario_114 outputs were already excluded. + +Each script below now takes every file it reads from git with `git show :`. It stops before scoring or writing anything unless each file's sha256 matches its pin, and it never reads the working tree. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + +### `scripts/leaderboard_impact.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/data.json.gz` | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | +| `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | +| `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | +| `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | +| `/analysis/impact_summary_by_model.csv` | `8b4c0ca1` | `e6e034adee408cc798897bc0825ace9740055ab20e528ce842bd06e2c81e0922` | +| `proposed_exclusions.json` | `9ce4ade8` | `fee2523738a2da96876e6cd3141325cb5939e0140a357c0b59b0f39d5b37eb76` | +| `verification/sweep_part_b.csv` | `9ce4ade8` | `a0f7820ca0b5b64118509dc39ae5469ecbde89e5f843065761d429b81458681b` | +| `verification/inputs/pr191_proposed_exclusions.json` | `9ce4ade8` | `3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272` | + +It stages each file under `/pass_inputs/` and reads only that copy. `verification/inputs/pr191_proposed_exclusions.json` is #191's `reference_audit/2026-10-05/proposed_exclusions.json` at its head `8af912a0`, byte for byte. `verification/sweep_part_b.csv` is read only with `--weights-check`. The legacy impact summary comes from `scripts/freeze_snapshot.py` in the checkout. + +### `scripts/propose_exclusions.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `app/src/modelMeta.ts` | `8b4c0ca1` | `c2f39f6f891ed93f564534283800bd15e51504ea179097c2a77a9278d0157a00` | +| `verification/sweep_part_b_summary.json` | `9ce4ade8` | `f294a7952ee8a3e6b4e570cba403d84df6ddd8f6ae295fb31e976aaa381526a4` | +| `verification/sweep_part_b_households.json` | `9ce4ade8` | `c34bee0320747cba71f33e9a8e00f5c28a6b82473542ce966352d9041554e1aa` | +| `verification/inputs/pr191_proposed_exclusions.json` | `9ce4ade8` | `3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272` | + +`app/src/modelMeta.ts` supplies the dashboard's model labels. + +### Reproduce + +The scoring code is the checkout's. The impact script's check on its unchanged copy stops it if that code no longer scores the pinned run as the pinned payload records. The scripts run from any checkout that has both commits in its history; a shallow clone needs `git fetch --unshallow` first: + +```bash +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py --scratch --weights-check +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py +``` + +`--out-dir ` writes the files elsewhere. `tests/test_reference_audit_pins.py` checks the pins, that a changed byte in any input is refused, and that no script reads the working tree. It reruns `propose_exclusions.py` from a checkout whose working tree is junk and requires the committed `proposed_exclusions.json` and `verification/model_answers.csv` byte for byte. Its slow test regenerates `leaderboard_impact.json`, `leaderboard_impact_models.csv` and `leaderboard_impact_marginal.csv` (with `--weights-check`) from this checkout, whose run #202 rewrote, and requires the committed files byte for byte. The slow test takes one to two minutes per audit, so CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_audit_pins.py +``` diff --git a/reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py index 12d050d4..0370e2fe 100644 --- a/reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py @@ -1,6 +1,19 @@ """Score the frozen run under each combination of the Part B and SALT proposals. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions, and the legacy impact +summary) as PASS_COMMIT (release dashboard-data-20260930, #187) holds it, and this +audit's proposals, sweep and copy of #191's records as AUDIT_COMMIT (#202, which +merged this audit) holds them. Later releases rewrite the working tree's run: +dashboard-data-20261006 (#202) installed #191's three records and this audit's +Virginia record and rewrote the legacy impact summary, so a copy of today's run with +these records appended carries duplicate exclusions, which the analyze CLI refuses. +Each input in INPUTS is staged under ``/pass_inputs`` and must match its +pinned sha256, or the script stops before scoring anything. The scoring code is the +checkout's; the ``published`` checks below stop the script if that code no longer +scores the pinned run as the pinned payload and impact summary record. + +Copies the staged run (predictions, references, scenarios, exclusions) to scratch directories and never writes the snapshot. Each copy is scored with ``python -m policybench.cli analyze``, the command the freeze runs, and the dashboard payloads are compared. The cases: @@ -29,11 +42,16 @@ #191's records are read from ``verification/inputs/pr191_proposed_exclusions.json``, a copy of ``reference_audit/2026-10-05/proposed_exclusions.json`` at #191's head -8af912a0, checked against its sha256. +8af912a0; its pinned sha256 is that file's. PYTHONPATH= /.venv-pe21517/bin/python \\ reference_audit/2026-10-05-medicare-part-b/scripts/leaderboard_impact.py \\ --scratch --weights-check + +The checkout needs PASS_COMMIT and AUDIT_COMMIT in its history (a shallow clone needs +``git fetch --unshallow``). ``--out-dir`` writes the verification files elsewhere; +tests/test_reference_audit_pins.py regenerates them that way and requires the +committed ones byte for byte. """ from __future__ import annotations @@ -53,16 +71,66 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -PROPOSED = HERE / "proposed_exclusions.json" -SWEEP = HERE / "verification/sweep_part_b.csv" OUT_DIR = HERE / "verification" -SALT_COPY = HERE / "verification/inputs/pr191_proposed_exclusions.json" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_PATH = "reference_audit/2026-10-05-medicare-part-b" +FROZEN_IMPACT_PATH = f"{RUN_PATH}/analysis/impact_summary_by_model.csv" +PROPOSED_PATH = f"{AUDIT_PATH}/proposed_exclusions.json" +SWEEP_PATH = f"{AUDIT_PATH}/verification/sweep_part_b.csv" +SALT_COPY_PATH = f"{AUDIT_PATH}/verification/inputs/pr191_proposed_exclusions.json" +# #191's reference_audit/2026-10-05/proposed_exclusions.json at its head 8af912a0. SALT_SHA256 = "3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272" -FROZEN_IMPACT = RUN / "analysis/impact_summary_by_model.csv" +# Every file the script reads from the repository: path -> (commit, sha256). +# pass_inputs() stages each at the same path under its target. +INPUTS = { + f"{RUN_PATH}/data.json.gz": ( + PASS_COMMIT, + "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + ), + f"{RUN_PATH}/predictions.csv.gz": ( + PASS_COMMIT, + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + ), + f"{RUN_PATH}/reference_outputs.csv": ( + PASS_COMMIT, + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + ), + f"{RUN_PATH}/reference_outputs.csv.meta.json": ( + PASS_COMMIT, + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b", + ), + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + f"{RUN_PATH}/scenarios.csv": ( + PASS_COMMIT, + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + ), + f"{RUN_PATH}/scenarios.csv.meta.json": ( + PASS_COMMIT, + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb", + ), + FROZEN_IMPACT_PATH: ( + PASS_COMMIT, + "e6e034adee408cc798897bc0825ace9740055ab20e528ce842bd06e2c81e0922", + ), + PROPOSED_PATH: ( + AUDIT_COMMIT, + "fee2523738a2da96876e6cd3141325cb5939e0140a357c0b59b0f39d5b37eb76", + ), + SWEEP_PATH: ( + AUDIT_COMMIT, + "a0f7820ca0b5b64118509dc39ae5469ecbde89e5f843065761d429b81458681b", + ), + SALT_COPY_PATH: (AUDIT_COMMIT, SALT_SHA256), +} +# Copied into each copy and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -115,14 +183,71 @@ def always_zero(run_dir: Path) -> dict[str, float]: } +def overlaps(scratch: Path, root: Path) -> bool: + """Whether ``scratch`` and ``root`` are one directory or one holds the other. + + Judged by filesystem identity (device and inode), not spelling, so case on a + case-insensitive filesystem or a link cannot hide an overlap. ``scratch`` need + not exist yet; its nearest existing ancestors decide. + """ + + def identities(path: Path) -> set[tuple[int, int]]: + found = set() + for part in (path, *path.parents): + try: + status = part.stat() + except OSError: + continue + found.add((status.st_dev, status.st_ino)) + return found + + root_status = root.stat() + if (root_status.st_dev, root_status.st_ino) in identities(scratch): + return True + try: + scratch_status = scratch.stat() + except OSError: + return False + return (scratch_status.st_dev, scratch_status.st_ino) in identities(root) + + def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def salt_records() -> list[dict]: - if sha256(SALT_COPY) != SALT_SHA256: - raise SystemExit(f"{SALT_COPY} does not match #191's file at 8af912a0") - return json.loads(SALT_COPY.read_text())["exclusions"] +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target + + +def pass_inputs(target: Path) -> Path: + """Stage every input in INPUTS at its repository path under ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, (commit, pinned) in INPUTS.items(): + git_input(commit, path, pinned, target / path) + return target + + +def salt_records(path: Path) -> list[dict]: + """#191's records, from the staged copy at ``path`` (pass_inputs checked it).""" + return json.loads(path.read_text())["exclusions"] def legacy_impact_summary(run_dir: Path) -> pd.DataFrame: @@ -157,12 +282,15 @@ def legacy_impact_change(base: pd.DataFrame, case: pd.DataFrame) -> dict: } -def stage(target: Path, extra: list[dict], weights: pd.DataFrame | None = None) -> Path: +def stage( + source: Path, target: Path, extra: list[dict], weights: pd.DataFrame | None = None +) -> Path: + """Copy the staged run at ``source`` to ``target`` and apply the case.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if extra: record = json.loads((target / "reference_exclusions.json").read_text()) record["exclusions"].extend(extra) @@ -232,8 +360,9 @@ def payload_differences(a: dict, b: dict) -> list[str]: return out -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) diffs = payload_differences(published, base) if diffs: raise SystemExit( @@ -329,25 +458,37 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--scratch", required=True) parser.add_argument("--weights-check", action="store_true") + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() - scratch = Path(args.scratch) - - proposal = json.loads(PROPOSED.read_text()) + scratch = Path(args.scratch).resolve() + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/, so scratch and + # the repository must not overlap: neither may lie inside the other. + if overlaps(scratch, ROOT): + parser.error(f"--scratch must not overlap the repository ({ROOT})") + inputs = pass_inputs(scratch / "pass_inputs") + source = inputs / RUN_PATH + + proposal = json.loads((inputs / PROPOSED_PATH).read_text()) va = proposal["exclusions"] federal = [c["record"] for c in proposal["conditional_on_salt_decision"]] - salt = salt_records() + salt = salt_records(inputs / SALT_COPY_PATH) cases = { "part_b": va + federal, "salt": salt, "salt_and_part_b": salt + va, } - base_dir = stage(scratch / "published", []) + base_dir = stage(source, scratch / "published", []) base = analyze(base_dir) - check_reproduces_published(base) + check_reproduces_published(base, source) zero_published = always_zero(base_dir) impact_base = legacy_impact_summary(base_dir) - frozen_impact = pd.read_csv(FROZEN_IMPACT) + frozen_impact = pd.read_csv(inputs / FROZEN_IMPACT_PATH) pd.testing.assert_frame_equal( impact_base.reset_index(drop=True), frozen_impact, @@ -358,7 +499,7 @@ def main() -> None: summaries, tables = [], {} for name, records in cases.items(): - run_dir = stage(scratch / name, records) + run_dir = stage(source, scratch / name, records) case = analyze(run_dir) zero = {"published": zero_published, "case": always_zero(run_dir)} summary = summarize(name, records, base, case, zero) @@ -398,8 +539,8 @@ def main() -> None: weights_check = None if args.weights_check: - sweep = pd.read_csv(SWEEP) - run_dir = stage(scratch / "weights_2_15_17", [], weights=sweep) + sweep = pd.read_csv(inputs / SWEEP_PATH) + run_dir = stage(source, scratch / "weights_2_15_17", [], weights=sweep) moved = int( ( sweep["reference_impact_weight"].notna() @@ -421,7 +562,7 @@ def main() -> None: ), } - OUT_DIR.mkdir(parents=True, exist_ok=True) + out_dir.mkdir(parents=True, exist_ok=True) wide = pd.concat( { name: table[ @@ -437,8 +578,8 @@ def main() -> None: wide.insert(1, "exact_rank_published", first["exact_rank_published"]) wide.insert(2, "score_published", first["score_published"]) wide.insert(3, "score_rank_published", first["score_rank_published"]) - wide.to_csv(OUT_DIR / "leaderboard_impact_models.csv") - marginal.to_csv(OUT_DIR / "leaderboard_impact_marginal.csv") + wide.to_csv(out_dir / "leaderboard_impact_models.csv") + marginal.to_csv(out_dir / "leaderboard_impact_marginal.csv") result = { "published_reproduced": True, "legacy_impact_summary_reproduced": True, @@ -446,7 +587,7 @@ def main() -> None: "virginia_record_on_top_of_salt": marginal_summary, "impact_weights_check": weights_check, } - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(result, indent=2) + "\n" ) with pd.option_context("display.width", 250, "display.max_rows", 100): diff --git a/reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py b/reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py index 2533b939..94d1ce0f 100644 --- a/reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py +++ b/reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py @@ -15,37 +15,80 @@ ruling; ``decided_on`` is the date the proposal was drafted and a release sets it to the ruling's date. +Reads the pass's inputs from git, never from the working tree: the frozen run's +predictions and exclusions and the dashboard's model labels as PASS_COMMIT (release +dashboard-data-20260930, #187) holds them, and this audit's sweep and copy of #191's +records as AUDIT_COMMIT (#202, which merged this audit) holds them. Each must match +its pinned sha256 in INPUTS, or the script stops before writing anything. Release +dashboard-data-20261006 (#202) excluded both scenario_114 outputs, so on its run the +script stops on already-excluded outputs. + PYTHONPATH= /.venv-pe21517/bin/python \\ reference_audit/2026-10-05-medicare-part-b/scripts/propose_exclusions.py + +``--out-dir`` writes proposed_exclusions.json and verification/model_answers.csv under +another directory; tests/test_reference_audit_pins.py regenerates them that way and +requires the committed ones byte for byte. """ from __future__ import annotations +import argparse import hashlib +import io import json import re +import subprocess from pathlib import Path import pandas as pd HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +OUT_DIR = HERE +OUT = "proposed_exclusions.json" +ANSWERS = "verification/model_answers.csv" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" ) -SUMMARY = HERE / "verification/sweep_part_b_summary.json" -HOUSEHOLDS = HERE / "verification/sweep_part_b_households.json" -OUT = HERE / "proposed_exclusions.json" -ANSWERS = HERE / "verification/model_answers.csv" -MODEL_META = ROOT / "app/src/modelMeta.ts" +AUDIT_PATH = "reference_audit/2026-10-05-medicare-part-b" +SUMMARY_PATH = f"{AUDIT_PATH}/verification/sweep_part_b_summary.json" +HOUSEHOLDS_PATH = f"{AUDIT_PATH}/verification/sweep_part_b_households.json" +MODEL_META_PATH = "app/src/modelMeta.ts" # PolicyEngine/policybench#191 at the head this audit read. SALT_PR = "PolicyEngine/policybench#191" SALT_HEAD = "8af912a062dd7ae1373de4c043ae2727d3406930" # reference_audit/2026-10-05/proposed_exclusions.json at SALT_HEAD, kept here so the # script does not depend on #191's branch. -SALT_COPY = HERE / "verification/inputs/pr191_proposed_exclusions.json" +SALT_COPY_PATH = f"{AUDIT_PATH}/verification/inputs/pr191_proposed_exclusions.json" SALT_SHA256 = "3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272" +# Every file the script reads from the repository: path -> (commit, sha256). +INPUTS = { + f"{RUN_PATH}/predictions.csv.gz": ( + PASS_COMMIT, + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + ), + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + MODEL_META_PATH: ( + PASS_COMMIT, + "c2f39f6f891ed93f564534283800bd15e51504ea179097c2a77a9278d0157a00", + ), + SUMMARY_PATH: ( + AUDIT_COMMIT, + "f294a7952ee8a3e6b4e570cba403d84df6ddd8f6ae295fb31e976aaa381526a4", + ), + HOUSEHOLDS_PATH: ( + AUDIT_COMMIT, + "c34bee0320747cba71f33e9a8e00f5c28a6b82473542ce966352d9041554e1aa", + ), + SALT_COPY_PATH: (AUDIT_COMMIT, SALT_SHA256), +} SALT_DECISION = "d963" DRAFTED_ON = "2026-10-05" ENGINE = "policyengine-us 2.15.17" @@ -79,20 +122,48 @@ def listed(value: float) -> str: def model_labels() -> dict[str, str]: """The dashboard's display names (app/src/modelMeta.ts MODEL_LABELS).""" - text = MODEL_META.read_text() + text = pass_input(MODEL_META_PATH).decode() block = text.split("export const MODEL_LABELS", 1)[1].split("};", 1)[0] return dict(re.findall(r'"([^"]+)":\s*"([^"]+)"', block)) +def git_bytes(commit: str, path: str, pinned: str) -> bytes: + """``path`` as ``commit`` holds it, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + return result.stdout + + +def pass_input(path: str) -> bytes: + """The input at ``path``, as INPUTS pins it.""" + commit, pinned = INPUTS[path] + return git_bytes(commit, path, pinned) + + def salt_proposal() -> dict: - if hashlib.sha256(SALT_COPY.read_bytes()).hexdigest() != SALT_SHA256: - raise SystemExit(f"{SALT_COPY} does not match #191's file at {SALT_HEAD}") - return json.loads(SALT_COPY.read_text()) + """#191's proposal at SALT_HEAD, from this audit's copy (SALT_SHA256 pins it).""" + return json.loads(pass_input(SALT_COPY_PATH)) def model_answers(rows: list[dict], household: dict) -> pd.DataFrame: """Every model's answer on each moved output, tagged with the readings it matches.""" - predictions = pd.read_csv(RUN / "predictions.csv.gz", low_memory=False) + predictions = pd.read_csv( + io.BytesIO(pass_input(f"{RUN_PATH}/predictions.csv.gz")), + compression="gzip", + low_memory=False, + ) out = [] for row in rows: sid, variable = row["scenario_id"], row["variable"] @@ -294,13 +365,26 @@ def record(row: dict, household: dict, summary: dict, answers: pd.DataFrame) -> def main() -> None: - summary = json.loads(SUMMARY.read_text()) + parser = argparse.ArgumentParser() + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help=f"where to write {OUT} and {ANSWERS} (default: %(default)s)", + ) + out_dir = Path(parser.parse_args().out_dir) + # Check every pin before reading anything, so a refused input writes nothing. + for path in INPUTS: + pass_input(path) + summary = json.loads(pass_input(SUMMARY_PATH)) households = { - h["scenario_id"]: h for h in json.loads(HOUSEHOLDS.read_text())["households"] + h["scenario_id"]: h + for h in json.loads(pass_input(HOUSEHOLDS_PATH))["households"] } if summary["scored_reference_mismatches"]: raise SystemExit("the sweep's baseline does not reproduce the references") - frozen = json.loads((RUN / "reference_exclusions.json").read_text())["exclusions"] + frozen = json.loads(pass_input(f"{RUN_PATH}/reference_exclusions.json"))[ + "exclusions" + ] frozen_keys = {(e["scenario_id"], e["variable"]) for e in frozen} salt = salt_proposal() salt_keys = {(e["scenario_id"], e["variable"]): e for e in salt["exclusions"]} @@ -311,7 +395,8 @@ def main() -> None: if other != {(m["scenario_id"], m["variable"]) for m in moved}: raise SystemExit(f"{reading} moves a different set of outputs") answers = model_answers(moved, households) - answers.to_csv(ANSWERS, index=False) + (out_dir / ANSWERS).parent.mkdir(parents=True, exist_ok=True) + answers.to_csv(out_dir / ANSWERS, index=False) exclusions, conditional, already = [], [], [] for row in moved: @@ -382,7 +467,7 @@ def main() -> None: "irmaa_from_2026_income", ], } - OUT.write_text(json.dumps(payload, indent=1) + "\n") + (out_dir / OUT).write_text(json.dumps(payload, indent=1) + "\n") print(json.dumps(payload, indent=1)) print(answers.to_string(index=False)) diff --git a/reference_audit/2026-10-05-payroll/README.md b/reference_audit/2026-10-05-payroll/README.md index 74a1277c..ba6afa6d 100644 --- a/reference_audit/2026-10-05-payroll/README.md +++ b/reference_audit/2026-10-05-payroll/README.md @@ -161,8 +161,8 @@ Run from this checkout. `` is `results/local/adds202609/triage` in the m 2. **Classify.** As above; `program_classification.json` is the result, and `verification/law_research.json` holds the records and verdicts. 3. **Sweep.** `scripts/sweep_payroll_scope.py` recomputes all 1,984 outputs with and without `fixes/payroll_mandatory_scope.py`, an output-scope adapter that drops the five reviewed optional contributions from the list behind `employee_state_payroll_tax` and changes no formula or rate. The baseline reproduces all 1,928 scored references; the 19 excluded outputs that differ keep the values they were decided on, each listed in the sidecar's `excluded_outputs_rechecked`. The adapter moves four outputs, the four payroll references above, and no other output by even a cent (`verification/sweep_payroll_scope.csv` and `.log`). `CountryTaxBenefitSystem` applies a reform twice to the same system (`policyengine_us/system.py`), so the adapter is written to be idempotent: its second pass leaves the already-flattened list unchanged. 4. **Propose.** `scripts/propose_changes.py` writes both alternatives in the format a release installs: `proposed_exclusions.json` (four `reference_depends_on_unlisted_input` records sharing one unlisted input) and `proposed_regenerations.json` (four regenerated values with the adapter's sha256). It refuses a sweep that moves any output the proposal does not cover, and an adapter whose program list differs from the reviewed optional programs. -5. **Answers.** `scripts/model_answers.py` writes `verification/model_answers.csv` and `model_answers_summary.json`. -6. **Impact.** `scripts/leaderboard_impact.py --with-salt` writes `verification/leaderboard_impact*.csv`, `leaderboard_impact.json` and `.log`. It reads #191's records from its commit `8af912a0`. +5. **Answers.** `scripts/model_answers.py` writes `verification/model_answers.csv` and `model_answers_summary.json`, from the payload as git holds it at the pass's commit ([Pinned inputs](#pinned-inputs)). +6. **Impact.** `scripts/leaderboard_impact.py --with-salt` writes `verification/leaderboard_impact*.csv` and `leaderboard_impact.json`. It reads the run, this audit's proposals and #191's records from git at the pass's commits, not from the working tree ([Pinned inputs](#pinned-inputs)). ``` PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH=$PWD /.venv-pe21517/bin/python reference_audit/2026-10-05-payroll/scripts/decompose_payroll.py --out-dir reference_audit/2026-10-05-payroll/verification @@ -230,3 +230,49 @@ The release-checklist reviewer found both options installable, and neither a rec - **Engine.** policyengine-us has no input for the employer's deduction choice. An upstream change could add one, defaulting to the largest share, so users can model either reading; it would also need to settle Delaware, Maine, Vermont and Washington. The Connecticut rate parameter cites 31-49e, where 31-49g(b)(1) holds the rate. California's SDI rate parameter carries 1.3% back to 0000-01-01, which is wrong for earlier years (1.2% in 2025). The Massachusetts parameters do not encode Acts 2026 c. 101. None of these moves a benchmark number. - **#191.** Its README rounds GPT-6 Sol's 95.6146% to 95.62%. Its scenario_081 record treats the $805.01 Massachusetts share as a contribution the household paid, which this audit classifies as optional. - **SALT.** policyengine-us 2.15.17 leaves state payroll contributions out of the federal SALT deduction, the defect #191 records. + +## Pinned inputs + +This audit's passes read release dashboard-data-20260930's run, as #187 committed it at `8b4c0ca1`, and this directory's inputs as #202 merged them at `9ce4ade8`. Every committed file the scripts below write regenerates byte for byte from those inputs. Release dashboard-data-20261006 (#202) then installed the four proposed exclusions and #191's three records, and rewrote the run's payload and exclusion record. On that working tree the unpinned impact script stopped at the `salt` copy with `--with-salt`: its appended records duplicated #202's, and the analyze CLI refuses a duplicate exclusion. The unpinned `model_answers.py` exited 0 after rewriting `verification/model_answers.csv`: its `scored` column turned false on the four outputs' 184 rows. + +Each script below now takes every file it reads from git with `git show :`. It stops before scoring or writing anything unless each file's sha256 matches its pin, and it never reads the working tree. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + +### `scripts/leaderboard_impact.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/data.json.gz` | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | +| `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | +| `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | +| `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | +| `proposed_exclusions.json` | `9ce4ade8` | `f23088c2d76ce2a0f1c535529952e579798920450c0ca47a9e975f3fe3c44157` | +| `proposed_regenerations.json` | `9ce4ade8` | `58386f369f6164cc9d63cc4dcc4423495e214e45948fb2e2479910fc8165efef` | +| `reference_audit/2026-10-05/proposed_exclusions.json` | `9ce4ade8` | `3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272` | + +It stages each file under `/pass_inputs/` and reads only that copy. `reference_audit/2026-10-05/proposed_exclusions.json` holds #191's records and is read only with `--with-salt`. The pass read it at #191's head `8af912a0`, which `leaderboard_impact.json` records as `salt_commit`. #202 merged the same bytes, so the script reads them from `9ce4ade8` and does not need #191's closed branch. + +### `scripts/model_answers.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/data.json.gz` | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | +| `verification/payroll_decomposition.csv` | `9ce4ade8` | `fa34d6d98b7ea5fa00023ed93946948e48ea728e45702cea222355e008be9cab` | +| `program_classification.json` | `9ce4ade8` | `282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a` | + +### Reproduce + +The scoring code is the checkout's. The impact script's check on its unchanged copy stops it if that code no longer scores the pinned run as the pinned payload records. The scripts run from any checkout that has both commits in its history; a shallow clone needs `git fetch --unshallow` first: + +```bash +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py --scratch --with-salt +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05-payroll/scripts/model_answers.py +``` + +`--out-dir ` writes the files elsewhere. `tests/test_reference_audit_pins.py` checks the pins, that a changed byte in any input is refused, and that no script reads the working tree. It reruns `model_answers.py` from a checkout whose working tree is junk and requires the committed `verification/model_answers.csv` and `model_answers_summary.json` byte for byte. Its slow test regenerates `leaderboard_impact.json` and the ten `leaderboard_impact_*.csv` files (with `--with-salt`) from this checkout, whose run #202 rewrote, and requires the committed files byte for byte. The slow test takes one to two minutes per audit, so CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_audit_pins.py +``` diff --git a/reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py index eb8f9fd7..ad76b8f4 100644 --- a/reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py @@ -1,6 +1,18 @@ """Score the frozen run as published, with the proposed exclusions, and with regenerated references. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as PASS_COMMIT (release +dashboard-data-20260930, #187) holds it, and this audit's proposals and #191's +records as AUDIT_COMMIT (#202, which merged both audits) holds them. Later releases +rewrite the working tree's run: dashboard-data-20261006 (#202) installed the four +proposed exclusions and #191's three, so a copy of today's run with them appended +carries duplicate exclusions, which the analyze CLI refuses. Each input in INPUTS is +staged under ``/pass_inputs`` and must match its pinned sha256, or the script +stops before scoring anything. The scoring code is the checkout's; the ``published`` +check below stops the script if that code no longer scores the pinned run as the +pinned payload records. + +Copies the staged run (predictions, references, scenarios, exclusions) to scratch directories and never writes the snapshot. Three copies are scored with ``python -m policybench.cli analyze``, the command the freeze runs: @@ -17,12 +29,19 @@ Sensitivity: the open state income tax withholding proposal (PolicyEngine/policybench#191, decision d963) excludes three federal outputs and may land in the same release. With ``--with-salt`` the script also scores that proposal alone (``salt``) and each option on -top of it (``exclude+salt``, ``regenerate+salt``), reading its records from the pinned -commit ``SALT_COMMIT``; the ``*+salt`` deltas are measured against ``salt``. +top of it (``exclude+salt``, ``regenerate+salt``), with its records as #191's head +``SALT_COMMIT`` holds them; the ``*+salt`` deltas are measured against ``salt``. +#202 merged that file unchanged (SALT_SHA256), so the script reads it from +AUDIT_COMMIT and does not need #191's closed branch. PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-payroll/scripts/leaderboard_impact.py --scratch \\ [--with-salt] + +The checkout needs PASS_COMMIT and AUDIT_COMMIT in its history (a shallow clone needs +``git fetch --unshallow``). ``--out-dir`` writes the verification files elsewhere; +tests/test_reference_audit_pins.py regenerates them that way and requires the +committed ones byte for byte. """ from __future__ import annotations @@ -41,13 +60,63 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -EXCLUSIONS = HERE / "proposed_exclusions.json" -REGENERATIONS = HERE / "proposed_regenerations.json" OUT_DIR = HERE / "verification" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_PATH = "reference_audit/2026-10-05-payroll" +EXCLUSIONS_PATH = f"{AUDIT_PATH}/proposed_exclusions.json" +REGENERATIONS_PATH = f"{AUDIT_PATH}/proposed_regenerations.json" +# #191's records: its reference_audit/2026-10-05/proposed_exclusions.json at its head +# SALT_COMMIT, which leaderboard_impact.json records; #202 merged the same bytes. +SALT_COMMIT = "8af912a062dd7ae1373de4c043ae2727d3406930" +SALT_RECORDS = "reference_audit/2026-10-05/proposed_exclusions.json" +SALT_SHA256 = "3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272" +# Every file the script reads from the repository: path -> (commit, sha256). +# pass_inputs() stages each at the same path under its target. +INPUTS = { + f"{RUN_PATH}/data.json.gz": ( + PASS_COMMIT, + "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + ), + f"{RUN_PATH}/predictions.csv.gz": ( + PASS_COMMIT, + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + ), + f"{RUN_PATH}/reference_outputs.csv": ( + PASS_COMMIT, + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + ), + f"{RUN_PATH}/reference_outputs.csv.meta.json": ( + PASS_COMMIT, + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b", + ), + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + f"{RUN_PATH}/scenarios.csv": ( + PASS_COMMIT, + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + ), + f"{RUN_PATH}/scenarios.csv.meta.json": ( + PASS_COMMIT, + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb", + ), + EXCLUSIONS_PATH: ( + AUDIT_COMMIT, + "f23088c2d76ce2a0f1c535529952e579798920450c0ca47a9e975f3fe3c44157", + ), + REGENERATIONS_PATH: ( + AUDIT_COMMIT, + "58386f369f6164cc9d63cc4dcc4423495e214e45948fb2e2479910fc8165efef", + ), + SALT_RECORDS: (AUDIT_COMMIT, SALT_SHA256), +} +# Copied into each copy and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -70,26 +139,75 @@ RANKED = ("exact", "within1pct", "score") VARIANTS = ("exclude", "regenerate") OUTPUT = "payroll_tax" -SALT_COMMIT = "8af912a062dd7ae1373de4c043ae2727d3406930" -SALT_RECORDS = "reference_audit/2026-10-05/proposed_exclusions.json" -def salt_exclusions() -> list[dict]: - """The #191 proposal's records, read from its pinned commit.""" - shown = subprocess.run( - ["git", "show", f"{SALT_COMMIT}:{SALT_RECORDS}"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - return json.loads(shown.stdout)["exclusions"] +def overlaps(scratch: Path, root: Path) -> bool: + """Whether ``scratch`` and ``root`` are one directory or one holds the other. + + Judged by filesystem identity (device and inode), not spelling, so case on a + case-insensitive filesystem or a link cannot hide an overlap. ``scratch`` need + not exist yet; its nearest existing ancestors decide. + """ + + def identities(path: Path) -> set[tuple[int, int]]: + found = set() + for part in (path, *path.parents): + try: + status = part.stat() + except OSError: + continue + found.add((status.st_dev, status.st_ino)) + return found + + root_status = root.stat() + if (root_status.st_dev, root_status.st_ino) in identities(scratch): + return True + try: + scratch_status = scratch.stat() + except OSError: + return False + return (scratch_status.st_dev, scratch_status.st_ino) in identities(root) def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target + + +def pass_inputs(target: Path) -> Path: + """Stage every input in INPUTS at its repository path under ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, (commit, pinned) in INPUTS.items(): + git_input(commit, path, pinned, target / path) + return target + + +def salt_exclusions(inputs: Path) -> list[dict]: + """The #191 proposal's records, from the inputs staged under ``inputs``.""" + return json.loads((inputs / SALT_RECORDS).read_text())["exclusions"] + + def always_zero(run_dir: Path) -> dict[str, float]: """The always-zero baseline, as paper_results._always_zero_weighted_rates scores it.""" import numpy as np @@ -120,18 +238,20 @@ def always_zero(run_dir: Path) -> dict[str, float]: } -def stage(target: Path, variant: str | None) -> Path: +def stage(inputs: Path, target: Path, variant: str | None) -> Path: + """Copy the run staged under ``inputs`` to ``target`` and apply ``variant``.""" + source = inputs / RUN_PATH if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) parts = set((variant or "").split("+")) - {""} extra = [] if "exclude" in parts: - extra += json.loads(EXCLUSIONS.read_text())["exclusions"] + extra += json.loads((inputs / EXCLUSIONS_PATH).read_text())["exclusions"] if "salt" in parts: - extra += salt_exclusions() + extra += salt_exclusions(inputs) if extra: record = json.loads((target / "reference_exclusions.json").read_text()) record["exclusions"].extend(extra) @@ -139,7 +259,7 @@ def stage(target: Path, variant: str | None) -> Path: json.dumps(record, indent=2) + "\n" ) if "regenerate" in parts: - changes = json.loads(REGENERATIONS.read_text())["regenerated"] + changes = json.loads((inputs / REGENERATIONS_PATH).read_text())["regenerated"] reference = pd.read_csv(target / "reference_outputs.csv", dtype={"value": str}) index = reference.set_index(["scenario_id", "variable"]).index for change in changes: @@ -149,7 +269,7 @@ def stage(target: Path, variant: str | None) -> Path: raise SystemExit(f"{key}: frozen value differs from the reference CSV") reference.at[row, "value"] = repr(float(change["regenerated_value"])) reference.to_csv(target / "reference_outputs.csv", index=False) - before = (RUN / "reference_outputs.csv").read_text().splitlines() + before = (source / "reference_outputs.csv").read_text().splitlines() after = (target / "reference_outputs.csv").read_text().splitlines() changed = [i for i, (a, b) in enumerate(zip(before, after)) if a != b] if len(before) != len(after) or len(changed) != len(changes): @@ -195,8 +315,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -326,43 +447,56 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--scratch", required=True) parser.add_argument("--with-salt", action="store_true") + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() - scratch = Path(args.scratch) - - base_dir = stage(scratch / "published", None) + scratch = Path(args.scratch).resolve() + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/, so scratch and + # the repository must not overlap: neither may lie inside the other. + if overlaps(scratch, ROOT): + parser.error(f"--scratch must not overlap the repository ({ROOT})") + inputs = pass_inputs(scratch / "pass_inputs") + + base_dir = stage(inputs, scratch / "published", None) base = analyze(base_dir) - check_reproduces_published(base) + check_reproduces_published(base, inputs / RUN_PATH) zero_published = always_zero(base_dir) - OUT_DIR.mkdir(parents=True, exist_ok=True) + out_dir.mkdir(parents=True, exist_ok=True) summary = {"published_reproduced": True} plan = [(variant, base, zero_published) for variant in VARIANTS] if args.with_salt: - salt_dir = stage(scratch / "salt", "salt") + salt_dir = stage(inputs, scratch / "salt", "salt") salt = analyze(salt_dir) salt_zero = always_zero(salt_dir) plan.insert(0, ("salt", base, zero_published)) plan += [(f"{variant}+salt", salt, salt_zero) for variant in VARIANTS] summary["salt_commit"] = SALT_COMMIT for variant, reference_payload, reference_zero in plan: - run_dir = stage(scratch / variant, variant) + run_dir = stage(inputs, scratch / variant, variant) other = analyze(run_dir) models = compare(reference_payload, other) programs = program_rows(reference_payload, other) zero = {"published": reference_zero, "proposed": always_zero(run_dir)} records = [] if "exclude" in variant: - records += json.loads(EXCLUSIONS.read_text())["exclusions"] + records += json.loads((inputs / EXCLUSIONS_PATH).read_text())["exclusions"] if "regenerate" in variant: - records += json.loads(REGENERATIONS.read_text())["regenerated"] + records += json.loads((inputs / REGENERATIONS_PATH).read_text())[ + "regenerated" + ] if variant == "salt": - records += salt_exclusions() + records += salt_exclusions(inputs) changed = [f"{r['scenario_id']}/{r['variable']}" for r in records] models.to_csv( - OUT_DIR / f"leaderboard_impact_{variant.replace('+', '_plus_')}_models.csv" + out_dir / f"leaderboard_impact_{variant.replace('+', '_plus_')}_models.csv" ) programs.to_csv( - OUT_DIR + out_dir / f"leaderboard_impact_{variant.replace('+', '_plus_')}_programs.csv" ) summary[variant] = summarize( @@ -391,7 +525,7 @@ def main() -> None: ].round(4) ) print(programs.round(4)) - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(json.dumps(summary, indent=2)) diff --git a/reference_audit/2026-10-05-payroll/scripts/model_answers.py b/reference_audit/2026-10-05-payroll/scripts/model_answers.py index dd70cbf0..f485b8b2 100644 --- a/reference_audit/2026-10-05-payroll/scripts/model_answers.py +++ b/reference_audit/2026-10-05-payroll/scripts/model_answers.py @@ -17,14 +17,29 @@ The published-reference scores must equal the payload's own per-row ``exact`` field; the script stops if any differs. +Reads the pass's inputs from git, never from the working tree: the published payload +as PASS_COMMIT (release dashboard-data-20260930, #187) holds it, and this audit's +decomposition and classification as AUDIT_COMMIT (#202, which merged this audit) +holds them. Each must match its pinned sha256 in INPUTS, or the script stops before +writing anything. Release dashboard-data-20261006 (#202) excluded the four outputs, +so its payload marks their rows unscored and the answers table would change. + PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-payroll/scripts/model_answers.py + +``--out-dir`` writes model_answers.csv and model_answers_summary.json elsewhere; +tests/test_reference_audit_pins.py regenerates them that way and requires the +committed ones byte for byte. """ from __future__ import annotations +import argparse import gzip +import hashlib +import io import json +import subprocess from pathlib import Path import pandas as pd @@ -33,18 +48,73 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) OUT = HERE / "verification" OUTPUT = "payroll_tax" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_PATH = "reference_audit/2026-10-05-payroll" +DECOMPOSITION_PATH = f"{AUDIT_PATH}/verification/payroll_decomposition.csv" +CLASSIFICATION_PATH = f"{AUDIT_PATH}/program_classification.json" +# Every file the script reads from the repository: path -> (commit, sha256). +INPUTS = { + f"{RUN_PATH}/data.json.gz": ( + PASS_COMMIT, + "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + ), + DECOMPOSITION_PATH: ( + AUDIT_COMMIT, + "fa34d6d98b7ea5fa00023ed93946948e48ea728e45702cea222355e008be9cab", + ), + CLASSIFICATION_PATH: ( + AUDIT_COMMIT, + "282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a", + ), +} + + +def git_bytes(commit: str, path: str, pinned: str) -> bytes: + """``path`` as ``commit`` holds it, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + return result.stdout + + +def pass_input(path: str) -> bytes: + """The input at ``path``, as INPUTS pins it.""" + commit, pinned = INPUTS[path] + return git_bytes(commit, path, pinned) def main() -> None: - payload = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) - decomposition = pd.read_csv(OUT / "payroll_decomposition.csv") - classification = json.loads((HERE / "program_classification.json").read_text()) + parser = argparse.ArgumentParser() + parser.add_argument( + "--out-dir", + default=str(OUT), + help="where to write the answer tables (default: %(default)s)", + ) + out_dir = Path(parser.parse_args().out_dir) + # Check every pin before reading anything, so a refused input writes nothing. + for path in INPUTS: + pass_input(path) + payload = json.loads(gzip.decompress(pass_input(f"{RUN_PATH}/data.json.gz"))) + decomposition = pd.read_csv(io.BytesIO(pass_input(DECOMPOSITION_PATH))) + classification = json.loads(pass_input(CLASSIFICATION_PATH)) optional = { leaf for program in classification["programs"] @@ -119,8 +189,9 @@ def main() -> None: } ) table = pd.DataFrame(rows) - table.to_csv(OUT / "model_answers.csv", index=False) - (OUT / "model_answers_summary.json").write_text( + out_dir.mkdir(parents=True, exist_ok=True) + table.to_csv(out_dir / "model_answers.csv", index=False) + (out_dir / "model_answers_summary.json").write_text( json.dumps({"outputs": summary}, indent=2) + "\n" ) with pd.option_context("display.width", 250, "display.max_columns", 30): diff --git a/reference_audit/2026-10-05/README.md b/reference_audit/2026-10-05/README.md index d1bb62c6..6a0166e2 100644 --- a/reference_audit/2026-10-05/README.md +++ b/reference_audit/2026-10-05/README.md @@ -40,8 +40,8 @@ A state's tax can depend on federal tax or on federal SALT, so the input is iter - no modeled Medicare Part B premium; - Maryland's 24 county rates; - scenario_120 under r02's IRA fix. -3. **Propose.** `scripts/propose_exclusions.py` writes `proposed_exclusions.json`. It holds one `reference_depends_on_unlisted_input` record, in the format of the frozen run's `reference_exclusions.json`, for every scored output the `liability` reading moves by more than the $1 exact-match tolerance. Outputs already excluded for another reason are listed with the note text a release should add to their records. -4. **Impact.** `scripts/leaderboard_impact.py` copies the frozen run twice to a scratch directory and never writes the snapshot. It appends the proposed records to one copy and scores both with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy must reproduce the published payload's scoring: modelStats except the cost and latency fields the freeze overlays, plus programStats, heatmap, globalWeights and failureModes. It does: every compared value is equal. Outputs: `verification/leaderboard_impact_models.csv`, `leaderboard_impact_programs.csv`, `leaderboard_impact.json` and `leaderboard_impact.log`. +3. **Propose.** `scripts/propose_exclusions.py` writes `proposed_exclusions.json`, from the run and the sweep as git holds them at the pass's commits ([Pinned inputs](#pinned-inputs)). It holds one `reference_depends_on_unlisted_input` record, in the format of the frozen run's `reference_exclusions.json`, for every scored output the `liability` reading moves by more than the $1 exact-match tolerance. Outputs already excluded for another reason are listed with the note text a release should add to their records. +4. **Impact.** `scripts/leaderboard_impact.py` copies the frozen run twice to a scratch directory and never writes the snapshot. It appends the proposed records to one copy and scores both with `python -m policybench.cli analyze`, the command the freeze runs. The unchanged copy must reproduce the published payload's scoring: modelStats except the cost and latency fields the freeze overlays, plus programStats, heatmap, globalWeights and failureModes. It does: every compared value is equal. It reads the run and the proposals from git at the pass's commits, not from the working tree ([Pinned inputs](#pinned-inputs)). Outputs: `verification/leaderboard_impact_models.csv`, `leaderboard_impact_programs.csv` and `leaderboard_impact.json`. 5. **Verify.** Five independent reviewers checked the work in a Claude Code workflow on 2026-10-05, each in its own scratch directory: - **Mechanism.** One traced the code path and recomputed the five withholding estimates and the nine federal values for 022, 081 and 114 by hand, all to the cent. - **Differential.** One re-implemented the sweep with a different mechanism: per-person state withholding inputs. Its values were bit-identical on all 1,984 outputs. @@ -121,3 +121,49 @@ The reviewers checked these against the code and tests. They are not done here. - Add the withholding reading to the unlisted-input list and the sweep narrative. - Re-render the abstract and the tables. 6. **Existing notes.** Add the `already_excluded` note text to scenario_078's record, whose note says the SALT cap "binds at every 2026 county rate", and to scenario_120's record. + +## Pinned inputs + +This audit's passes read release dashboard-data-20260930's run, as #187 committed it at `8b4c0ca1`, and this directory's inputs as #202 merged them at `9ce4ade8`. Every committed file the scripts below write regenerates byte for byte from those inputs. Release dashboard-data-20261006 (#202) then installed the three proposed records and rewrote the run's payload and exclusion record. On that working tree the unpinned impact script stopped at the proposed copy: its appended records duplicated #202's, and the analyze CLI refuses a duplicate exclusion. The unpinned `propose_exclusions.py` exited 0 after rewriting `proposed_exclusions.json` with no proposals, listing all five moved outputs as already excluded. + +Each script below now takes every file it reads from git with `git show :`. It stops before scoring or writing anything unless each file's sha256 matches its pin, and it never reads the working tree. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + +### `scripts/leaderboard_impact.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/data.json.gz` | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | +| `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | +| `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | +| `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | +| `proposed_exclusions.json` | `9ce4ade8` | `3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272` | + +It stages each file under `/pass_inputs/` and reads only that copy. `proposed_exclusions.json` is the file #191 proposed at its head `8af912a0`; #202 merged it unchanged. + +### `scripts/propose_exclusions.py` + +| Input | Commit | sha256 | +|---|---|---| +| `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | +| `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | +| `verification/sweep_salt_withholding.csv` | `9ce4ade8` | `5918e349699471b8dcf9baa9584afdd583f8054624301839bb228bdc90a52d0c` | +| `verification/sweep_salt_withholding_households.json` | `9ce4ade8` | `b36669b4a6b43483fbec5ce18f2d8affbeda98df34d3523ae39ff2deec0e4e40` | +| `verification/variants.json` | `9ce4ade8` | `5cf0588e6071d4c35cfe6f8e07a8f3aaf8873dfbc8b7e34d496813d047141764` | + +### Reproduce + +The scoring code is the checkout's. The impact script's check on its unchanged copy stops it if that code no longer scores the pinned run as the pinned payload records. The scripts run from any checkout that has both commits in its history; a shallow clone needs `git fetch --unshallow` first: + +```bash +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05/scripts/leaderboard_impact.py --scratch +PYTHONPATH=$PWD uv run python reference_audit/2026-10-05/scripts/propose_exclusions.py +``` + +`--out-dir ` writes the files elsewhere. `tests/test_reference_audit_pins.py` checks the pins, that a changed byte in any input is refused, and that no script reads the working tree. It reruns `propose_exclusions.py` from a checkout whose working tree is junk and requires the committed `proposed_exclusions.json` byte for byte. Its slow test regenerates `leaderboard_impact.json`, `leaderboard_impact_models.csv` and `leaderboard_impact_programs.csv` from this checkout, whose run #202 rewrote, and requires the committed files byte for byte. The slow test takes one to two minutes per audit, so CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_audit_pins.py +``` diff --git a/reference_audit/2026-10-05/scripts/leaderboard_impact.py b/reference_audit/2026-10-05/scripts/leaderboard_impact.py index 5827b0a9..0341ecaf 100644 --- a/reference_audit/2026-10-05/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05/scripts/leaderboard_impact.py @@ -1,6 +1,17 @@ """Score the frozen run with and without the proposed exclusions, through the analyze CLI. -Copies the frozen run (predictions, references, scenarios, exclusions) to two scratch +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as PASS_COMMIT (release +dashboard-data-20260930, #187) holds it, and proposed_exclusions.json as AUDIT_COMMIT +(#202, which merged this audit) holds it. Later releases rewrite the working tree's +run: dashboard-data-20261006 (#202) installed all three of these records, so a copy +of today's run with them appended carries duplicate exclusions, which the analyze CLI +refuses. Each input in INPUTS is staged under ``/pass_inputs`` and must +match its pinned sha256, or the script stops before scoring anything. The scoring +code is the checkout's; the check below stops the script if that code no longer +scores the pinned run as the pinned payload records. + +Copies the staged run (predictions, references, scenarios, exclusions) to two scratch directories and never writes the snapshot. In one copy the exclusion record is unchanged; in the other the proposed records (proposed_exclusions.json) are appended. Each copy is scored with ``python -m policybench.cli analyze``, the command the freeze @@ -12,6 +23,11 @@ PYTHONPATH= /bin/python \\ reference_audit/2026-10-05/scripts/leaderboard_impact.py --scratch + +The checkout needs PASS_COMMIT and AUDIT_COMMIT in its history (a shallow clone needs +``git fetch --unshallow``). ``--out-dir`` writes the verification files elsewhere; +tests/test_reference_audit_pins.py regenerates them that way and requires the +committed ones byte for byte. """ from __future__ import annotations @@ -30,12 +46,52 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -PROPOSED = HERE / "proposed_exclusions.json" OUT_DIR = HERE / "verification" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_PATH = "reference_audit/2026-10-05" +PROPOSED_PATH = f"{AUDIT_PATH}/proposed_exclusions.json" +# Every file the script reads from the repository: path -> (commit, sha256). +# pass_inputs() stages each at the same path under its target. +INPUTS = { + f"{RUN_PATH}/data.json.gz": ( + PASS_COMMIT, + "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + ), + f"{RUN_PATH}/predictions.csv.gz": ( + PASS_COMMIT, + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12", + ), + f"{RUN_PATH}/reference_outputs.csv": ( + PASS_COMMIT, + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + ), + f"{RUN_PATH}/reference_outputs.csv.meta.json": ( + PASS_COMMIT, + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b", + ), + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + f"{RUN_PATH}/scenarios.csv": ( + PASS_COMMIT, + "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + ), + f"{RUN_PATH}/scenarios.csv.meta.json": ( + PASS_COMMIT, + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb", + ), + PROPOSED_PATH: ( + AUDIT_COMMIT, + "3c330177762c46fa5c52b02f9f943e9d5a65e15855280b64e9b462ab27413272", + ), +} +# Copied into each copy and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -88,16 +144,75 @@ def always_zero(run_dir: Path) -> dict[str, float]: } +def overlaps(scratch: Path, root: Path) -> bool: + """Whether ``scratch`` and ``root`` are one directory or one holds the other. + + Judged by filesystem identity (device and inode), not spelling, so case on a + case-insensitive filesystem or a link cannot hide an overlap. ``scratch`` need + not exist yet; its nearest existing ancestors decide. + """ + + def identities(path: Path) -> set[tuple[int, int]]: + found = set() + for part in (path, *path.parents): + try: + status = part.stat() + except OSError: + continue + found.add((status.st_dev, status.st_ino)) + return found + + root_status = root.stat() + if (root_status.st_dev, root_status.st_ino) in identities(scratch): + return True + try: + scratch_status = scratch.stat() + except OSError: + return False + return (scratch_status.st_dev, scratch_status.st_ino) in identities(root) + + def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def stage(target: Path, extra: list[dict] | None) -> Path: +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target + + +def pass_inputs(target: Path) -> Path: + """Stage every input in INPUTS at its repository path under ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, (commit, pinned) in INPUTS.items(): + git_input(commit, path, pinned, target / path) + return target + + +def stage(source: Path, target: Path, extra: list[dict] | None) -> Path: + """Copy the staged run at ``source`` to ``target`` and append ``extra``.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if extra: record = json.loads((target / "reference_exclusions.json").read_text()) record["exclusions"].extend(extra) @@ -139,8 +254,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -191,13 +307,25 @@ def program_rows(base: dict, proposed: dict) -> pd.DataFrame: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--scratch", required=True) + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() - scratch = Path(args.scratch) - proposals = json.loads(PROPOSED.read_text())["exclusions"] + scratch = Path(args.scratch).resolve() + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/, so scratch and + # the repository must not overlap: neither may lie inside the other. + if overlaps(scratch, ROOT): + parser.error(f"--scratch must not overlap the repository ({ROOT})") + inputs = pass_inputs(scratch / "pass_inputs") + source = inputs / RUN_PATH + proposals = json.loads((inputs / PROPOSED_PATH).read_text())["exclusions"] - base = analyze(stage(scratch / "base", None)) - check_reproduces_published(base) - proposed = analyze(stage(scratch / "proposed", proposals)) + base = analyze(stage(source, scratch / "base", None)) + check_reproduces_published(base, source) + proposed = analyze(stage(source, scratch / "proposed", proposals)) models = compare(base, proposed) zero = { @@ -206,9 +334,9 @@ def main() -> None: } leader = models.sort_values("exact_rank_proposed").index[0] programs = program_rows(base, proposed) - OUT_DIR.mkdir(parents=True, exist_ok=True) - models.to_csv(OUT_DIR / "leaderboard_impact_models.csv") - programs.to_csv(OUT_DIR / "leaderboard_impact_programs.csv") + out_dir.mkdir(parents=True, exist_ok=True) + models.to_csv(out_dir / "leaderboard_impact_models.csv") + programs.to_csv(out_dir / "leaderboard_impact_programs.csv") summary = { "proposed_exclusions": [ f"{p['scenario_id']}/{p['variable']}" for p in proposals @@ -255,7 +383,7 @@ def main() -> None: .head(5) .index.tolist(), } - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) with pd.option_context("display.width", 250, "display.max_rows", 100): diff --git a/reference_audit/2026-10-05/scripts/propose_exclusions.py b/reference_audit/2026-10-05/scripts/propose_exclusions.py index 2c820d0d..d60d2be6 100644 --- a/reference_audit/2026-10-05/scripts/propose_exclusions.py +++ b/reference_audit/2026-10-05/scripts/propose_exclusions.py @@ -12,26 +12,67 @@ ruling; ``decided_on`` is the date the proposal was drafted and a release sets it to the ruling's date. +Reads the pass's inputs from git, never from the working tree: the frozen run's +exclusions and references as PASS_COMMIT (release dashboard-data-20260930, #187) +holds them, and this audit's sweep and side readings as AUDIT_COMMIT (#202, which +merged this audit) holds them. Each must match its pinned sha256 in INPUTS, or the +script stops before writing anything. Release dashboard-data-20261006 (#202) +installed the three records, so on its run every output would land under +``already_excluded`` and the file would lose its proposals. + python reference_audit/2026-10-05/scripts/propose_exclusions.py + +``--out-dir`` writes the file elsewhere; tests/test_reference_audit_pins.py +regenerates it that way and requires the committed one byte for byte. """ from __future__ import annotations +import argparse +import hashlib +import io import json +import subprocess from pathlib import Path import pandas as pd HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +OUT_DIR = HERE +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" ) -SWEEP = HERE / "verification/sweep_salt_withholding.csv" -HOUSEHOLDS = HERE / "verification/sweep_salt_withholding_households.json" -VARIANTS = HERE / "verification/variants.json" -OUT = HERE / "proposed_exclusions.json" +AUDIT_PATH = "reference_audit/2026-10-05" +SWEEP_PATH = f"{AUDIT_PATH}/verification/sweep_salt_withholding.csv" +HOUSEHOLDS_PATH = f"{AUDIT_PATH}/verification/sweep_salt_withholding_households.json" +VARIANTS_PATH = f"{AUDIT_PATH}/verification/variants.json" +# Every file the script reads from the repository: path -> (commit, sha256). +INPUTS = { + f"{RUN_PATH}/reference_exclusions.json": ( + PASS_COMMIT, + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2", + ), + f"{RUN_PATH}/reference_outputs.csv": ( + PASS_COMMIT, + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466", + ), + SWEEP_PATH: ( + AUDIT_COMMIT, + "5918e349699471b8dcf9baa9584afdd583f8054624301839bb228bdc90a52d0c", + ), + HOUSEHOLDS_PATH: ( + AUDIT_COMMIT, + "b36669b4a6b43483fbec5ce18f2d8affbeda98df34d3523ae39ff2deec0e4e40", + ), + VARIANTS_PATH: ( + AUDIT_COMMIT, + "5cf0588e6071d4c35cfe6f8e07a8f3aaf8873dfbc8b7e34d496813d047141764", + ), +} DRAFTED_ON = "2026-10-05" ENGINE = "policyengine-us 2.15.17" FEDERAL = "federal_income_tax_before_refundable_credits" @@ -255,19 +296,57 @@ def already_note(row: pd.Series, household: dict, variant: dict) -> str: return text + "." +def git_bytes(commit: str, path: str, pinned: str) -> bytes: + """``path`` as ``commit`` holds it, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + return result.stdout + + +def pass_input(path: str) -> bytes: + """The input at ``path``, as INPUTS pins it.""" + commit, pinned = INPUTS[path] + return git_bytes(commit, path, pinned) + + def main() -> None: - sweep = pd.read_csv(SWEEP) + parser = argparse.ArgumentParser() + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write proposed_exclusions.json (default: %(default)s)", + ) + out = Path(parser.parse_args().out_dir) / "proposed_exclusions.json" + # Check every pin before reading anything, so a refused input writes nothing. + for path in INPUTS: + pass_input(path) + sweep = pd.read_csv(io.BytesIO(pass_input(SWEEP_PATH))) households = { - h["scenario_id"]: h for h in json.loads(HOUSEHOLDS.read_text())["households"] + h["scenario_id"]: h + for h in json.loads(pass_input(HOUSEHOLDS_PATH))["households"] } variants = { - v["scenario_id"]: v for v in json.loads(VARIANTS.read_text())["households"] + v["scenario_id"]: v for v in json.loads(pass_input(VARIANTS_PATH))["households"] } - existing = json.loads((RUN / "reference_exclusions.json").read_text())["exclusions"] + existing = json.loads(pass_input(f"{RUN_PATH}/reference_exclusions.json"))[ + "exclusions" + ] existing_keys = {(e["scenario_id"], e["variable"]): e for e in existing} - reference = pd.read_csv(RUN / "reference_outputs.csv").set_index( - ["scenario_id", "variable"] - )["value"] + reference = pd.read_csv( + io.BytesIO(pass_input(f"{RUN_PATH}/reference_outputs.csv")) + ).set_index(["scenario_id", "variable"])["value"] moved = sweep[sweep["liability_moved"]] proposals, already = [], [] @@ -344,8 +423,9 @@ def main() -> None: <= {(p["scenario_id"], p["variable"]) for p in proposals} ), } - OUT.write_text(json.dumps(payload, indent=2) + "\n") - print(f"{len(proposals)} proposed, {len(already)} already excluded -> {OUT}") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2) + "\n") + print(f"{len(proposals)} proposed, {len(already)} already excluded -> {out}") if __name__ == "__main__": diff --git a/tests/test_reference_audit_pins.py b/tests/test_reference_audit_pins.py new file mode 100644 index 00000000..9499e212 --- /dev/null +++ b/tests/test_reference_audit_pins.py @@ -0,0 +1,716 @@ +"""The October 5 audits' scripts read the inputs their passes read, pinned in git. + +Seven scripts in reference_audit/2026-10-05, 2026-10-05-louisiana, +2026-10-05-medicare-part-b and 2026-10-05-payroll read the frozen run as +PASS_COMMIT (release dashboard-data-20260930, #187) committed it, and their audits' +own inputs as #202 merged them. Each input is checked against its sha256: + +- the four ``scripts/leaderboard_impact.py``, which score the run; +- the SALT and Part B audits' ``scripts/propose_exclusions.py`` and the payroll + audit's ``scripts/model_answers.py``, which read it directly. + +None reads the working tree's run, which later releases rewrite. On the run #202 +left, three impact scripts and Part B's proposals stop on exclusions #202 installed. +The Louisiana impact script, the SALT proposals and the payroll answers rewrite +their committed files without an error. CI checks out full history, so git holds +every pinned input. +""" + +from __future__ import annotations + +import ast +import difflib +import gzip +import hashlib +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from functools import cache +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +# The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" +# Release dashboard-data-20261006 (#202) merged the four audits, installed their +# eight records and rewrote the run's payload and exclusions. +AUDIT_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = ( + "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d" +) +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +RUN_FILES = { + "data.json.gz", + "predictions.csv.gz", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", +} +# The run files #202 rewrote. +REWRITTEN = {f"{RUN_PATH}/data.json.gz", f"{RUN_PATH}/reference_exclusions.json"} +# #191's head; its reference_audit/2026-10-05/proposed_exclusions.json holds the +# records the Part B and payroll passes read. +PR191_HEAD = "8af912a062dd7ae1373de4c043ae2727d3406930" +SALT_RECORDS = "reference_audit/2026-10-05/proposed_exclusions.json" +EXCLUDE = frozenset({"reference_exclusions.json"}) +REGENERATE = frozenset({"reference_outputs.csv", "reference_outputs.csv.meta.json"}) +SALT = "reference_audit/2026-10-05" +LOUISIANA = "reference_audit/2026-10-05-louisiana" +PART_B = "reference_audit/2026-10-05-medicare-part-b" +PAYROLL = "reference_audit/2026-10-05-payroll" + + +def _run(*names: str) -> set[str]: + return {f"{RUN_PATH}/{name}" for name in names} + + +# The scoring scripts: the flags their evidence was written with, every input, +# each scratch copy's changed run files, and the summary fields that come from +# the inputs rather than from scoring. +IMPACT = { + SALT: SimpleNamespace( + flags=[], + inputs=_run(*RUN_FILES) | {SALT_RECORDS}, + copies={"base": frozenset(), "proposed": EXCLUDE}, + from_inputs=lambda summary: ( + summary["proposed_exclusions"], + summary["always_zero_baseline"], + ), + ), + LOUISIANA: SimpleNamespace( + flags=[], + inputs=_run(*RUN_FILES) + | {f"{LOUISIANA}/verification/sweep_la_standard_deduction.csv"}, + copies={ + "keep": frozenset(), + "ldr_published": REGENERATE, + "ldr_official": REGENERATE, + "hold_2025": REGENERATE, + "exclude": EXCLUDE, + }, + from_inputs=lambda summary: ( + summary["outputs"], + summary["reference_values"], + {k: v["exact_models"] for k, v in summary["options"].items()}, + ), + ), + PART_B: SimpleNamespace( + flags=["--weights-check"], + inputs=_run(*RUN_FILES, "analysis/impact_summary_by_model.csv") + | { + f"{PART_B}/proposed_exclusions.json", + f"{PART_B}/verification/sweep_part_b.csv", + f"{PART_B}/verification/inputs/pr191_proposed_exclusions.json", + }, + copies={ + "published": frozenset(), + "part_b": EXCLUDE, + "salt": EXCLUDE, + "salt_and_part_b": EXCLUDE, + "weights_2_15_17": REGENERATE, + }, + from_inputs=lambda summary: ( + summary["legacy_impact_summary_reproduced"], + [ + ( + case["case"], + case["exclusions_added"], + case["always_zero_baseline"], + case["legacy_impact_summary"], + ) + for case in summary["cases"] + ], + summary["impact_weights_check"]["impact_weights_replaced"], + summary["impact_weights_check"]["legacy_impact_summary"], + ), + code=["scripts/freeze_snapshot.py"], + ), + PAYROLL: SimpleNamespace( + flags=["--with-salt"], + inputs=_run(*RUN_FILES) + | { + f"{PAYROLL}/proposed_exclusions.json", + f"{PAYROLL}/proposed_regenerations.json", + SALT_RECORDS, + }, + copies={ + "published": frozenset(), + "salt": EXCLUDE, + "exclude": EXCLUDE, + "regenerate": REGENERATE, + "exclude+salt": EXCLUDE, + "regenerate+salt": EXCLUDE | REGENERATE, + }, + from_inputs=lambda summary: ( + summary["salt_commit"], + { + name: ( + case["changed_outputs"], + case["measured_against"], + case["always_zero_baseline"], + ) + for name, case in summary.items() + if isinstance(case, dict) + }, + ), + ), +} +# The scripts that read the run directly: every input, and every file they write +# (relative to --out-dir) with the committed file it must reproduce. +DIRECT = { + f"{SALT}/scripts/propose_exclusions.py": SimpleNamespace( + inputs=_run("reference_exclusions.json", "reference_outputs.csv") + | { + f"{SALT}/verification/sweep_salt_withholding.csv", + f"{SALT}/verification/sweep_salt_withholding_households.json", + f"{SALT}/verification/variants.json", + }, + outputs={"proposed_exclusions.json": SALT_RECORDS}, + ), + f"{PART_B}/scripts/propose_exclusions.py": SimpleNamespace( + inputs=_run("predictions.csv.gz", "reference_exclusions.json") + | { + "app/src/modelMeta.ts", + f"{PART_B}/verification/sweep_part_b_summary.json", + f"{PART_B}/verification/sweep_part_b_households.json", + f"{PART_B}/verification/inputs/pr191_proposed_exclusions.json", + }, + outputs={ + "proposed_exclusions.json": f"{PART_B}/proposed_exclusions.json", + "verification/model_answers.csv": ( + f"{PART_B}/verification/model_answers.csv" + ), + }, + ), + f"{PAYROLL}/scripts/model_answers.py": SimpleNamespace( + inputs=_run("data.json.gz") + | { + f"{PAYROLL}/verification/payroll_decomposition.csv", + f"{PAYROLL}/program_classification.json", + }, + outputs={ + "model_answers.csv": f"{PAYROLL}/verification/model_answers.csv", + "model_answers_summary.json": ( + f"{PAYROLL}/verification/model_answers_summary.json" + ), + }, + ), +} + + +def _load(script: str, root: Path = ROOT): + name = "pins_" + re.sub(r"\W", "_", script) + spec = importlib.util.spec_from_file_location(name, root / script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _impact_script(audit: str) -> str: + return f"{audit}/scripts/leaderboard_impact.py" + + +MODULES = { + **{_impact_script(audit): _load(_impact_script(audit)) for audit in IMPACT}, + **{script: _load(script) for script in DIRECT}, +} +SPECS = {**{_impact_script(audit): spec for audit, spec in IMPACT.items()}, **DIRECT} + + +def _committed(audit: str) -> list[str]: + """The evidence an impact script writes (the Louisiana log is its stdout).""" + return sorted( + path.name + for path in (ROOT / audit / "verification").glob("leaderboard_impact*") + if path.suffix != ".log" + ) + + +def _argv(monkeypatch, script: str, out: Path, *extra: str, root: Path = ROOT) -> None: + argv = [str(root / script), "--out-dir", str(out), *extra] + monkeypatch.setattr(sys, "argv", argv) + + +def _impact_argv( + monkeypatch, audit: str, scratch: Path, out: Path, root: Path = ROOT +) -> None: + flags = ["--scratch", str(scratch), *IMPACT[audit].flags] + _argv(monkeypatch, _impact_script(audit), out, *flags, root=root) + + +def _from_junk_checkout(monkeypatch, path: Path, script: str, code: list[str] = ()): + """``script`` loaded from a checkout whose working tree holds junk. + + The checkout shares this repository's objects, so git holds every pin. In its + working tree every file the script reads is junk. Every other file is absent, + except the script itself and the ``code`` it loads, copied from this checkout. + The working directory moves into the junk checkout before the script loads, so + the module's ROOT and HERE and every relative path, even one read at import, + point into the junk: a script that read the working tree instead of git would + fail or write something else. + """ + subprocess.run( + ["git", "clone", "--quiet", "--shared", "--no-checkout", str(ROOT), str(path)], + check=True, + ) + for name in MODULES[script].INPUTS: + junk = path / name + junk.parent.mkdir(parents=True, exist_ok=True) + junk.write_bytes(b"not the pass's input\n") + for name in (script, *code): + (path / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / name, path / name) + monkeypatch.chdir(path) + return _load(script, root=path) + + +@pytest.fixture +def scratch(): + """A directory removed as soon as the test ends. + + A staged run takes about 50 MB, and pytest would otherwise keep every test's + scratch for three runs. + """ + with tempfile.TemporaryDirectory() as directory: + yield Path(directory).resolve() + + +@cache +def _pinned_bytes(commit: str, path: str) -> bytes: + return subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + check=True, + ).stdout + + +def _fetch(module, commit: str, path: str, pinned: str, target: Path) -> None: + """Read one pinned input with the script's own git reader.""" + if hasattr(module, "git_input"): + module.git_input(commit, path, pinned, target) + else: + content = module.git_bytes(commit, path, pinned) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + + +@pytest.mark.parametrize("script", SPECS) +def test_the_pins_cover_every_file_the_script_reads(script): + module = MODULES[script] + assert set(module.INPUTS) == SPECS[script].inputs + for path, (commit, pinned) in module.INPUTS.items(): + on_run = path.startswith(RUN_PATH) or path == "app/src/modelMeta.ts" + assert commit == (PASS_COMMIT if on_run else AUDIT_COMMIT), path + assert hashlib.sha256(_pinned_bytes(commit, path)).hexdigest() == pinned, path + + +@pytest.mark.parametrize("audit", IMPACT) +def test_pass_inputs_stage_exactly_the_pinned_bytes(audit, scratch): + impact = MODULES[_impact_script(audit)] + assert set(impact.RUN_FILES) == RUN_FILES - {"data.json.gz"} + inputs = impact.pass_inputs(scratch / "pass_inputs") + staged = sorted( + path.relative_to(inputs).as_posix() + for path in inputs.rglob("*") + if path.is_file() + ) + assert staged == sorted(impact.INPUTS) + for path, (_, pinned) in impact.INPUTS.items(): + assert impact.sha256(inputs / path) == pinned, path + # The README's frozen run: 46 models, 1,928 scored outputs each. + payload = json.loads( + gzip.decompress((inputs / RUN_PATH / "data.json.gz").read_bytes()) + ) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + (inputs / "stray").write_text("left over\n") + impact.pass_inputs(inputs) + assert not (inputs / "stray").exists() + + +def _constants(path: Path) -> dict: + """``path``'s module-level literal assignments, read without importing it.""" + found = {} + for node in ast.parse(path.read_text()).body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + try: + found[target.id] = ast.literal_eval(node.value) + except (ValueError, TypeError, SyntaxError): + pass + return found + + +def test_the_pins_agree_across_the_scripts_and_with_the_adversary(): + """Every script pins a shared input to the same commit and bytes, and the run + is the one the reference adversary pins.""" + pins: dict[str, set] = {} + for module in MODULES.values(): + for path, pin in module.INPUTS.items(): + pins.setdefault(path, set()).add(pin) + assert {path: len(found) for path, found in pins.items() if len(found) > 1} == {} + assert pins[f"{RUN_PATH}/data.json.gz"] == {(PASS_COMMIT, FROZEN_SHA256)} + + # The adversary's scripts/pass_inputs.py (PolicyEngine/policybench#207) pins + # its whole run; before it, two of its scripts pin the payload alone. + scripts = ROOT / "reference_audit/2026-10-05-reference-adversary/scripts" + if (scripts / "pass_inputs.py").exists(): + adversary = _constants(scripts / "pass_inputs.py") + assert adversary["PASS_COMMIT"] == PASS_COMMIT + assert adversary["RUN_PATH"] == RUN_PATH + assert set(adversary["RUN_SHA256"]) == RUN_FILES + for name, pinned in adversary["RUN_SHA256"].items(): + assert pins[f"{RUN_PATH}/{name}"] == {(PASS_COMMIT, pinned)}, name + else: + for name in ("definition_conformance.py", "publication_sources.py"): + found = _constants(scripts / name).get("PAYLOAD_SHA256") + assert found == FROZEN_SHA256, name + + +def test_the_salt_records_are_one_file_pinned_four_ways(): + """The SALT audit's proposals, Part B's copy of #191's records (read by two + scripts) and the records the payroll script reads are the same bytes.""" + salt = MODULES[_impact_script(SALT)].INPUTS[SALT_RECORDS] + copy = f"{PART_B}/verification/inputs/pr191_proposed_exclusions.json" + assert MODULES[_impact_script(PART_B)].INPUTS[copy] == salt + assert MODULES[f"{PART_B}/scripts/propose_exclusions.py"].INPUTS[copy] == salt + assert MODULES[_impact_script(PAYROLL)].INPUTS[SALT_RECORDS] == salt + assert MODULES[_impact_script(PAYROLL)].SALT_COMMIT == PR191_HEAD + for script in (_impact_script(PART_B), f"{PART_B}/scripts/propose_exclusions.py"): + assert MODULES[script].SALT_SHA256 == salt[1] + + +def test_the_salt_pin_is_191s_head(): + """#191 was closed unmerged, so its head is only on its branch.""" + shown = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{PR191_HEAD}:{SALT_RECORDS}"], + capture_output=True, + ) + if shown.returncode: + pytest.skip(f"{PR191_HEAD[:12]} is not in this clone") + pinned = MODULES[_impact_script(PAYROLL)].SALT_SHA256 + assert hashlib.sha256(shown.stdout).hexdigest() == pinned + + +@pytest.mark.parametrize("commit", [PASS_COMMIT, AUDIT_COMMIT]) +def test_every_pinned_commit_is_on_main(commit): + """No pin depends on a branch that may be deleted.""" + pinned = {pin[0] for module in MODULES.values() for pin in module.INPUTS.values()} + assert pinned == {PASS_COMMIT, AUDIT_COMMIT} + ancestor = subprocess.run( + ["git", "-C", str(ROOT), "merge-base", "--is-ancestor", commit, "HEAD"] + ) + assert ancestor.returncode == 0, commit + + +@st.composite +def _edits(draw) -> tuple[str, str, str, int, int]: + """One script, one of its pinned inputs and one change to the input's bytes.""" + script = draw(st.sampled_from(sorted(MODULES))) + path = draw(st.sampled_from(sorted(MODULES[script].INPUTS))) + kind = draw(st.sampled_from(["flip", "truncate", "append"])) + commit = MODULES[script].INPUTS[path][0] + position = draw(st.integers(0, len(_pinned_bytes(commit, path)) - 1)) + return script, path, kind, position, draw(st.integers(1, 255)) + + +def _shown(stdout: bytes) -> SimpleNamespace: + return SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=stdout, stderr=b"" + ) + ) + + +@settings(max_examples=80, deadline=None) +@given(_edits()) +def test_any_changed_byte_in_a_pinned_input_is_refused(edit): + """Invariant: a script's git reader returns or writes only the pinned bytes.""" + script, path, kind, position, byte = edit + module = MODULES[script] + commit, pinned = module.INPUTS[path] + original = _pinned_bytes(commit, path) + if kind == "flip": + changed = bytearray(original) + changed[position] ^= byte + tampered = bytes(changed) + elif kind == "truncate": + tampered = original[:position] + else: + tampered = original + bytes([byte]) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "staged" / Path(path).name + with mock.patch.object(module, "subprocess", _shown(tampered)): + with pytest.raises(SystemExit, match="not the pinned"): + _fetch(module, commit, path, pinned, target) + assert not target.exists() + # The unchanged bytes pass the same check. + with mock.patch.object(module, "subprocess", _shown(original)): + _fetch(module, commit, path, pinned, target) + assert target.read_bytes() == original + + +def _refusals(script: str) -> list[tuple[str, tuple[str, str], str]]: + module = MODULES[script] + rewritten = sorted(REWRITTEN & set(module.INPUTS))[0] + pinned = module.INPUTS[rewritten][1] + current = hashlib.sha256(_pinned_bytes(AUDIT_COMMIT, rewritten)).hexdigest() + own = sorted(path for path in module.INPUTS if path.startswith("reference_audit")) + return [ + # The file as #202 rewrote it, which the working tree holds today. + ( + rewritten, + (AUDIT_COMMIT, pinned), + f"has sha256 {current}, not the pinned {pinned}", + ), + # The audit's own input before the audit existed. + (own[0], (PASS_COMMIT, module.INPUTS[own[0]][1]), "cannot read"), + (own[0], (AUDIT_COMMIT, "0" * 64), f"not the pinned {'0' * 64}"), + (rewritten, ("0" * 40, pinned), "cannot read"), + ] + + +@pytest.mark.parametrize( + ("audit", "path", "pin", "message"), + [ + (audit, *refusal) + for audit in IMPACT + for refusal in _refusals(_impact_script(audit)) + ], +) +def test_pass_inputs_refuse_anything_but_the_pinned_bytes( + scratch, monkeypatch, audit, path, pin, message +): + impact = MODULES[_impact_script(audit)] + monkeypatch.setitem(impact.INPUTS, path, pin) + with pytest.raises(SystemExit, match=re.escape(message)): + impact.pass_inputs(scratch / "pass_inputs") + + +def _never(name: str): + def call(*args, **kwargs): + raise AssertionError(f"{name} ran after the inputs were refused") + + return call + + +@pytest.mark.parametrize( + ("audit", "path"), + [(audit, path) for audit in IMPACT for path in IMPACT[audit].inputs], +) +def test_any_refused_input_stops_the_impact_script_before_it_scores( + scratch, monkeypatch, audit, path +): + impact = MODULES[_impact_script(audit)] + commit, _ = impact.INPUTS[path] + monkeypatch.setitem(impact.INPUTS, path, (commit, "0" * 64)) + for name in ("analyze", "always_zero", "stage"): + if hasattr(impact, name): + monkeypatch.setattr(impact, name, _never(name)) + out = scratch / "out" + _impact_argv(monkeypatch, audit, scratch / "scratch", out) + with pytest.raises(SystemExit, match=f"{re.escape(path)} has sha256 .* not the"): + impact.main() + assert not out.exists() + + +@pytest.mark.parametrize( + ("script", "path"), + [(script, path) for script in DIRECT for path in DIRECT[script].inputs], +) +def test_any_refused_input_stops_the_direct_script_before_it_writes( + scratch, monkeypatch, script, path +): + module = MODULES[script] + commit, _ = module.INPUTS[path] + monkeypatch.setitem(module.INPUTS, path, (commit, "0" * 64)) + out = scratch / "out" + _argv(monkeypatch, script, out) + with pytest.raises(SystemExit, match=f"{re.escape(path)} has sha256 .* not the"): + module.main() + assert not out.exists() + + +@pytest.mark.parametrize( + ("script", "path", "pin", "message"), + [(script, *refusal) for script in DIRECT for refusal in _refusals(script)], +) +def test_a_refused_input_stops_the_direct_script_before_it_writes( + scratch, monkeypatch, script, path, pin, message +): + module = MODULES[script] + monkeypatch.setitem(module.INPUTS, path, pin) + out = scratch / "out" + _argv(monkeypatch, script, out) + with pytest.raises(SystemExit, match=re.escape(message)): + module.main() + assert not out.exists() + + +def _case_insensitive() -> bool: + """Whether the filesystem holding the repository ignores case (macOS's default).""" + swapped = Path(str(ROOT).swapcase()) + return swapped.exists() and swapped.samefile(ROOT) + + +def _overlap(kind: str, scratch: Path) -> Path: + if kind == "inside": + return ROOT / "results/scratch" + if kind == "the repository": + return ROOT + if kind == "a parent": + return ROOT.parent + if kind == "a link to a parent": + link = scratch / "link" + link.symlink_to(ROOT.parent, target_is_directory=True) + return link + # A parent spelled in another case, which only a case-insensitive filesystem + # treats as the same directory. + if not _case_insensitive(): + pytest.skip("the filesystem is case-sensitive") + return Path(str(ROOT.parent).swapcase()) + + +@pytest.mark.parametrize("audit", IMPACT) +@pytest.mark.parametrize( + "kind", + ["inside", "the repository", "a parent", "a link to a parent", "another case"], +) +def test_scratch_overlapping_the_repository_is_refused( + scratch, monkeypatch, audit, kind +): + """stage() and pass_inputs() delete scratch/, so a scratch that holds the + repository, however it is spelled, could delete it.""" + impact = MODULES[_impact_script(audit)] + monkeypatch.setattr(impact, "pass_inputs", _never("pass_inputs")) + _impact_argv(monkeypatch, audit, _overlap(kind, scratch), scratch / "out") + with pytest.raises(SystemExit) as stopped: + impact.main() + assert stopped.value.code == 2 + assert not (ROOT / "results/scratch").exists() + + +@pytest.mark.parametrize("audit", IMPACT) +def test_the_impact_script_scores_only_the_staged_inputs(scratch, monkeypatch, audit): + """Run from a checkout whose working tree is junk, every copy still starts from + the pinned bytes, everything computed from the inputs alone matches the + committed evidence, and the full set of evidence files is written. Only the + analyze CLI is stubbed; the always-zero baseline and Part B's legacy impact + summary run for real on the staged copies.""" + spec = IMPACT[audit] + script = _impact_script(audit) + impact = _from_junk_checkout( + monkeypatch, scratch / "co", script, getattr(spec, "code", []) + ) + scored: dict[str, dict[str, str]] = {} + published: dict = {} + + def analyze(run_dir: Path) -> dict: + scored[run_dir.name] = { + name: impact.sha256(run_dir / name) for name in impact.RUN_FILES + } + if not published: + payload = run_dir.parent / "pass_inputs" / RUN_PATH / "data.json.gz" + published.update(json.loads(gzip.decompress(payload.read_bytes()))) + return published + + monkeypatch.setattr(impact, "analyze", analyze) + out = scratch / "out" + _impact_argv(monkeypatch, audit, scratch / "scratch", out, root=scratch / "co") + impact.main() + + pinned = {name: impact.INPUTS[f"{RUN_PATH}/{name}"][1] for name in impact.RUN_FILES} + assert set(scored) == set(spec.copies) + for copy, digests in scored.items(): + changed = {name for name in pinned if digests[name] != pinned[name]} + assert changed == spec.copies[copy], copy + assert sorted(path.name for path in out.iterdir()) == _committed(audit) + summary = json.loads((out / "leaderboard_impact.json").read_text()) + committed = json.loads( + (ROOT / audit / "verification/leaderboard_impact.json").read_text() + ) + assert summary["published_reproduced"] is True + assert spec.from_inputs(summary) == spec.from_inputs(committed) + + +@pytest.mark.parametrize("script", DIRECT) +def test_the_direct_script_reproduces_its_files_from_a_junk_checkout( + scratch, monkeypatch, script +): + """Run for real from a checkout whose working tree is junk: every file the + script writes is the committed one, byte for byte.""" + module = _from_junk_checkout(monkeypatch, scratch / "co", script) + out = scratch / "out" + _argv(monkeypatch, script, out, root=scratch / "co") + module.main() + written = sorted( + path.relative_to(out).as_posix() for path in out.rglob("*") if path.is_file() + ) + assert written == sorted(DIRECT[script].outputs) + for name, committed in DIRECT[script].outputs.items(): + assert (out / name).read_bytes() == (ROOT / committed).read_bytes(), name + + +@pytest.mark.slow +@pytest.mark.parametrize("audit", IMPACT) +def test_regeneration_reproduces_the_committed_evidence(scratch, audit): + """Score every copy for real and require the committed files byte for byte. + + Runs from this checkout, whose run a later release has rewritten, so the + evidence can only match if the script reads the pinned inputs. One to two + minutes of ``policybench analyze`` runs per audit, so CI deselects it. Run it + with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_audit_pins.py + """ + out = scratch / "out" + result = subprocess.run( + [sys.executable, str(ROOT / _impact_script(audit))] + + ["--scratch", str(scratch / "scratch"), "--out-dir", str(out)] + + IMPACT[audit].flags, + cwd=ROOT, + env=dict(os.environ, PYTHONPATH=str(ROOT)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + committed = _committed(audit) + assert sorted(path.name for path in out.iterdir()) == committed + if audit == LOUISIANA: + (out / "leaderboard_impact.log").write_text(result.stdout) + committed.append("leaderboard_impact.log") + verification = ROOT / audit / "verification" + differing = [ + name + for name in committed + if (out / name).read_bytes() != (verification / name).read_bytes() + ] + if differing: + first = differing[0] + diff = difflib.unified_diff( + (verification / first).read_text().splitlines(), + (out / first).read_text().splitlines(), + f"committed/{first}", + f"regenerated/{first}", + lineterm="", + ) + pytest.fail( + f"{len(differing)} files differ: {differing}\n" + "\n".join(list(diff)[:60]) + )