From 59c32d60ff07fe3df7cedc8da20b3cdf59362e67 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 14:06:19 -0400 Subject: [PATCH 1/7] Pin the reference adversary's leaderboard impact to the pass's inputs leaderboard_impact.py copied the working tree's run files and compared against the working tree's payload, so regenerating after release dashboard-data-20261006 (#202) scored that release's payload and exclusions and silently rewrote 19 of the 23 committed verification/leaderboard_impact* files (1,928 -> 1,920 scored cells). The script now stages the run bundle from 8b4c0ca1 (release dashboard-data-20260930) and proposed_changes.json from 4db91b5f (#200) with git show, checks each file against its pinned sha256 and stops before scoring on any mismatch. It reads only the staged copies. --out-dir writes the evidence elsewhere. tests/test_reference_adversary_impact.py checks the pins, the refusals (including #202's payload), that a checkout whose working-tree run and proposals are junk still scores the pinned bytes, and (slow) that a full regeneration reproduces all 23 committed files byte for byte. The README lists every pinned input with its sha256 and the scripts that read it, and the Reproduce steps restore the whole run and annotations directory from 8b4c0ca1 for the scripts that still read the working tree. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-05-reference-adversary/README.md | 39 ++- .../scripts/leaderboard_impact.py | 141 ++++++++--- tests/test_reference_adversary_impact.py | 234 ++++++++++++++++++ 3 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 tests/test_reference_adversary_impact.py diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index 26e0b045..c4a0707d 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -46,7 +46,25 @@ PolicyBench's diagnosis judge (`docs/audit.md`) explains why models miss. Its pr Inputs: -- Payload: `data.json.gz` of release dashboard-data-20260930, as committed at `8b4c0ca1` (#187), sha256 `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18`. Release dashboard-data-20261006 (#202) later rewrote the working tree's payload and annotations; the tests and the Reproduce steps read the pass's inputs from `8b4c0ca1`. +- The frozen run and its annotations, as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Release dashboard-data-20261006 (#202) later rewrote the working tree's payload, exclusion record and annotations, so reproducing the pass needs each file below as `8b4c0ca1` holds it, with that sha256. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + + | Input at `8b4c0ca1` | sha256 | Read by | + |---|---|---| + | `/data.json.gz`, the payload | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | `consensus-flags`, `adversary-prepare`, `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/predictions.csv.gz` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | `leaderboard_impact.py` | + | `/reference_outputs.csv` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/reference_outputs.csv.meta.json` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/reference_exclusions.json` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/scenarios.csv` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | + | `/scenarios.csv.meta.json` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | + | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | + +- `proposed_changes.json` as #200 merged it at `4db91b5f`, sha256 `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf`, read by `leaderboard_impact.py`. +- Who checks the pins: + - `leaderboard_impact.py` stages every input it reads from git and stops on any sha256 mismatch before scoring. + - `definition_conformance.py` and `publication_sources.py` stop when the working tree's payload is not `1e029aaa…`, but read their other inputs from the working tree unchecked. + - `engine_probe.py` and `build_proposals.py` check no input's sha256. + - `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. `tests/test_reference_adversary_impact.py` checks the impact script's pins and that it ignores the working tree's run and proposals. - Top 5 by the payload's `modelStats` order: gpt-6-sol, claude-opus-5.5, gpt-5.6-sol, gpt-6-luna and claude-sonnet-5.5. - `consensus_flags.json` holds the default run, and `consensus_flags_prototype.json` reproduces the 2026-10-05 prototype. - `verification/flags_table.md` lists all 61 cells, each with its cluster, its exact-model count and whether the prototype flagged it. @@ -286,7 +304,7 @@ The root cause is a gap in scenario generation and definition scope, not an engi ## Leaderboard impact -`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. +`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. It takes the run and the proposals from git at their pinned commits (Inputs above), not from the working tree. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. | Variant | Scored cells | Exact-rate change across models | Leader (gpt-6-sol) exact | Exact-rank changes | |---|---:|---|---|---| @@ -443,11 +461,18 @@ The pass's own expectations are not part of either ruling (`status.not_ruled`). | `proposed_changes.json` | Exclusion and regeneration records per root cause, and the rulings | | `scripts/` | The scripts that wrote the files above | -The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. +The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_impact.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. ## Reproduce -`/data.json.gz` and `annotations//` must hold the pass's inputs as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. A worktree at `8b4c0ca1` has them, or `git show 8b4c0ca1:` each file. +Every step reads the pass's inputs (the table under [Flag parameters](#flag-parameters)) as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. `scripts/leaderboard_impact.py` stages its own inputs from git, so it runs from any checkout that has `8b4c0ca1` and `4db91b5f` in its history. The commands below read the payload and annotations at the paths they are given. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `build_proposals.py` read the working tree's `/`. So first restore every file in the table (the run directory and the run's annotations directory) from `8b4c0ca1`, and put today's back when done: + +```bash +run=paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace +git restore --source 8b4c0ca1 -- "$run" "annotations/${run##*/}" +# ... the steps below ... +git restore -- "$run" "annotations/${run##*/}" +``` ```bash uv run policybench consensus-flags --payload /data.json.gz \ @@ -481,3 +506,9 @@ uv run python reference_audit/2026-10-05-reference-adversary/scripts/search_expo Preparing `runs/claude` today reproduces its `cases.jsonl` and derivations byte for byte. Each stage-1 prompt then differs from the committed one in one line, because the source rule now asks for `blocked_domains`. The re-judge's preparation reproduces `runs/claude-rejudge`'s inputs byte for byte. The scripts in `scripts/` regenerate the verification files; each script's docstring gives its command. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment. + +`tests/test_reference_adversary_impact.py` regenerates the `verification/leaderboard_impact*` files into a scratch directory and requires the committed ones byte for byte. The test takes about five minutes, so it is marked slow and CI deselects it: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_impact.py +``` diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py index d728c5ee..23439c69 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py @@ -1,9 +1,19 @@ """Score the frozen run as published and under each alternative in proposed_changes.json. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch -directories and never writes the snapshot. Each copy is scored with -``python -m policybench.cli analyze``, the command the freeze runs (the method of -PolicyEngine/policybench#191 and #194): +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as commit PASS_COMMIT +(release dashboard-data-20260930, #187) holds it, and proposed_changes.json as +PROPOSALS_COMMIT (#200) holds it. Later releases rewrite the working tree's run +(dashboard-data-20261006, #202, rewrote its payload and exclusions), and scoring +those would regenerate different evidence into this pass's verification files. +Each file is staged under ``/pass_inputs`` and must match its pinned +sha256, or the script stops before scoring anything. The scoring code is the +checkout's; ``published`` below stops the script if that code no longer scores +the pinned run as the pinned payload records. + +Copies the staged run to scratch directories and never writes the snapshot. Each +copy is scored with ``python -m policybench.cli analyze``, the command the freeze +runs (the method of PolicyEngine/policybench#191 and #194): ``published`` Unchanged. It must reproduce the published payload's scoring: every modelStats field @@ -24,6 +34,11 @@ PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py \\ --scratch + +The checkout needs PASS_COMMIT and PROPOSALS_COMMIT in its history (a shallow +clone needs ``git fetch --unshallow``). ``--out-dir`` writes the verification +files elsewhere; tests/test_reference_adversary_impact.py regenerates them that +way and requires the committed ones byte for byte. """ from __future__ import annotations @@ -42,12 +57,36 @@ HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -PROPOSALS = HERE / "proposed_changes.json" OUT_DIR = HERE / "verification" +# The pass's inputs, pinned by commit and sha256. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +RUN_SHA256 = { + "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "predictions.csv.gz": ( + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12" + ), + "reference_outputs.csv": ( + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" + ), + "reference_outputs.csv.meta.json": ( + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" + ), + "reference_exclusions.json": ( + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" + ), + "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +PROPOSALS_COMMIT = "4db91b5f10581947f60a07f899e0ec2867b0405e" +PROPOSALS_PATH = "reference_audit/2026-10-05-reference-adversary/proposed_changes.json" +PROPOSALS_SHA256 = "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf" +PROPOSALS_NAME = Path(PROPOSALS_PATH).name +# Copied into each variant and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -74,13 +113,46 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def proposals() -> dict: - return json.loads(PROPOSALS.read_text())["root_causes"] +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.stdout) + return target -def variants() -> dict[str, dict[str, list[dict]]]: - """Each variant's exclusion and regeneration records.""" - causes = proposals() +def pass_inputs(target: Path) -> Path: + """Stage the pass's run bundle under ``target/run`` and its proposals beside it.""" + if target.exists(): + shutil.rmtree(target) + for name, pinned in RUN_SHA256.items(): + git_input(PASS_COMMIT, f"{RUN_PATH}/{name}", pinned, target / "run" / name) + git_input( + PROPOSALS_COMMIT, PROPOSALS_PATH, PROPOSALS_SHA256, target / PROPOSALS_NAME + ) + return target + + +def proposals(path: Path) -> dict: + return json.loads(path.read_text())["root_causes"] + + +def variants(path: Path) -> dict[str, dict[str, list[dict]]]: + """Each variant's exclusion and regeneration records, from ``path``'s proposals.""" + causes = proposals(path) plan: dict[str, dict[str, list[dict]]] = {} for cause, spec in causes.items(): plan[f"exclude:{cause}"] = {"exclude": spec["exclusions"], "regenerate": []} @@ -140,12 +212,13 @@ def always_zero(run_dir: Path) -> dict[str, float]: } -def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: +def stage(source: Path, target: Path, records: dict[str, list[dict]] | None) -> Path: + """Copy the staged run at ``source`` to ``target`` and apply ``records``.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if not records: return target if records["exclude"]: @@ -165,7 +238,7 @@ def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: raise SystemExit(f"{key}: frozen value differs from the reference CSV") reference.at[row, "value"] = repr(float(change["regenerated_value"])) reference.to_csv(target / "reference_outputs.csv", index=False) - before = (RUN / "reference_outputs.csv").read_text().splitlines() + before = (source / "reference_outputs.csv").read_text().splitlines() after = (target / "reference_outputs.csv").read_text().splitlines() changed = [i for i, (a, b) in enumerate(zip(before, after)) if a != b] if len(before) != len(after) or len(changed) != len(changes): @@ -211,8 +284,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -331,27 +405,36 @@ def main() -> None: parser.add_argument( "--only", nargs="*", help="score only these variants (default: all)" ) + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() scratch = Path(args.scratch).resolve() - # stage() deletes and rewrites scratch/; never inside the repository. + out_dir = Path(args.out_dir) + # stage() and pass_inputs() delete and rewrite scratch/; never inside + # the repository. if scratch == ROOT.resolve() or scratch.is_relative_to(ROOT.resolve()): parser.error(f"--scratch must be outside the repository ({ROOT})") - plan = variants() + inputs = pass_inputs(scratch / "pass_inputs") + source = inputs / "run" + plan = variants(inputs / PROPOSALS_NAME) unknown = sorted(set(args.only or ()) - set(plan)) if unknown: parser.error(f"unknown variant(s) {unknown}; choose from {sorted(plan)}") - base_dir = stage(scratch / "published", None) + base_dir = stage(source, scratch / "published", None) base = analyze(base_dir) - check_reproduces_published(base) + check_reproduces_published(base, source) zero_published = always_zero(base_dir) - OUT_DIR.mkdir(parents=True, exist_ok=True) - summary: dict = {"published_reproduced": True, "proposals": PROPOSALS.name} + out_dir.mkdir(parents=True, exist_ok=True) + summary: dict = {"published_reproduced": True, "proposals": PROPOSALS_NAME} if args.only: plan = {name: plan[name] for name in args.only} for name, records in plan.items(): - run_dir = stage(scratch / name.replace(":", "__"), records) + run_dir = stage(source, scratch / name.replace(":", "__"), records) other = analyze(run_dir) models = compare(base, other) programs = program_rows(base, other) @@ -363,8 +446,8 @@ def main() -> None: } ) slug = name.replace(":", "_") - models.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_models.csv") - programs.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_programs.csv") + models.to_csv(out_dir / f"leaderboard_impact_{slug}_models.csv") + programs.to_csv(out_dir / f"leaderboard_impact_{slug}_programs.csv") summary[name] = summarize(models, base, other, zero, cells) summary[name]["programs"] = programs.round(6).to_dict(orient="index") with pd.option_context("display.width", 250, "display.max_rows", 100): @@ -385,7 +468,7 @@ def main() -> None: ].round(4) ) print(programs.round(4)) - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(json.dumps({k: v for k, v in summary.items() if k != "proposals"}, indent=2)[:4000]) diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py new file mode 100644 index 00000000..2f94c6cf --- /dev/null +++ b/tests/test_reference_adversary_impact.py @@ -0,0 +1,234 @@ +"""The reference adversary's leaderboard impact comes from the pass's pinned inputs. + +``reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py`` +reads the frozen run as PASS_COMMIT (release dashboard-data-20260930, #187) +committed it and the proposals as #200 merged them, each checked against its +sha256. It never reads the working tree's run, which later releases rewrite +(#202 did). CI checks out full history, so git holds every pinned input. +""" + +from __future__ import annotations + +import difflib +import gzip +import importlib.util +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" +SCRIPT = AUDIT / "scripts/leaderboard_impact.py" +VERIFICATION = AUDIT / "verification" +# The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" +# Release dashboard-data-20261006 (#202) rewrote the run's payload and exclusions. +RELEASE_20261006_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = ( + "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d" +) +COMMITTED = sorted(path.name for path in VERIFICATION.glob("leaderboard_impact*")) + + +def _load_script(): + spec = importlib.util.spec_from_file_location("adversary_impact", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +impact = _load_script() + + +def _tampered_checkout(path: Path) -> Path: + """A checkout sharing this repository's objects whose working tree holds junk. + + Its run directory and proposals are not the pass's inputs, so a script that + read them instead of git would fail or regenerate different evidence. + """ + subprocess.run( + ["git", "clone", "--quiet", "--shared", "--no-checkout", str(ROOT), str(path)], + check=True, + ) + for name in impact.RUN_SHA256: + junk = path / impact.RUN_PATH / name + junk.parent.mkdir(parents=True, exist_ok=True) + junk.write_bytes(b"not the pass's input\n") + proposals = path / impact.PROPOSALS_PATH + proposals.parent.mkdir(parents=True, exist_ok=True) + proposals.write_text('{"root_causes": {}}\n') + return path + + +def _argv(monkeypatch, tmp_path: Path) -> Path: + out = tmp_path / "out" + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + ) + return out + + +def test_the_pins_cover_every_file_the_script_reads(tmp_path): + assert impact.PASS_COMMIT == PASS_COMMIT + assert impact.RUN_SHA256["data.json.gz"] == FROZEN_SHA256 + assert set(impact.RUN_SHA256) == {*impact.RUN_FILES, "data.json.gz"} + + inputs = impact.pass_inputs(tmp_path / "pass_inputs") + staged = sorted( + path.relative_to(inputs).as_posix() + for path in inputs.rglob("*") + if path.is_file() + ) + assert staged == sorted( + [f"run/{name}" for name in impact.RUN_SHA256] + [impact.PROPOSALS_NAME] + ) + for name, pinned in impact.RUN_SHA256.items(): + assert impact.sha256(inputs / "run" / name) == pinned + assert impact.sha256(inputs / impact.PROPOSALS_NAME) == impact.PROPOSALS_SHA256 + # The README's frozen run: 46 models, 1,928 scored cells. + payload = json.loads(gzip.decompress((inputs / "run/data.json.gz").read_bytes())) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + (inputs / "stray").write_text("left over\n") + impact.pass_inputs(inputs) + assert not (inputs / "stray").exists() + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ( + "PASS_COMMIT", + RELEASE_20261006_COMMIT, + f"data.json.gz has sha256 {RELEASE_20261006_SHA256}, " + f"not the pinned {FROZEN_SHA256}", + ), + ("PROPOSALS_SHA256", "0" * 64, f"not the pinned {'0' * 64}"), + ("PASS_COMMIT", "0" * 40, "cannot read"), + ], +) +def test_pass_inputs_refuse_anything_but_the_pinned_bytes( + tmp_path, monkeypatch, attribute, value, message +): + monkeypatch.setattr(impact, attribute, value) + with pytest.raises(SystemExit, match=re.escape(message)): + impact.pass_inputs(tmp_path / "pass_inputs") + + +def test_a_refused_input_stops_the_script_before_it_scores(tmp_path, monkeypatch): + def analyze(run_dir): + raise AssertionError(f"scored {run_dir} after its inputs were refused") + + monkeypatch.setattr(impact, "PASS_COMMIT", RELEASE_20261006_COMMIT) + monkeypatch.setattr(impact, "analyze", analyze) + out = _argv(monkeypatch, tmp_path) + with pytest.raises(SystemExit, match=f"not the pinned {FROZEN_SHA256}"): + impact.main() + assert not out.exists() + + +def test_the_script_scores_only_the_staged_inputs(tmp_path, monkeypatch): + """Run from a checkout whose working-tree run is junk, every variant still + starts from the pinned bytes and the full set of evidence files is written.""" + monkeypatch.setattr(impact, "ROOT", _tampered_checkout(tmp_path / "checkout")) + scored: dict[str, dict[str, str]] = {} + published: dict = {} + + def analyze(run_dir: Path) -> dict: + scored[run_dir.name] = { + name: impact.sha256(run_dir / name) for name in impact.RUN_FILES + } + if not published: + payload = run_dir.parent / "pass_inputs/run/data.json.gz" + published.update(json.loads(gzip.decompress(payload.read_bytes()))) + return published + + monkeypatch.setattr(impact, "analyze", analyze) + monkeypatch.setattr( + impact, "always_zero", lambda run_dir: {"exact": 0.0, "within1pct": 0.0} + ) + out = _argv(monkeypatch, tmp_path) + impact.main() + + pinned = {name: impact.RUN_SHA256[name] for name in impact.RUN_FILES} + assert scored.pop("published") == pinned + exclusions = {"reference_exclusions.json"} + regenerations = {"reference_outputs.csv", "reference_outputs.csv.meta.json"} + expected = { + "recommended": exclusions | regenerations, + "exclude_all": exclusions, + } + assert set(scored) >= set(expected) + for variant, digests in scored.items(): + changed = {name for name in pinned if digests[name] != pinned[name]} + if variant.startswith("exclude__"): + assert changed == exclusions, variant + elif variant.startswith("regenerate__"): + assert changed == regenerations, variant + else: + assert changed == expected[variant], variant + assert sorted(path.name for path in out.iterdir()) == COMMITTED + summary = json.loads((out / "leaderboard_impact.json").read_text()) + assert summary["published_reproduced"] is True + assert summary["proposals"] == "proposed_changes.json" + + +@pytest.mark.slow +def test_regeneration_reproduces_the_committed_evidence(tmp_path): + """Score every variant for real and require the committed files byte for byte. + + About five minutes of ``policybench analyze`` runs, so CI deselects it. Run it + with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_adversary_impact.py + """ + assert len(COMMITTED) == 23 + out = tmp_path / "out" + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + cwd=ROOT, + env=dict(os.environ, PYTHONPATH=str(ROOT)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + assert sorted(path.name for path in out.iterdir()) == COMMITTED + differing = [ + name + for name in COMMITTED + if (out / name).read_bytes() != (VERIFICATION / name).read_bytes() + ] + if differing: + first = differing[0] + diff = difflib.unified_diff( + (VERIFICATION / first).read_text().splitlines(), + (out / first).read_text().splitlines(), + f"committed/{first}", + f"regenerated/{first}", + lineterm="", + ) + pytest.fail( + f"{len(differing)} files differ: {differing}\n" + "\n".join(list(diff)[:60]) + ) From 95aaa8fb49af1d4304a13afa18bb5ead34af88a1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 14:10:48 -0400 Subject: [PATCH 2/7] Property-test the input pins and cross-check the payload pin A Hypothesis test flips, truncates or extends any pinned run file and requires git_input to refuse it without writing the target, and to accept the unchanged bytes. A differential check requires the payload pin to equal definition_conformance.py's and publication_sources.py's. Co-Authored-By: Claude Opus 5.5 --- tests/test_reference_adversary_impact.py | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py index 2f94c6cf..612d9f42 100644 --- a/tests/test_reference_adversary_impact.py +++ b/tests/test_reference_adversary_impact.py @@ -9,6 +9,7 @@ from __future__ import annotations +import ast import difflib import gzip import importlib.util @@ -17,9 +18,15 @@ import re import subprocess import sys +import tempfile +from functools import cache from pathlib import Path +from types import SimpleNamespace +from unittest import mock import pytest +from hypothesis import given, settings +from hypothesis import strategies as st ROOT = Path(__file__).resolve().parents[1] AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" @@ -109,6 +116,83 @@ def test_the_pins_cover_every_file_the_script_reads(tmp_path): assert not (inputs / "stray").exists() +def test_the_payload_pin_matches_the_engine_side_scripts_pins(): + """The engine-side scripts pin the same payload (read without importing them).""" + for name in ("definition_conformance.py", "publication_sources.py"): + tree = ast.parse((AUDIT / "scripts" / name).read_text()) + pins = [ + node.value.value + for node in tree.body + if isinstance(node, ast.Assign) + and [target.id for target in node.targets] == ["PAYLOAD_SHA256"] + ] + assert pins == [impact.RUN_SHA256["data.json.gz"]], name + + +@cache +def _pinned_bytes(name: str) -> bytes: + return subprocess.run( + ["git", "-C", str(ROOT), "show", f"{PASS_COMMIT}:{impact.RUN_PATH}/{name}"], + capture_output=True, + check=True, + ).stdout + + +@st.composite +def _edits(draw) -> tuple[str, str, int, int]: + """One pinned run file and one change to its bytes: flip, cut or extend.""" + name = draw(st.sampled_from(sorted(impact.RUN_SHA256))) + kind = draw(st.sampled_from(["flip", "truncate", "append"])) + position = draw(st.integers(0, len(_pinned_bytes(name)) - 1)) + return name, kind, position, draw(st.integers(1, 255)) + + +@settings(max_examples=40, deadline=None) +@given(_edits()) +def test_any_changed_byte_in_a_pinned_input_is_refused_unwritten(edit): + """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" + name, kind, position, byte = edit + original = _pinned_bytes(name) + if kind == "flip": + changed = bytearray(original) + changed[position] ^= byte + tampered = bytes(changed) + elif kind == "truncate": + tampered = original[:position] + else: + tampered = original + bytes([byte]) + shown = SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=tampered, stderr=b"" + ) + ) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / name + with mock.patch.object(impact, "subprocess", shown): + with pytest.raises(SystemExit, match="not the pinned"): + impact.git_input( + PASS_COMMIT, + f"{impact.RUN_PATH}/{name}", + impact.RUN_SHA256[name], + target, + ) + assert not target.exists() + # The unchanged bytes pass the same check. + unchanged = SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=original, stderr=b"" + ) + ) + with mock.patch.object(impact, "subprocess", unchanged): + impact.git_input( + PASS_COMMIT, + f"{impact.RUN_PATH}/{name}", + impact.RUN_SHA256[name], + target, + ) + assert target.read_bytes() == original + + @pytest.mark.parametrize( ("attribute", "value", "message"), [ From 54a6a2bacdf19e6cf7ea62d295a28be32468d67d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 14:39:08 -0400 Subject: [PATCH 3/7] Pin the reference adversary's other scripts to the pass's inputs definition_conformance.py, publication_sources.py, engine_probe.py and build_proposals.py read the working tree's run, which release dashboard-data-20261006 (#202) rewrote (payload and exclusion record), and the two engine-side scripts built their reference system from the working tree's convention modules, one of which #204 rewrote. Each now stages every input through scripts/pass_inputs.py, which writes `git show :` to scratch and stops unless the bytes match the pinned sha256, before the script computes or writes anything. leaderboard_impact.py (#205) uses the same module, so the pins live in one place. build_proposals.py now writes to a required --out and refuses the committed proposed_changes.json, which is pinned at 3a6e5920. No evidence file, reference, exclusion or score changes. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-05-reference-adversary/README.md | 101 ++- .../scripts/build_proposals.py | 95 ++- .../scripts/definition_conformance.py | 117 +-- .../scripts/engine_probe.py | 33 +- .../scripts/leaderboard_impact.py | 96 +-- .../scripts/pass_inputs.py | 185 +++++ .../scripts/publication_sources.py | 67 +- tests/test_reference_adversary_impact.py | 133 +-- tests/test_reference_adversary_inputs.py | 755 ++++++++++++++++++ tests/working_tree_fence.py | 91 +++ 10 files changed, 1381 insertions(+), 292 deletions(-) create mode 100644 reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py create mode 100644 tests/test_reference_adversary_inputs.py create mode 100644 tests/working_tree_fence.py diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index c4a0707d..b2f3df44 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -46,25 +46,52 @@ PolicyBench's diagnosis judge (`docs/audit.md`) explains why models miss. Its pr Inputs: -- The frozen run and its annotations, as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Release dashboard-data-20261006 (#202) later rewrote the working tree's payload, exclusion record and annotations, so reproducing the pass needs each file below as `8b4c0ca1` holds it, with that sha256. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. - - | Input at `8b4c0ca1` | sha256 | Read by | +- The pass read the frozen run and its annotations as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Later commits rewrote several of the working tree's copies: release dashboard-data-20261006 (#202) the run's payload, exclusion record and annotations, and #204 one of the reference system's convention modules. So every script in `scripts/` reads the inputs below only through `scripts/pass_inputs.py`. It writes `git show :` to scratch and stops (SystemExit) unless the bytes match the pinned sha256, and each script stages everything it reads before it computes or writes anything. No script reads the working tree's copy. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + + | Input | Commit | sha256 | Read by | + |---|---|---|---| + | `/data.json.gz`, the payload | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | `consensus-flags`, `adversary-prepare`, `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | `leaderboard_impact.py` | + | `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | + | `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | + | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `8b4c0ca1` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | + | `policybench/benchmark_specs.json`, the output definitions | `8b4c0ca1` | `ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5` | `definition_conformance.py` | + | `reference_audit/2026-10-05-payroll/program_classification.json`, #194's classification of state paid-leave programs | `049f4f09` | `282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a` | `definition_conformance.py` | + | `reference_audit/2026-10-05-reference-adversary/proposed_changes.json` | `4db91b5f` | `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf` | `leaderboard_impact.py` | + | `reference_audit/2026-10-05-reference-adversary/verification/definition_conformance.json` | `4db91b5f` | `05e503f1b291c505b90182395c4afb88844dcb45fb2952c004ca09d4273894a0` | `build_proposals.py` | + +- The reference system is policyengine-us 2.15.17 plus `latest_final`, built from the convention modules below as `8b4c0ca1` held them. They are read by `definition_conformance.py`, `publication_sources.py` and `engine_probe.py`, and each worker process stages its own copy. `latest_final` loads `latest_conventions` (the nine `latest_c_*` modules) and `latest_md_local_output_scope`. #204 later rewrote `latest_alt_snap_mortgage_residence.py`, which `latest_final` does not load, but `verification/definition_conformance.json` records every module's sha256. + + | Module | Commit | sha256 | |---|---|---| - | `/data.json.gz`, the payload | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | `consensus-flags`, `adversary-prepare`, `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | - | `/predictions.csv.gz` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | `leaderboard_impact.py` | - | `/reference_outputs.csv` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | - | `/reference_outputs.csv.meta.json` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | - | `/reference_exclusions.json` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | - | `/scenarios.csv` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | - | `/scenarios.csv.meta.json` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | - | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | - -- `proposed_changes.json` as #200 merged it at `4db91b5f`, sha256 `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf`, read by `leaderboard_impact.py`. + | `reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json` | `8b4c0ca1` | `7cf93799aad6a8c3a6fc70e7df4acc6043202a1ecc083f2c3101f76b87a85eae` | + | `reference_audit/2026-09-28/fixes/alt_conventions_r25.py` | `8b4c0ca1` | `0f77fdd44e758581f7e563b0f7b70c4d4359a833b08a4ac45a614c380938775e` | + | `reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py` | `8b4c0ca1` | `3e71bab1db867988544e6f87f6dda4dc7c719d51320fbbb7558cd574a0374f6e` | + | `reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py` | `8b4c0ca1` | `d8a314093466caf5fe0a226f1060acba2e4364c4b87befbd80639ebf41cc4a2c` | + | `reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py` | `8b4c0ca1` | `00cd13886a8c3d03c4e84944d35f7e8a1448af26838e8bc0761b914b887c70cd` | + | `reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py` | `8b4c0ca1` | `5e6538a903112243ad85051ccba26b6133bdfbd5b9e5ba3dfa81ba690c061b32` | + | `reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py` | `8b4c0ca1` | `f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2` | + | `reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py` | `8b4c0ca1` | `39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5` | + | `reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py` | `8b4c0ca1` | `7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e` | + | `reference_audit/2026-09-28/fixes/latest_c_md_2026.py` | `8b4c0ca1` | `8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76` | + | `reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py` | `8b4c0ca1` | `787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e` | + | `reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py` | `8b4c0ca1` | `02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522` | + | `reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py` | `8b4c0ca1` | `0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916` | + | `reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py` | `8b4c0ca1` | `3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd` | + | `reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py` | `8b4c0ca1` | `4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce` | + | `reference_audit/2026-09-28/fixes/latest_conventions.py` | `8b4c0ca1` | `4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb` | + | `reference_audit/2026-09-28/fixes/latest_final.py` | `8b4c0ca1` | `dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e` | + | `reference_audit/2026-09-28/fixes/latest_map_stated_hours.py` | `8b4c0ca1` | `b210e780a31cfa0a3a0da2c49a8068ee14f00029b3ed39c00be67a1e917eda21` | + | `reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py` | `8b4c0ca1` | `53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd` | + | `reference_audit/2026-09-28/fixes/latest_state_conventions.py` | `8b4c0ca1` | `56e62e399d20ba8c8d9c5dc9bc7ea6681e74d2b529fa38561dbc8e8b2a90b442` | + - Who checks the pins: - - `leaderboard_impact.py` stages every input it reads from git and stops on any sha256 mismatch before scoring. - - `definition_conformance.py` and `publication_sources.py` stop when the working tree's payload is not `1e029aaa…`, but read their other inputs from the working tree unchecked. - - `engine_probe.py` and `build_proposals.py` check no input's sha256. - - `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. `tests/test_reference_adversary_impact.py` checks the impact script's pins and that it ignores the working tree's run and proposals. + - `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py` and `leaderboard_impact.py` stage every input above that they read, and stop on any sha256 mismatch before they compute or write anything. `pass_inputs.py` is the only file that holds a pin. + - `tests/test_reference_adversary_inputs.py` checks that each commit holds its pinned bytes, that the committed evidence records the same hashes, and that this table lists each pin with the scripts that stage it. It also checks that any changed byte is refused and that each script refuses #202's run before it computes or writes. An audit hook (`tests/working_tree_fence.py`) fails any open of the working tree's copy of a pinned input while the scripts stage and read their inputs, and in every process, workers included, of the slow regenerations below. `tests/test_reference_adversary_impact.py` checks the impact script the same way, from a checkout whose working-tree run and proposals are junk. + - `consensus-flags` and `adversary-prepare` read the payload and annotations at the paths they are given, and record the payload's path and sha256 in their output. `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. - Top 5 by the payload's `modelStats` order: gpt-6-sol, claude-opus-5.5, gpt-5.6-sol, gpt-6-luna and claude-sonnet-5.5. - `consensus_flags.json` holds the default run, and `consensus_flags_prototype.json` reproduces the 2026-10-05 prototype. - `verification/flags_table.md` lists all 61 cells, each with its cluster, its exact-model count and whether the prototype flagged it. @@ -459,18 +486,20 @@ The pass's own expectations are not part of either ruling (`status.not_ruled`). | `verification/leaderboard_impact*` | Leaderboard impact of each alternative | | `verification/pytest_*.txt` | Test runs: after the rebase, the rolling-pool change, and the final run (391 passed) | | `proposed_changes.json` | Exclusion and regeneration records per root cause, and the rulings | -| `scripts/` | The scripts that wrote the files above | +| `scripts/` | The scripts that wrote the files above, and `pass_inputs.py`, which pins and stages their inputs | -The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_impact.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. +The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_inputs.py`, `test_reference_adversary_impact.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. ## Reproduce -Every step reads the pass's inputs (the table under [Flag parameters](#flag-parameters)) as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. `scripts/leaderboard_impact.py` stages its own inputs from git, so it runs from any checkout that has `8b4c0ca1` and `4db91b5f` in its history. The commands below read the payload and annotations at the paths they are given. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `build_proposals.py` read the working tree's `/`. So first restore every file in the table (the run directory and the run's annotations directory) from `8b4c0ca1`, and put today's back when done: +The scripts in `scripts/` stage the pass's inputs from git (the tables under [Flag parameters](#flag-parameters)), so they run from any checkout that has `8b4c0ca1`, `049f4f09` and `4db91b5f` in its history, whatever its working tree's run, convention modules and records hold. The code they run is the checkout's. A shallow clone needs `git fetch --unshallow`. Each script's docstring gives its command. + +The CLI steps read the payload and annotations at the paths they are given and record the payload's path in their output (`consensus_flags.json` `source`). So they need the working tree's run and annotations as `8b4c0ca1` committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. Restore both from `8b4c0ca1` first, and put today's back when done: ```bash run=paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace git restore --source 8b4c0ca1 -- "$run" "annotations/${run##*/}" -# ... the steps below ... +# ... the CLI steps below ... git restore -- "$run" "annotations/${run##*/}" ``` @@ -505,10 +534,34 @@ uv run python reference_audit/2026-10-05-reference-adversary/scripts/search_expo Preparing `runs/claude` today reproduces its `cases.jsonl` and derivations byte for byte. Each stage-1 prompt then differs from the committed one in one line, because the source rule now asks for `blocked_domains`. The re-judge's preparation reproduces `runs/claude-rejudge`'s inputs byte for byte. -The scripts in `scripts/` regenerate the verification files; each script's docstring gives its command. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment. +`definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment, which `uv sync --locked --extra dev --python 3.12` provides. Write their output to scratch and compare it with the committed files: + +```bash +a=reference_audit/2026-10-05-reference-adversary +export OPENBLAS_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. +uv run python $a/scripts/definition_conformance.py --out-dir +uv run python $a/scripts/publication_sources.py --out-dir +uv run python $a/scripts/engine_probe.py scenario_043 \ + --variables co_sales_tax_refund co_refundable_credits co_eitc co_tabor_cash_back state_refundable_credits \ + --parameters gov.states.co.tax.income.credits.sales_tax_refund.amount.flat_amount_enabled \ + gov.states.co.tax.income.credits.sales_tax_refund.amount.amount \ + --out /co_sales_tax_refund_scenario_043.json +uv run python $a/scripts/leaderboard_impact.py --scratch --out-dir /impact +uv run python $a/scripts/build_proposals.py --out /proposed_changes.json +``` + +Each committed probe records its own arguments: `scenario_id`, `period`, the keys of `variables` and `parameters`, and `counterfactual.set`. `build_proposals.py` never writes the committed `proposed_changes.json`, which `leaderboard_impact.py` and the Haiku 5.5 release pin at `3a6e5920…`. It refuses that path for `--out`. + +The tests regenerate every one of these outputs into scratch and require the committed file byte for byte: + +- `build_proposals.py`: `proposed_changes.json`, in the fast suite. +- `definition_conformance.py`: both files, except two lines of `definition_conformance.json`: `run.seconds`, the wall time, and `run.script_sha256`, the script's own sha256. The committed file records `33ce4c3e…`, the script as #200 merged it; the test requires a regeneration to record the current script's hash. +- `publication_sources.py`: both files, except `meta.seconds`. +- `engine_probe.py`: all 8 probes, from the arguments each records. +- `leaderboard_impact.py`: all 23 `verification/leaderboard_impact*` files. -`tests/test_reference_adversary_impact.py` regenerates the `verification/leaderboard_impact*` files into a scratch directory and requires the committed ones byte for byte. The test takes about five minutes, so it is marked slow and CI deselects it: +The engine-side regenerations take about 15 minutes on a loaded machine, so they are marked slow and CI deselects them: ```bash -OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_impact.py +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_inputs.py tests/test_reference_adversary_impact.py ``` diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py index aa32e595..6bd9a9aa 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py @@ -1,4 +1,4 @@ -"""Write proposed_changes.json: the exclusion and regeneration records this pass proposes. +"""Build proposed_changes.json: the exclusion and regeneration records this pass proposes. Verification (verification/independent/*.md) found four engine defects behind four scored references and one definition-scope ambiguity behind four more. Each root cause gets an @@ -17,22 +17,43 @@ from verification/definition_conformance.json (the engine with the dependent as the head of their own tax unit). - python reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py +Every input is the pass's, staged from git by pass_inputs.py and checked against its pinned +sha256 before anything is built: the frozen run's references and exclusion record as release +dashboard-data-20260930 (8b4c0ca1) committed them, and verification/definition_conformance.json +as #200 merged it (4db91b5f). The working tree's run, which later releases rewrite (#202 +rewrote its exclusion record), is never read. + +The committed proposed_changes.json is pinned at sha256 PROPOSED_SHA256 below: +leaderboard_impact.py reads it at that hash, and the Haiku 5.5 release pins it too. So the +script never writes it. It writes to --out, which must be another path, and a regeneration is +checked by comparing the two: + + python reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py \\ + --out /proposed_changes.json + cmp /proposed_changes.json \\ + reference_audit/2026-10-05-reference-adversary/proposed_changes.json """ from __future__ import annotations +import argparse import csv import json +import sys +import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -OUT = HERE / "proposed_changes.json" +# The committed records, which the script never writes. +PROPOSED = HERE / "proposed_changes.json" +PROPOSED_SHA256 = pass_inputs.PROPOSALS_SHA256 +# The frozen run's files the script reads, staged from git. +RUN_FILES = ("reference_outputs.csv", "reference_exclusions.json") +CONFORMANCE_NAME = Path(pass_inputs.CONFORMANCE_PATH).name DRAFTED = "2026-10-06" ENGINE = "policyengine-us 2.15.17" STATE_TAX = "state_income_tax_before_refundable_credits" @@ -215,23 +236,36 @@ } -def frozen_references() -> dict[tuple[str, str], float]: - with (RUN / "reference_outputs.csv").open() as handle: +def stage_inputs(target: Path) -> Path: + """Stage the run's files under ``target/run`` and the conformance scan beside them, + each refused unless it matches its pin.""" + pass_inputs.stage_run(target / "run", RUN_FILES) + pass_inputs.git_input( + pass_inputs.RECORDS_COMMIT, + pass_inputs.CONFORMANCE_PATH, + pass_inputs.CONFORMANCE_SHA256, + target / CONFORMANCE_NAME, + ) + return target + + +def frozen_references(run: Path) -> dict[tuple[str, str], float]: + with (run / "reference_outputs.csv").open() as handle: return { (row["scenario_id"], row["variable"]): float(row["value"]) for row in csv.DictReader(handle) } -def excluded() -> set[tuple[str, str]]: - record = json.loads((RUN / "reference_exclusions.json").read_text()) +def excluded(run: Path) -> set[tuple[str, str]]: + record = json.loads((run / "reference_exclusions.json").read_text()) return {(e["scenario_id"], e["variable"]) for e in record["exclusions"]} -def household_scope_values() -> dict[tuple[str, str], float]: +def household_scope_values(inputs: Path) -> dict[tuple[str, str], float]: """Reference plus the dependent's own return, from the conformance scan.""" - scan = json.loads((HERE / "verification/definition_conformance.json").read_text()) - frozen = frozen_references() + scan = json.loads((inputs / CONFORMANCE_NAME).read_text()) + frozen = frozen_references(inputs / "run") values = {} for row in scan["household_scope_records"]: for output in (STATE_TAX, FED_TAX): @@ -240,10 +274,11 @@ def household_scope_values() -> dict[tuple[str, str], float]: return values -def main() -> None: - frozen = frozen_references() - already = excluded() - scope = household_scope_values() +def build(inputs: Path) -> dict: + """The records, from the inputs ``stage_inputs`` staged under ``inputs``.""" + frozen = frozen_references(inputs / "run") + already = excluded(inputs / "run") + scope = household_scope_values(inputs) out = { "schema_version": 1, "status": { @@ -342,8 +377,28 @@ def main() -> None: "exclusions": exclusions, "regenerations": regenerations, } - OUT.write_text(json.dumps(out, indent=2) + "\n") - print(f"wrote {OUT.relative_to(ROOT)}") + return out + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument( + "--out", + required=True, + help="where to write the records; never the committed proposed_changes.json", + ) + args = parser.parse_args() + out = Path(args.out) + if out.resolve() == PROPOSED.resolve(): + parser.error( + f"{PROPOSED.relative_to(ROOT)} is pinned at sha256 {PROPOSED_SHA256}; " + "write elsewhere and compare" + ) + with tempfile.TemporaryDirectory(prefix="build_proposals_inputs_") as scratch: + records = build(stage_inputs(Path(scratch))) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(records, indent=2) + "\n") + print(f"wrote {out}") if __name__ == "__main__": diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py index 7a1e344c..a6205f24 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py @@ -27,7 +27,16 @@ PolicyEngine/policybench#194 (commit 049f4f09, reference_audit/2026-10-05-payroll/program_classification.json), read from git. -Run from a policybench checkout with the policyengine-us 2.15.17 venv: +Every input is the pass's, staged from git by pass_inputs.py and checked against its +pinned sha256 before anything is computed: the frozen run's payload, references, +reference sidecar and scenarios and the output definitions (benchmark_specs.json) as +release dashboard-data-20260930 (8b4c0ca1) committed them, #194's table, and +latest_final with its parts as 8b4c0ca1 held them (each worker stages its own copy). +The working tree's run, which later releases rewrite, is never read. The run record +names each input by its repository path. + +Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout +needs 8b4c0ca1 and 049f4f09 in its history): PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ /.venv-pe21517/bin/python \\ @@ -38,13 +47,13 @@ from __future__ import annotations import argparse +import atexit import copy import gzip import hashlib import importlib.util import json import shutil -import subprocess import sys import tempfile import time @@ -55,16 +64,17 @@ import numpy as np import pandas as pd +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + ROOT = Path(__file__).resolve().parents[3] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +# The frozen run's files main reads, staged from git. +RUN_FILES = ( + "data.json.gz", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "scenarios.csv", ) -PAYLOAD = RUN / "data.json.gz" -PAYLOAD_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" -AUDIT_0928 = ROOT / "reference_audit/2026-09-28" -AUDIT_0922 = ROOT / "reference_audit/2026-09-22" -SPECS = ROOT / "policybench/benchmark_specs.json" YEAR = 2026 PERIOD = str(YEAR) INSTANT = "2026-01-01" @@ -72,9 +82,10 @@ TOLERANCE = 0.005 # sweep.py's rename for the 1.755.4 harness; sweep_latest.py kept it on 2.15.17. RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} -LAW_COMMIT = "049f4f09" -LAW_PATH = "reference_audit/2026-10-05-payroll/program_classification.json" -LAW_SOURCE = f"PolicyEngine/policybench#194 @ {LAW_COMMIT}:{LAW_PATH}" +LAW_SOURCE = ( + f"PolicyEngine/policybench#194 @ {pass_inputs.LAW_COMMIT[:8]}:" + f"{pass_inputs.LAW_PATH}" +) # Gross income sources the engine counts as earned (irs_gross_income reads # irs_employment_income, itself max(0, employment_income - pre_tax_contributions)). EARNED_SOURCES = ( @@ -102,6 +113,8 @@ ) _SYSTEM = None +# The directory this process's reference system was built from. +_FIX_DIR = None def _sha256(path: Path) -> str: @@ -109,11 +122,30 @@ def _sha256(path: Path) -> str: def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads.""" + """latest_final and its parts, plus the sales tax table the IRS module reads, + staged from git (pass_inputs.FIXES_SHA256).""" + global _FIX_DIR target = Path(tempfile.mkdtemp(prefix="definition_conformance_fixes_")) - for path in (AUDIT_0928 / "fixes").glob("*.py"): - shutil.copy2(path, target / path.name) - shutil.copy2(AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json", target) + _FIX_DIR = pass_inputs.stage_fixes(target) + return _FIX_DIR + + +def stage_inputs(target: Path) -> Path: + """Stage what main reads besides the engine: the run's files under ``run/``, the + output definitions and #194's law table, each refused unless it matches its pin.""" + pass_inputs.stage_run(target / "run", RUN_FILES) + pass_inputs.git_input( + pass_inputs.PASS_COMMIT, + pass_inputs.SPECS_PATH, + pass_inputs.SPECS_SHA256, + target / Path(pass_inputs.SPECS_PATH).name, + ) + pass_inputs.git_input( + pass_inputs.LAW_COMMIT, + pass_inputs.LAW_PATH, + pass_inputs.LAW_SHA256, + target / Path(pass_inputs.LAW_PATH).name, + ) return target @@ -347,16 +379,9 @@ def scenario_cells(job: dict) -> dict: } -def law_classifications() -> tuple[dict, dict]: - """Variable -> cited classification, from the payroll audit's program table.""" - try: - raw = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{LAW_COMMIT}:{LAW_PATH}"], - check=True, - capture_output=True, - ).stdout - except (subprocess.CalledProcessError, FileNotFoundError) as error: - return {}, {"source": LAW_SOURCE, "available": False, "error": str(error)} +def law_classifications(path: Path) -> tuple[dict, dict]: + """Variable -> cited classification, from the payroll audit's staged program table.""" + raw = path.read_bytes() table = json.loads(raw) classifications = {} entries = list(table.get("programs", [])) + list( @@ -496,19 +521,22 @@ def main() -> None: started = time.time() out_dir = Path(args.out_dir) - if _sha256(PAYLOAD) != PAYLOAD_SHA256: - raise SystemExit(f"{PAYLOAD} does not match its pinned sha256") - payload = json.loads(gzip.decompress(PAYLOAD.read_bytes())) - meta = json.loads((RUN / "reference_outputs.csv.meta.json").read_text()) + staged = Path(tempfile.mkdtemp(prefix="definition_conformance_inputs_")) + atexit.register(shutil.rmtree, staged, True) + stage_inputs(staged) + run = staged / "run" + specs_path = staged / Path(pass_inputs.SPECS_PATH).name + payload = json.loads(gzip.decompress((run / "data.json.gz").read_bytes())) + meta = json.loads((run / "reference_outputs.csv.meta.json").read_text()) programs = list(meta["programs"]) specs = [ spec - for spec in json.loads(SPECS.read_text())["specs"]["policybench"]["countries"][ - "us" - ] + for spec in json.loads(specs_path.read_text())["specs"]["policybench"][ + "countries" + ]["us"] if spec["id"] in programs ] - reference = pd.read_csv(RUN / "reference_outputs.csv") + reference = pd.read_csv(run / "reference_outputs.csv") indexed = reference.set_index(["scenario_id", "variable"])["value"] scored = _scored_cells(payload) if args.scenarios: @@ -532,7 +560,7 @@ def main() -> None: graph = variable_graph_from_system( system, INSTANT, roots=[spec["pe_variable"] for spec in specs] ) - laws, law_meta = law_classifications() + laws, law_meta = law_classifications(staged / Path(pass_inputs.LAW_PATH).name) # A first, cell-free pass names the variables whose values measure a rule's # effect when that is not the component itself. @@ -583,7 +611,7 @@ def main() -> None: if graph[spec["pe_variable"]].entity == "tax_unit" } - scenarios = pd.read_csv(RUN / "scenarios.csv") + scenarios = pd.read_csv(run / "scenarios.csv") scored_by_scenario: dict[str, list[str]] = {} for scenario_id, output_id in scored: scored_by_scenario.setdefault(scenario_id, []).append(output_id) @@ -660,7 +688,6 @@ def main() -> None: ) ) script = Path(__file__).resolve() - fixes = sorted((AUDIT_0928 / "fixes").glob("*.py")) report["run"] = { "script": str(script.relative_to(ROOT)), "script_sha256": _sha256(script), @@ -670,15 +697,15 @@ def main() -> None: "reference_system": "policyengine-us 2.15.17 + latest_final " "(reference_audit/2026-09-28/fixes)", "fixes_sha256": { - str(path.relative_to(ROOT)): _sha256(path) - for path in fixes + [AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json"] + path: _sha256(_FIX_DIR / Path(path).name) + for path in pass_inputs.FIXES_SHA256 }, "instant": INSTANT, - "payload": str(PAYLOAD.relative_to(ROOT)), - "payload_sha256": PAYLOAD_SHA256, - "reference_outputs_sha256": _sha256(RUN / "reference_outputs.csv"), - "scenarios_sha256": _sha256(RUN / "scenarios.csv"), - "benchmark_specs_sha256": _sha256(SPECS), + "payload": f"{pass_inputs.RUN_PATH}/data.json.gz", + "payload_sha256": _sha256(run / "data.json.gz"), + "reference_outputs_sha256": _sha256(run / "reference_outputs.csv"), + "scenarios_sha256": _sha256(run / "scenarios.csv"), + "benchmark_specs_sha256": _sha256(specs_path), "household_prompt_phrase": HOUSEHOLD_PROMPT_PHRASE, "household_prompt_in_every_scenario": household_prompt, "reproduction": { diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py index da568751..7c1bf483 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py @@ -14,6 +14,16 @@ It is the engine-evidence probe for the reference adversary's non-holding verdicts; it reads no verdict and changes no reference. +Every input is the pass's, staged from git by pass_inputs.py and checked against +its pinned sha256 before the reference system is built: the frozen run's +scenarios, references and exclusion record as release dashboard-data-20260930 +(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them. +The working tree's run, which later releases rewrite (#202 rewrote its exclusion +record), is never read. Each committed probe records its own arguments +(``scenario_id``, ``period``, the ``variables`` and ``parameters`` keys and +``counterfactual.set``), and tests/test_reference_adversary_inputs.py regenerates +every probe from them. + OPENBLAS_NUM_THREADS=1 PYTHONPATH=. <2.15.17 venv>/bin/python \\ reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py \\ scenario_018 --variables az_standard_deduction az_taxable_income \\ @@ -24,10 +34,13 @@ from __future__ import annotations import argparse +import atexit import copy import importlib.util import json +import shutil import sys +import tempfile from importlib.metadata import version from pathlib import Path @@ -35,8 +48,19 @@ import pandas as pd HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + YEAR = 2026 INSTANT = "2026-01-01" +# The frozen run's files the probe reads, staged from git. +RUN_FILES = ("scenarios.csv", "reference_outputs.csv", "reference_exclusions.json") + + +def stage_inputs(target: Path) -> Path: + """Stage the run's files the probe reads under ``target``, each refused unless it + matches its pin.""" + return pass_inputs.stage_run(target, RUN_FILES) def _conformance(): @@ -164,8 +188,11 @@ def main() -> None: parser.add_argument("--out") args = parser.parse_args() + run = Path(tempfile.mkdtemp(prefix="engine_probe_inputs_")) + atexit.register(shutil.rmtree, run, True) + stage_inputs(run) conformance = _conformance() - scenarios = pd.read_csv(conformance.RUN / "scenarios.csv") + scenarios = pd.read_csv(run / "scenarios.csv") rows = scenarios[scenarios["scenario_id"] == args.scenario_id] if rows.empty: raise SystemExit(f"no {args.scenario_id} in the frozen run") @@ -176,8 +203,8 @@ def main() -> None: system = conformance._system() sim = conformance._simulation(situation) - reference = pd.read_csv(conformance.RUN / "reference_outputs.csv") - exclusions = json.loads((conformance.RUN / "reference_exclusions.json").read_text()) + reference = pd.read_csv(run / "reference_outputs.csv") + exclusions = json.loads((run / "reference_exclusions.json").read_text()) excluded = { (record["scenario_id"], record["variable"]) for record in ( diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py index 23439c69..1ba7c54e 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py @@ -3,13 +3,13 @@ Reads the pass's inputs from git, never from the working tree: the frozen run (payload, predictions, references, scenarios, exclusions) as commit PASS_COMMIT (release dashboard-data-20260930, #187) holds it, and proposed_changes.json as -PROPOSALS_COMMIT (#200) holds it. Later releases rewrite the working tree's run -(dashboard-data-20261006, #202, rewrote its payload and exclusions), and scoring -those would regenerate different evidence into this pass's verification files. -Each file is staged under ``/pass_inputs`` and must match its pinned -sha256, or the script stops before scoring anything. The scoring code is the -checkout's; ``published`` below stops the script if that code no longer scores -the pinned run as the pinned payload records. +RECORDS_COMMIT (#200) holds it, both pinned in pass_inputs.py. Later releases +rewrite the working tree's run (dashboard-data-20261006, #202, rewrote its payload +and exclusions), and scoring those would regenerate different evidence into this +pass's verification files. Each file is staged under ``/pass_inputs`` and +must match its pinned sha256, or the script stops before scoring anything. The +scoring code is the checkout's; ``published`` below stops the script if that code +no longer scores the pinned run as the pinned payload records. Copies the staged run to scratch directories and never writes the snapshot. Each copy is scored with ``python -m policybench.cli analyze``, the command the freeze @@ -35,7 +35,7 @@ reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py \\ --scratch -The checkout needs PASS_COMMIT and PROPOSALS_COMMIT in its history (a shallow +The checkout needs PASS_COMMIT and RECORDS_COMMIT in its history (a shallow clone needs ``git fetch --unshallow``). ``--out-dir`` writes the verification files elsewhere; tests/test_reference_adversary_impact.py regenerates them that way and requires the committed ones byte for byte. @@ -45,7 +45,6 @@ import argparse import gzip -import hashlib import json import os import shutil @@ -55,37 +54,13 @@ import pandas as pd +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] OUT_DIR = HERE / "verification" -# The pass's inputs, pinned by commit and sha256. -PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" -RUN_PATH = ( - "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -RUN_SHA256 = { - "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", - "predictions.csv.gz": ( - "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12" - ), - "reference_outputs.csv": ( - "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" - ), - "reference_outputs.csv.meta.json": ( - "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" - ), - "reference_exclusions.json": ( - "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" - ), - "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", - "scenarios.csv.meta.json": ( - "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" - ), -} -PROPOSALS_COMMIT = "4db91b5f10581947f60a07f899e0ec2867b0405e" -PROPOSALS_PATH = "reference_audit/2026-10-05-reference-adversary/proposed_changes.json" -PROPOSALS_SHA256 = "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf" -PROPOSALS_NAME = Path(PROPOSALS_PATH).name +PROPOSALS_NAME = Path(pass_inputs.PROPOSALS_PATH).name # Copied into each variant and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", @@ -109,39 +84,16 @@ RANKED = ("exact", "within1pct", "score") -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: - """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" - result = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], - capture_output=True, - ) - if result.returncode: - raise SystemExit( - f"cannot read {path} at {commit[:12]}; fetch full history " - f"(git fetch --unshallow): {result.stderr.decode().strip()}" - ) - digest = hashlib.sha256(result.stdout).hexdigest() - if digest != pinned: - raise SystemExit( - f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" - ) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(result.stdout) - return target - - -def pass_inputs(target: Path) -> Path: +def stage_inputs(target: Path) -> Path: """Stage the pass's run bundle under ``target/run`` and its proposals beside it.""" if target.exists(): shutil.rmtree(target) - for name, pinned in RUN_SHA256.items(): - git_input(PASS_COMMIT, f"{RUN_PATH}/{name}", pinned, target / "run" / name) - git_input( - PROPOSALS_COMMIT, PROPOSALS_PATH, PROPOSALS_SHA256, target / PROPOSALS_NAME + pass_inputs.stage_run(target / "run") + pass_inputs.git_input( + pass_inputs.RECORDS_COMMIT, + pass_inputs.PROPOSALS_PATH, + pass_inputs.PROPOSALS_SHA256, + target / PROPOSALS_NAME, ) return target @@ -247,7 +199,9 @@ def stage(source: Path, target: Path, records: dict[str, list[dict]] | None) -> ) meta_path = target / "reference_outputs.csv.meta.json" meta = json.loads(meta_path.read_text()) - meta["reference_csv_sha256"] = sha256(target / "reference_outputs.csv") + meta["reference_csv_sha256"] = pass_inputs.sha256( + target / "reference_outputs.csv" + ) meta_path.write_text(json.dumps(meta, indent=2) + "\n") return target @@ -273,7 +227,7 @@ def analyze(run_dir: Path) -> dict: "--app-data-output", str(dashboard), "--reference-digest", - sha256(run_dir / "reference_outputs.csv"), + pass_inputs.sha256(run_dir / "reference_outputs.csv"), ], cwd=ROOT, env=env, @@ -413,11 +367,11 @@ def main() -> None: args = parser.parse_args() scratch = Path(args.scratch).resolve() out_dir = Path(args.out_dir) - # stage() and pass_inputs() delete and rewrite scratch/; never inside + # stage() and stage_inputs() delete and rewrite scratch/; never inside # the repository. if scratch == ROOT.resolve() or scratch.is_relative_to(ROOT.resolve()): parser.error(f"--scratch must be outside the repository ({ROOT})") - inputs = pass_inputs(scratch / "pass_inputs") + inputs = stage_inputs(scratch / "pass_inputs") source = inputs / "run" plan = variants(inputs / PROPOSALS_NAME) unknown = sorted(set(args.only or ()) - set(plan)) diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py new file mode 100644 index 00000000..b39b37ee --- /dev/null +++ b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py @@ -0,0 +1,185 @@ +"""The reference adversary's inputs, staged from git and checked against pinned sha256s. + +The pass read the frozen run as commit PASS_COMMIT (release dashboard-data-20260930, +#187) committed it, and built its reference system (policyengine-us 2.15.17 plus +``latest_final``) from the convention modules as that commit held them. Later commits +rewrite the working tree's copies: release dashboard-data-20261006 (#202) rewrote the +run's payload and exclusion record, and #204 rewrote +``latest_alt_snap_mortgage_residence.py``. So every script in this directory reads +these inputs only through ``git_input``: ``git show :`` into scratch, +refused (SystemExit) unless the bytes match the pinned sha256, and read only from the +staged copy. A script stages everything it reads before it computes or writes +anything. The code (this checkout's ``policybench`` package and the installed engine) +is not pinned here; each script's own reproduction check covers it. + +The README's Inputs table lists every pin and the scripts that read it, and +tests/test_reference_adversary_inputs.py checks that the table and these pins agree. +""" + +from __future__ import annotations + +import hashlib +import shutil +import subprocess +from collections.abc import Iterable +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +# Release dashboard-data-20260930 (#187): the frozen run, the output definitions and +# the reference system's convention modules, as the pass read them. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +RUN_SHA256 = { + "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "predictions.csv.gz": ( + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12" + ), + "reference_outputs.csv": ( + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" + ), + "reference_outputs.csv.meta.json": ( + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" + ), + "reference_exclusions.json": ( + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" + ), + "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +SPECS_PATH = "policybench/benchmark_specs.json" +SPECS_SHA256 = "ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5" +# latest_final and every module beside it, plus the sales tax table +# latest_c_irs_sales_tax_2025 reads; staged flat into one directory, as the +# reference builder copied them. +FIXES_SHA256 = { + "reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json": ( + "7cf93799aad6a8c3a6fc70e7df4acc6043202a1ecc083f2c3101f76b87a85eae" + ), + "reference_audit/2026-09-28/fixes/alt_conventions_r25.py": ( + "0f77fdd44e758581f7e563b0f7b70c4d4359a833b08a4ac45a614c380938775e" + ), + "reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py": ( + "3e71bab1db867988544e6f87f6dda4dc7c719d51320fbbb7558cd574a0374f6e" + ), + "reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py": ( + "d8a314093466caf5fe0a226f1060acba2e4364c4b87befbd80639ebf41cc4a2c" + ), + "reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py": ( + "00cd13886a8c3d03c4e84944d35f7e8a1448af26838e8bc0761b914b887c70cd" + ), + "reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py": ( + "5e6538a903112243ad85051ccba26b6133bdfbd5b9e5ba3dfa81ba690c061b32" + ), + "reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py": ( + "f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2" + ), + "reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py": ( + "39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5" + ), + "reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py": ( + "7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e" + ), + "reference_audit/2026-09-28/fixes/latest_c_md_2026.py": ( + "8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76" + ), + "reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py": ( + "787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e" + ), + "reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py": ( + "02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522" + ), + "reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py": ( + "0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916" + ), + "reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py": ( + "3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd" + ), + "reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py": ( + "4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce" + ), + "reference_audit/2026-09-28/fixes/latest_conventions.py": ( + "4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb" + ), + "reference_audit/2026-09-28/fixes/latest_final.py": ( + "dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e" + ), + "reference_audit/2026-09-28/fixes/latest_map_stated_hours.py": ( + "b210e780a31cfa0a3a0da2c49a8068ee14f00029b3ed39c00be67a1e917eda21" + ), + "reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py": ( + "53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd" + ), + "reference_audit/2026-09-28/fixes/latest_state_conventions.py": ( + "56e62e399d20ba8c8d9c5dc9bc7ea6681e74d2b529fa38561dbc8e8b2a90b442" + ), +} + +# PolicyEngine/policybench#194's cited classification of state paid-leave programs. +LAW_COMMIT = "049f4f091397e9c3ec2d9b21f3acbeb9c07c9764" +LAW_PATH = "reference_audit/2026-10-05-payroll/program_classification.json" +LAW_SHA256 = "282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a" + +# The pass's own records as #200 merged them. +RECORDS_COMMIT = "4db91b5f10581947f60a07f899e0ec2867b0405e" +PROPOSALS_PATH = "reference_audit/2026-10-05-reference-adversary/proposed_changes.json" +PROPOSALS_SHA256 = "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf" +CONFORMANCE_PATH = ( + "reference_audit/2026-10-05-reference-adversary/verification/" + "definition_conformance.json" +) +CONFORMANCE_SHA256 = "05e503f1b291c505b90182395c4afb88844dcb45fb2952c004ca09d4273894a0" + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def git_bytes(commit: str, path: str, pinned: str) -> bytes: + """``path`` as ``commit`` holds it, refusing any bytes but the pinned ones.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + return result.stdout + + +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + data = git_bytes(commit, path, pinned) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + return target + + +def stage_run(target: Path, names: Iterable[str] | None = None) -> Path: + """The frozen run's files ``names`` (default: all) as PASS_COMMIT holds them, in an + empty ``target``.""" + if target.exists(): + shutil.rmtree(target) + for name in RUN_SHA256 if names is None else names: + git_input(PASS_COMMIT, f"{RUN_PATH}/{name}", RUN_SHA256[name], target / name) + return target + + +def stage_fixes(target: Path) -> Path: + """``latest_final``, its parts and the sales tax table, flat in an empty ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, pinned in FIXES_SHA256.items(): + git_input(PASS_COMMIT, path, pinned, target / Path(path).name) + return target diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py index fec7d92d..556e2267 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py @@ -58,7 +58,16 @@ recorded per-output simulations, and refuses to report if any scored reference does not reproduce. -Run from a policybench checkout with the policyengine-us 2.15.17 venv: +Every input is the pass's, staged from git by pass_inputs.py and checked against its +pinned sha256 before any system is built: the frozen run's payload, references, +reference sidecar, exclusion record and scenarios as release dashboard-data-20260930 +(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them (each +worker stages its own copy). The working tree's run, which later releases rewrite +(#202 rewrote its payload and exclusion record), is never read. The report names each +input by its repository path. + +Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout +needs 8b4c0ca1 in its history): PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ /.venv-pe21517/bin/python \\ @@ -87,18 +96,21 @@ import numpy as np import pandas as pd -ROOT = Path(__file__).resolve().parents[3] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +# The frozen run's files main reads, staged from git. +RUN_FILES = ( + "data.json.gz", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", ) -AUDIT_0928 = ROOT / "reference_audit/2026-09-28" -AUDIT_0922 = ROOT / "reference_audit/2026-09-22" YEAR = 2026 TOLERANCE = 1e-3 BINARY_SUFFIXES = ("_eligible",) RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} -PAYLOAD_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" SCORED_CELLS = 1928 SKIP_PREFIXES = ("gov.abolitions.",) SCALE_COMPONENTS = ("threshold", "rate", "amount", "average_rate", "base") @@ -125,13 +137,11 @@ def _jsonable(value): def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads.""" + """latest_final and its parts, plus the sales tax table the IRS module reads, + staged from git (pass_inputs.FIXES_SHA256).""" target = Path(tempfile.mkdtemp(prefix="publication_sources_fixes_")) atexit.register(shutil.rmtree, target, True) - for path in (AUDIT_0928 / "fixes").glob("*.py"): - shutil.copy2(path, target / path.name) - shutil.copy2(AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json", target) - return target + return pass_inputs.stage_fixes(target) def _load_reform(fix_dir: Path): @@ -736,13 +746,19 @@ def _parameter_fact(ref: Tree, plain: Tree, name: str, instant: str) -> dict: # --------------------------------------------------------------------------- -def _scored_cells_from_payload() -> tuple[set, str]: +def stage_inputs(target: Path) -> Path: + """Stage the run's files main reads under ``target``, each refused unless it + matches its pin.""" + return pass_inputs.stage_run(target, RUN_FILES) + + +def _scored_cells_from_payload(path: Path) -> tuple[set, str]: from policybench.consensus import file_sha256, load_us_payload - path = RUN / "data.json.gz" sha = file_sha256(path) - if sha != PAYLOAD_SHA256: - raise SystemExit(f"{path}: sha256 {sha} is not the frozen {PAYLOAD_SHA256}") + pinned = pass_inputs.RUN_SHA256["data.json.gz"] + if sha != pinned: + raise SystemExit(f"{path}: sha256 {sha} is not the frozen {pinned}") payload = load_us_payload(path) scored = set() for sid, outputs in payload["scenarioPredictions"].items(): @@ -782,6 +798,9 @@ def main() -> None: return started = time.time() + run = Path(tempfile.mkdtemp(prefix="publication_sources_inputs_")) + atexit.register(shutil.rmtree, run, True) + stage_inputs(run) install_build_hooks() BUILD["label"] = "plain" @@ -789,24 +808,24 @@ def main() -> None: from policybench.spec import expand_programs_for_scenario out_dir = Path(args.out_dir) - meta = json.loads((RUN / "reference_outputs.csv.meta.json").read_text()) + meta = json.loads((run / "reference_outputs.csv.meta.json").read_text()) programs = meta["programs"] if isinstance(programs, str): programs = ast.literal_eval(programs) - reference = pd.read_csv(RUN / "reference_outputs.csv") + reference = pd.read_csv(run / "reference_outputs.csv") indexed = reference.set_index(["scenario_id", "variable"])["value"] - exclusions = json.loads((RUN / "reference_exclusions.json").read_text())[ + exclusions = json.loads((run / "reference_exclusions.json").read_text())[ "exclusions" ] excluded = {(e["scenario_id"], e["variable"]) for e in exclusions} - scored, payload_sha = _scored_cells_from_payload() + scored, payload_sha = _scored_cells_from_payload(run / "data.json.gz") by_exclusion = {key for key in indexed.index if key not in excluded} if scored != by_exclusion or len(scored) != SCORED_CELLS: raise SystemExit( f"scored cells: payload {len(scored)}, reference rows minus exclusions " f"{len(by_exclusion)}, expected {SCORED_CELLS}" ) - scenarios = pd.read_csv(RUN / "scenarios.csv") + scenarios = pd.read_csv(run / "scenarios.csv") if args.scenarios: scenarios = scenarios[scenarios["scenario_id"].isin(args.scenarios)] @@ -988,9 +1007,9 @@ def main() -> None: "policyengine_core": version("policyengine-core"), "reference_system": "policyengine-us 2.15.17 + latest_final " "(reference_audit/2026-09-28/fixes)", - "payload": str((RUN / "data.json.gz").relative_to(ROOT)), + "payload": f"{pass_inputs.RUN_PATH}/data.json.gz", "payload_sha256": payload_sha, - "reference_outputs": str((RUN / "reference_outputs.csv").relative_to(ROOT)), + "reference_outputs": f"{pass_inputs.RUN_PATH}/reference_outputs.csv", "scenarios": int(len(results)), "outputs_recomputed": checked, "scored_cells_recomputed": n_scored, diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py index 612d9f42..64022da4 100644 --- a/tests/test_reference_adversary_impact.py +++ b/tests/test_reference_adversary_impact.py @@ -3,13 +3,14 @@ ``reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py`` reads the frozen run as PASS_COMMIT (release dashboard-data-20260930, #187) committed it and the proposals as #200 merged them, each checked against its -sha256. It never reads the working tree's run, which later releases rewrite -(#202 did). CI checks out full history, so git holds every pinned input. +sha256 by the shared ``scripts/pass_inputs.py``. It never reads the working +tree's run, which later releases rewrite (#202 did). The pins themselves, and the +refusal of any changed byte, are tested in tests/test_reference_adversary_inputs.py. +CI checks out full history, so git holds every pinned input. """ from __future__ import annotations -import ast import difflib import gzip import importlib.util @@ -18,20 +19,17 @@ import re import subprocess import sys -import tempfile -from functools import cache from pathlib import Path -from types import SimpleNamespace -from unittest import mock import pytest -from hypothesis import given, settings -from hypothesis import strategies as st ROOT = Path(__file__).resolve().parents[1] AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" SCRIPT = AUDIT / "scripts/leaderboard_impact.py" VERIFICATION = AUDIT / "verification" +sys.path.insert(0, str(SCRIPT.parent)) +import pass_inputs as pins # noqa: E402 + # The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" @@ -63,11 +61,11 @@ def _tampered_checkout(path: Path) -> Path: ["git", "clone", "--quiet", "--shared", "--no-checkout", str(ROOT), str(path)], check=True, ) - for name in impact.RUN_SHA256: - junk = path / impact.RUN_PATH / name + for name in pins.RUN_SHA256: + junk = path / pins.RUN_PATH / name junk.parent.mkdir(parents=True, exist_ok=True) junk.write_bytes(b"not the pass's input\n") - proposals = path / impact.PROPOSALS_PATH + proposals = path / pins.PROPOSALS_PATH proposals.parent.mkdir(parents=True, exist_ok=True) proposals.write_text('{"root_causes": {}}\n') return path @@ -90,109 +88,32 @@ def _argv(monkeypatch, tmp_path: Path) -> Path: def test_the_pins_cover_every_file_the_script_reads(tmp_path): - assert impact.PASS_COMMIT == PASS_COMMIT - assert impact.RUN_SHA256["data.json.gz"] == FROZEN_SHA256 - assert set(impact.RUN_SHA256) == {*impact.RUN_FILES, "data.json.gz"} + assert pins.PASS_COMMIT == PASS_COMMIT + assert pins.RUN_SHA256["data.json.gz"] == FROZEN_SHA256 + assert set(pins.RUN_SHA256) == {*impact.RUN_FILES, "data.json.gz"} - inputs = impact.pass_inputs(tmp_path / "pass_inputs") + inputs = impact.stage_inputs(tmp_path / "pass_inputs") staged = sorted( path.relative_to(inputs).as_posix() for path in inputs.rglob("*") if path.is_file() ) assert staged == sorted( - [f"run/{name}" for name in impact.RUN_SHA256] + [impact.PROPOSALS_NAME] + [f"run/{name}" for name in pins.RUN_SHA256] + [impact.PROPOSALS_NAME] ) - for name, pinned in impact.RUN_SHA256.items(): - assert impact.sha256(inputs / "run" / name) == pinned - assert impact.sha256(inputs / impact.PROPOSALS_NAME) == impact.PROPOSALS_SHA256 + for name, pinned in pins.RUN_SHA256.items(): + assert pins.sha256(inputs / "run" / name) == pinned + assert pins.sha256(inputs / impact.PROPOSALS_NAME) == pins.PROPOSALS_SHA256 # The README's frozen run: 46 models, 1,928 scored cells. payload = json.loads(gzip.decompress((inputs / "run/data.json.gz").read_bytes())) assert len(payload["modelStats"]) == 46 assert {row["n"] for row in payload["modelStats"]} == {1928} (inputs / "stray").write_text("left over\n") - impact.pass_inputs(inputs) + impact.stage_inputs(inputs) assert not (inputs / "stray").exists() -def test_the_payload_pin_matches_the_engine_side_scripts_pins(): - """The engine-side scripts pin the same payload (read without importing them).""" - for name in ("definition_conformance.py", "publication_sources.py"): - tree = ast.parse((AUDIT / "scripts" / name).read_text()) - pins = [ - node.value.value - for node in tree.body - if isinstance(node, ast.Assign) - and [target.id for target in node.targets] == ["PAYLOAD_SHA256"] - ] - assert pins == [impact.RUN_SHA256["data.json.gz"]], name - - -@cache -def _pinned_bytes(name: str) -> bytes: - return subprocess.run( - ["git", "-C", str(ROOT), "show", f"{PASS_COMMIT}:{impact.RUN_PATH}/{name}"], - capture_output=True, - check=True, - ).stdout - - -@st.composite -def _edits(draw) -> tuple[str, str, int, int]: - """One pinned run file and one change to its bytes: flip, cut or extend.""" - name = draw(st.sampled_from(sorted(impact.RUN_SHA256))) - kind = draw(st.sampled_from(["flip", "truncate", "append"])) - position = draw(st.integers(0, len(_pinned_bytes(name)) - 1)) - return name, kind, position, draw(st.integers(1, 255)) - - -@settings(max_examples=40, deadline=None) -@given(_edits()) -def test_any_changed_byte_in_a_pinned_input_is_refused_unwritten(edit): - """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" - name, kind, position, byte = edit - original = _pinned_bytes(name) - if kind == "flip": - changed = bytearray(original) - changed[position] ^= byte - tampered = bytes(changed) - elif kind == "truncate": - tampered = original[:position] - else: - tampered = original + bytes([byte]) - shown = SimpleNamespace( - run=lambda *args, **kwargs: subprocess.CompletedProcess( - args[0], 0, stdout=tampered, stderr=b"" - ) - ) - with tempfile.TemporaryDirectory() as directory: - target = Path(directory) / name - with mock.patch.object(impact, "subprocess", shown): - with pytest.raises(SystemExit, match="not the pinned"): - impact.git_input( - PASS_COMMIT, - f"{impact.RUN_PATH}/{name}", - impact.RUN_SHA256[name], - target, - ) - assert not target.exists() - # The unchanged bytes pass the same check. - unchanged = SimpleNamespace( - run=lambda *args, **kwargs: subprocess.CompletedProcess( - args[0], 0, stdout=original, stderr=b"" - ) - ) - with mock.patch.object(impact, "subprocess", unchanged): - impact.git_input( - PASS_COMMIT, - f"{impact.RUN_PATH}/{name}", - impact.RUN_SHA256[name], - target, - ) - assert target.read_bytes() == original - - @pytest.mark.parametrize( ("attribute", "value", "message"), [ @@ -206,19 +127,19 @@ def test_any_changed_byte_in_a_pinned_input_is_refused_unwritten(edit): ("PASS_COMMIT", "0" * 40, "cannot read"), ], ) -def test_pass_inputs_refuse_anything_but_the_pinned_bytes( +def test_staged_inputs_refuse_anything_but_the_pinned_bytes( tmp_path, monkeypatch, attribute, value, message ): - monkeypatch.setattr(impact, attribute, value) + monkeypatch.setattr(pins, attribute, value) with pytest.raises(SystemExit, match=re.escape(message)): - impact.pass_inputs(tmp_path / "pass_inputs") + impact.stage_inputs(tmp_path / "pass_inputs") def test_a_refused_input_stops_the_script_before_it_scores(tmp_path, monkeypatch): def analyze(run_dir): raise AssertionError(f"scored {run_dir} after its inputs were refused") - monkeypatch.setattr(impact, "PASS_COMMIT", RELEASE_20261006_COMMIT) + monkeypatch.setattr(pins, "PASS_COMMIT", RELEASE_20261006_COMMIT) monkeypatch.setattr(impact, "analyze", analyze) out = _argv(monkeypatch, tmp_path) with pytest.raises(SystemExit, match=f"not the pinned {FROZEN_SHA256}"): @@ -229,13 +150,15 @@ def analyze(run_dir): def test_the_script_scores_only_the_staged_inputs(tmp_path, monkeypatch): """Run from a checkout whose working-tree run is junk, every variant still starts from the pinned bytes and the full set of evidence files is written.""" - monkeypatch.setattr(impact, "ROOT", _tampered_checkout(tmp_path / "checkout")) + checkout = _tampered_checkout(tmp_path / "checkout") + monkeypatch.setattr(pins, "ROOT", checkout) + monkeypatch.setattr(impact, "ROOT", checkout) scored: dict[str, dict[str, str]] = {} published: dict = {} def analyze(run_dir: Path) -> dict: scored[run_dir.name] = { - name: impact.sha256(run_dir / name) for name in impact.RUN_FILES + name: pins.sha256(run_dir / name) for name in impact.RUN_FILES } if not published: payload = run_dir.parent / "pass_inputs/run/data.json.gz" @@ -249,7 +172,7 @@ def analyze(run_dir: Path) -> dict: out = _argv(monkeypatch, tmp_path) impact.main() - pinned = {name: impact.RUN_SHA256[name] for name in impact.RUN_FILES} + pinned = {name: pins.RUN_SHA256[name] for name in impact.RUN_FILES} assert scored.pop("published") == pinned exclusions = {"reference_exclusions.json"} regenerations = {"reference_outputs.csv", "reference_outputs.csv.meta.json"} diff --git a/tests/test_reference_adversary_inputs.py b/tests/test_reference_adversary_inputs.py new file mode 100644 index 00000000..dbe593f0 --- /dev/null +++ b/tests/test_reference_adversary_inputs.py @@ -0,0 +1,755 @@ +"""The reference adversary's scripts read only the pass's pinned inputs. + +``reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py`` pins +every input the pass read, by commit and sha256: the frozen run as release +dashboard-data-20260930 (8b4c0ca1) committed it, the output definitions and the +reference system's convention modules as that commit held them, #194's law +table, and the pass's own records as #200 merged them. Later commits rewrote +the working tree's copies (#202 the run's payload and exclusion record, #204 +one convention module). These tests check that + +* the pins are what the pass read: each commit holds those bytes, the + committed evidence records the same hashes, and the README's Inputs table + lists each pin with the scripts that stage it; +* any other bytes are refused, and each script refuses before it computes or + writes anything; +* no script opens the working tree's copy of a pinned input (an audit hook + fails any such open); +* each script regenerates its committed outputs byte for byte. The engine-side + regenerations need policyengine-us 2.15.17 and take minutes, so they are + marked slow and CI deselects them; build_proposals.py's regeneration is fast. + +CI checks out full history, so git holds every pinned input. +""" + +from __future__ import annotations + +import gzip +import hashlib +import importlib.util +import io +import json +import os +import re +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from functools import cache +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" +SCRIPTS = AUDIT / "scripts" +VERIFICATION = AUDIT / "verification" +PROBES = sorted((VERIFICATION / "probes").glob("*.json")) +sys.path.insert(0, str(SCRIPTS)) +import pass_inputs as pins # noqa: E402 + +from tests import working_tree_fence as fence # noqa: E402 + +# Release dashboard-data-20261006 (#202): the run's payload and exclusion record +# as it rewrote them. +RELEASE_20261006_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = { + "data.json.gz": "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d", + "reference_exclusions.json": ( + "92741dfd047298eacb79a0dadeb7e4965e3c6f986beab0341b12ff8f5b023815" + ), +} +# latest_alt_snap_mortgage_residence.py as #204 rewrote it (held at 4db91b5f). +MODULE_204 = "reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py" +MODULE_204_SHA256 = "5e701f8fa1b31bfeed882a0bc5160faa4ee1d2b5a67974f9d9dc60e6f6638053" +# Every pin: (commit, path, sha256). +PINNED = ( + [ + (pins.PASS_COMMIT, f"{pins.RUN_PATH}/{name}", sha) + for name, sha in pins.RUN_SHA256.items() + ] + + [(pins.PASS_COMMIT, path, sha) for path, sha in pins.FIXES_SHA256.items()] + + [ + (pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256), + (pins.LAW_COMMIT, pins.LAW_PATH, pins.LAW_SHA256), + (pins.RECORDS_COMMIT, pins.PROPOSALS_PATH, pins.PROPOSALS_SHA256), + (pins.RECORDS_COMMIT, pins.CONFORMANCE_PATH, pins.CONFORMANCE_SHA256), + ] +) + + +def _load(name: str): + spec = importlib.util.spec_from_file_location( + f"reference_adversary_{name}", SCRIPTS / f"{name}.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +conformance = _load("definition_conformance") +sources = _load("publication_sources") +probe = _load("engine_probe") +proposals = _load("build_proposals") +impact = _load("leaderboard_impact") +SCRIPT_MODULES = { + "definition_conformance": conformance, + "publication_sources": sources, + "engine_probe": probe, + "build_proposals": proposals, +} + + +# --------------------------------------------------------------------------- +# A fence on the working tree's copy of every pinned input +# --------------------------------------------------------------------------- + +# The working tree's copies; policybench may read its own benchmark_specs.json. +FENCE = { + "forbidden": [ + str(ROOT / path) + for path in ( + pins.RUN_PATH, + *pins.FIXES_SHA256, + pins.SPECS_PATH, + pins.LAW_PATH, + pins.PROPOSALS_PATH, + pins.CONFORMANCE_PATH, + ) + ], + "exempt": {str(ROOT / pins.SPECS_PATH): str(ROOT / "policybench")}, + "root": str(ROOT), +} + + +@contextmanager +def fenced(): + """Inside, opening the working tree's copy of any pinned input raises.""" + fence.arm(**FENCE) + try: + yield + finally: + fence.disarm() + + +def _fenced_env(directory: Path) -> dict[str, str]: + """A subprocess environment whose every interpreter, workers included, is fenced.""" + return dict( + os.environ, + PYTHONPATH=f"{fence.sitecustomize(directory)}{os.pathsep}{ROOT}", + PYTHONDONTWRITEBYTECODE="1", + OPENBLAS_NUM_THREADS="1", + **{fence.ENV: json.dumps(FENCE)}, + ) + + +def test_the_fence_catches_a_working_tree_read(): + """The hook is not vacuous: each way the scripts read files trips it.""" + run = ROOT / pins.RUN_PATH + with fenced(): + with pytest.raises(PermissionError, match="working tree"): + pd.read_csv(run / "scenarios.csv") + with pytest.raises(PermissionError, match="working tree"): + gzip.open(run / "data.json.gz").read() + with pytest.raises(PermissionError, match="working tree"): + (ROOT / MODULE_204).read_text() + with pytest.raises(PermissionError, match="working tree"): + (ROOT / pins.CONFORMANCE_PATH).read_bytes() + # Only policybench's own code may read its output definitions. + with pytest.raises(PermissionError, match="working tree"): + (ROOT / pins.SPECS_PATH).read_bytes() + from policybench import spec + + spec._raw_spec_data.cache_clear() + assert spec._raw_spec_data() + # git show reads the object store, not the working tree. + assert pins.git_bytes( + pins.PASS_COMMIT, MODULE_204, pins.FIXES_SHA256[MODULE_204] + ) + assert (run / "scenarios.csv").read_bytes() + + +FENCE_PROBE = """ +import sys +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + + +def read(path): + return len(Path(path).read_bytes()) + + +if __name__ == "__main__": + with ProcessPoolExecutor(1) as pool: + print(pool.submit(read, sys.argv[1]).result()) +""" + + +@pytest.mark.parametrize("where", ["main", "worker"]) +def test_the_fence_reaches_every_process_of_a_subprocess(where, tmp_path): + """Armed through sitecustomize, it fails a read in the main process and in a + spawned worker alike; unarmed, the same read succeeds.""" + script = tmp_path / "read.py" + if where == "main": + script.write_text("import sys\nprint(len(open(sys.argv[1], 'rb').read()))\n") + else: + script.write_text(FENCE_PROBE) + target = str(ROOT / pins.RUN_PATH / "scenarios.csv") + env = _fenced_env(tmp_path / "fence") + fenced_run = subprocess.run( + [sys.executable, str(script), target], env=env, capture_output=True, text=True + ) + assert fenced_run.returncode != 0 + assert "opened the working tree's" in fenced_run.stderr + env.pop(fence.ENV) + plain = subprocess.run( + [sys.executable, str(script), target], env=env, capture_output=True, text=True + ) + assert plain.returncode == 0, plain.stderr + assert int(plain.stdout) > 0 + + +# --------------------------------------------------------------------------- +# The pins are what the pass read +# --------------------------------------------------------------------------- + + +def test_every_pin_is_what_its_commit_holds(): + for commit, path, pinned in PINNED: + data = pins.git_bytes(commit, path, pinned) + assert hashlib.sha256(data).hexdigest() == pinned, path + # The README's frozen run: 46 models, 1,928 scored cells. + payload = json.loads( + gzip.decompress( + pins.git_bytes( + pins.PASS_COMMIT, + f"{pins.RUN_PATH}/data.json.gz", + pins.RUN_SHA256["data.json.gz"], + ) + ) + ) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + +def test_the_committed_evidence_records_the_pins(): + """Differential: the hashes the pass's own outputs recorded equal the pins.""" + run = json.loads((VERIFICATION / "definition_conformance.json").read_text())["run"] + assert run["payload"] == f"{pins.RUN_PATH}/data.json.gz" + assert run["payload_sha256"] == pins.RUN_SHA256["data.json.gz"] + assert run["reference_outputs_sha256"] == pins.RUN_SHA256["reference_outputs.csv"] + assert run["scenarios_sha256"] == pins.RUN_SHA256["scenarios.csv"] + assert run["benchmark_specs_sha256"] == pins.SPECS_SHA256 + assert run["fixes_sha256"] == pins.FIXES_SHA256 + assert run["law_classification"]["sha256"] == pins.LAW_SHA256 + assert run["law_classification"]["source"] == conformance.LAW_SOURCE + + meta = json.loads((VERIFICATION / "publication_sources.json").read_text())["meta"] + assert meta["payload"] == f"{pins.RUN_PATH}/data.json.gz" + assert meta["payload_sha256"] == pins.RUN_SHA256["data.json.gz"] + assert meta["reference_outputs"] == f"{pins.RUN_PATH}/reference_outputs.csv" + + flags = json.loads((AUDIT / "consensus_flags.json").read_text()) + assert flags["source"] == f"{pins.RUN_PATH}/data.json.gz" + assert flags["source_sha256"] == pins.RUN_SHA256["data.json.gz"] + + +def test_later_commits_hold_other_bytes_than_the_pins(): + """Why the pins exist: #202 rewrote the run's payload and exclusion record and + #204 a convention module, so reading those commits' copies is refused.""" + for name, sha in RELEASE_20261006_SHA256.items(): + with pytest.raises(SystemExit, match=f"has sha256 {sha}, not the pinned"): + pins.git_bytes( + RELEASE_20261006_COMMIT, + f"{pins.RUN_PATH}/{name}", + pins.RUN_SHA256[name], + ) + with pytest.raises(SystemExit, match=f"has sha256 {MODULE_204_SHA256}, not"): + pins.git_bytes(pins.RECORDS_COMMIT, MODULE_204, pins.FIXES_SHA256[MODULE_204]) + + +def _staged(script: str, tmp_path: Path) -> set[tuple[str, str]]: + """Every (commit, path) a script stages, recorded as git_input is called.""" + seen: set[tuple[str, str]] = set() + original = pins.git_input + + def record(commit, path, pinned, target): + seen.add((commit, path)) + return original(commit, path, pinned, target) + + with mock.patch.object(pins, "git_input", record): + if script == "leaderboard_impact": + impact.stage_inputs(tmp_path / script) + else: + SCRIPT_MODULES[script].stage_inputs(tmp_path / script) + if script in ("definition_conformance", "publication_sources"): + SCRIPT_MODULES[script]._assemble_fixes() + if script == "engine_probe": + # The probe builds its reference system with definition_conformance's + # builder, which stages the convention modules. + conformance._assemble_fixes() + return seen + + +def _readme_inputs() -> dict[str, tuple[str, str, set[str]]]: + """README Inputs rows: path -> (commit, sha256, readers).""" + rows = {} + pattern = re.compile( + r"^\s*\| `(?P[^`]+)`[^|]* \| `(?P[0-9a-f]{8})` \| " + r"`(?P[0-9a-f]{64})` \|(?: (?P[^|]*) \|)?$" + ) + for line in (AUDIT / "README.md").read_text().splitlines(): + match = pattern.match(line) + if match is None: + continue + path = match["path"].replace("", pins.RUN_PATH) + readers = set(re.findall(r"`([^`]+)`", match["readers"] or "")) + assert path not in rows, f"README lists {path} twice" + rows[path] = (match["commit"], match["sha"], readers) + return rows + + +def test_the_readme_lists_each_pin_and_the_scripts_that_stage_it(tmp_path): + rows = _readme_inputs() + staged = { + script: _staged(script, tmp_path) + for script in [*SCRIPT_MODULES, "leaderboard_impact"] + } + readers: dict[tuple[str, str], set[str]] = {} + for script, inputs in staged.items(): + for key in inputs: + readers.setdefault(key, set()).add(f"{script}.py") + fixes = set(pins.FIXES_SHA256) + for commit, path, pinned in PINNED: + assert path in rows, f"README's Inputs table omits {path}" + row_commit, row_sha, row_readers = rows.pop(path) + assert commit.startswith(row_commit), path + assert row_sha == pinned, path + expected = readers.get((commit, path), set()) + assert expected, f"no script stages {path}" + if path in fixes: + # The convention modules' table names their readers once, above it. + assert row_readers == set(), path + continue + scripts = {reader for reader in row_readers if reader.endswith(".py")} + assert scripts == expected, path + # Rows left over are the CLI's inputs, which no script here reads. + for path, (commit, sha, row_readers) in rows.items(): + assert not any(reader.endswith(".py") for reader in row_readers), path + assert ( + hashlib.sha256( + subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + check=True, + ).stdout + ).hexdigest() + == sha + ), path + fix_readers = { + script + for script, inputs in staged.items() + if inputs & {(pins.PASS_COMMIT, path) for path in fixes} + } + assert fix_readers == { + "definition_conformance", + "publication_sources", + "engine_probe", + } + text = (AUDIT / "README.md").read_text() + assert ( + "read by `definition_conformance.py`, `publication_sources.py` and " + "`engine_probe.py`" in text + ) + + +def test_no_script_holds_its_own_pin_or_names_the_run(): + """pass_inputs.py is the only place a pin or the run's path is written.""" + hexdigest = re.compile(r"\b[0-9a-f]{64}\b") + for path in sorted(SCRIPTS.glob("*.py")): + if path.name == "pass_inputs.py": + continue + text = path.read_text() + assert not hexdigest.search(text), path.name + assert "paper/snapshot" not in text, path.name + + +# --------------------------------------------------------------------------- +# Any other bytes are refused +# --------------------------------------------------------------------------- + + +@cache +def _pinned_bytes(index: int) -> bytes: + commit, path, pinned = PINNED[index] + return pins.git_bytes(commit, path, pinned) + + +@st.composite +def _edits(draw) -> tuple[int, str, int, int]: + """One pinned input and one change to its bytes: flip, cut or extend.""" + index = draw(st.integers(0, len(PINNED) - 1)) + kind = draw(st.sampled_from(["flip", "truncate", "append"])) + position = draw(st.integers(0, len(_pinned_bytes(index)) - 1)) + return index, kind, position, draw(st.integers(1, 255)) + + +def _shown(data: bytes) -> SimpleNamespace: + return SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=data, stderr=b"" + ) + ) + + +@settings(max_examples=60, deadline=None) +@given(_edits()) +def test_any_changed_byte_in_any_pinned_input_is_refused_unwritten(edit): + """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" + index, kind, position, byte = edit + commit, path, pinned = PINNED[index] + original = _pinned_bytes(index) + if kind == "flip": + changed = bytearray(original) + changed[position] ^= byte + tampered = bytes(changed) + elif kind == "truncate": + tampered = original[:position] + else: + tampered = original + bytes([byte]) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / Path(path).name + with mock.patch.object(pins, "subprocess", _shown(tampered)): + with pytest.raises(SystemExit, match=f"not the pinned {pinned}"): + pins.git_input(commit, path, pinned, target) + assert not target.exists() + # The unchanged bytes pass the same check. + with mock.patch.object(pins, "subprocess", _shown(original)): + pins.git_input(commit, path, pinned, target) + assert target.read_bytes() == original + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ( + "PASS_COMMIT", + pins.RECORDS_COMMIT, + f"latest_alt_snap_mortgage_residence.py has sha256 {MODULE_204_SHA256}, " + f"not the pinned {pins.FIXES_SHA256[MODULE_204]}", + ), + ("PASS_COMMIT", "0" * 40, "cannot read"), + ], +) +@pytest.mark.parametrize("script", ["definition_conformance", "publication_sources"]) +def test_the_reference_system_refuses_other_convention_modules( + script, attribute, value, message, monkeypatch +): + """#204's rewrite of a convention module (on 4db91b5f) is refused.""" + monkeypatch.setattr(pins, attribute, value) + with pytest.raises(SystemExit, match=re.escape(message)): + SCRIPT_MODULES[script]._assemble_fixes() + + +def _never(name: str): + def called(*args, **kwargs): + raise AssertionError(f"{name} ran after an input was refused") + + return called + + +def _argv(script: str, out: Path) -> list[str]: + if script == "engine_probe": + return [script, "scenario_043", "--out", str(out / "probe.json")] + if script == "build_proposals": + return [script, "--out", str(out / "proposed_changes.json")] + return [script, "--out-dir", str(out)] + + +# The first input each script stages that #202 rewrote. +FIRST_REWRITTEN = { + "definition_conformance": "data.json.gz", + "publication_sources": "data.json.gz", + "engine_probe": "reference_exclusions.json", + "build_proposals": "reference_exclusions.json", +} + + +@pytest.mark.parametrize("script", sorted(SCRIPT_MODULES)) +def test_a_mismatched_input_stops_the_script_before_it_computes_or_writes( + script, tmp_path, monkeypatch +): + """Pointed at #202's run, each script refuses before any engine work or write.""" + module = SCRIPT_MODULES[script] + monkeypatch.setattr(pins, "PASS_COMMIT", RELEASE_20261006_COMMIT) + for name in ("_system", "_assemble_fixes", "install_build_hooks", "_conformance"): + if hasattr(module, name): + monkeypatch.setattr(module, name, _never(name)) + monkeypatch.setattr(module, "build", _never("build"), raising=False) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv(script, out)) + name = FIRST_REWRITTEN[script] + with pytest.raises( + SystemExit, + match=re.escape( + f"{name} has sha256 {RELEASE_20261006_SHA256[name]}, " + f"not the pinned {pins.RUN_SHA256[name]}" + ), + ): + module.main() + assert not out.exists() + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ("CONFORMANCE_SHA256", "0" * 64, f"not the pinned {'0' * 64}"), + ("RECORDS_COMMIT", pins.PASS_COMMIT, "cannot read"), + ], +) +def test_build_proposals_refuses_another_conformance_scan( + attribute, value, message, tmp_path, monkeypatch +): + monkeypatch.setattr(pins, attribute, value) + monkeypatch.setattr(proposals, "build", _never("build")) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv("build_proposals", out)) + with pytest.raises(SystemExit, match=re.escape(message)): + proposals.main() + assert not out.exists() + + +def test_definition_conformance_refuses_another_law_table(tmp_path, monkeypatch): + monkeypatch.setattr(pins, "LAW_SHA256", "0" * 64) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + conformance.stage_inputs(tmp_path / "inputs") + + +def test_build_proposals_never_writes_the_committed_records(monkeypatch, capsys): + committed = (AUDIT / "proposed_changes.json").read_bytes() + monkeypatch.setattr(proposals, "build", _never("build")) + for target in ( + AUDIT / "proposed_changes.json", + SCRIPTS / ".." / "proposed_changes.json", + ): + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(target)]) + with pytest.raises(SystemExit): + proposals.main() + assert "is pinned at sha256" in capsys.readouterr().err + monkeypatch.setattr(sys, "argv", ["build_proposals"]) + with pytest.raises(SystemExit): + proposals.main() + assert (AUDIT / "proposed_changes.json").read_bytes() == committed + assert hashlib.sha256(committed).hexdigest() == pins.PROPOSALS_SHA256 + + +# --------------------------------------------------------------------------- +# The working tree is never read +# --------------------------------------------------------------------------- + + +def test_build_proposals_regenerates_the_committed_records_from_git( + tmp_path, monkeypatch +): + """The whole script, with every working-tree input fenced off, writes the + committed proposed_changes.json byte for byte.""" + out = tmp_path / "proposed_changes.json" + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(out)]) + with fenced(): + proposals.main() + assert out.read_bytes() == (AUDIT / "proposed_changes.json").read_bytes() + assert pins.sha256(out) == pins.PROPOSALS_SHA256 + + +def test_definition_conformance_reads_only_staged_inputs(tmp_path): + with fenced(): + staged = conformance.stage_inputs(tmp_path / "inputs") + laws, meta = conformance.law_classifications( + staged / "program_classification.json" + ) + assert sorted(path.name for path in (staged / "run").iterdir()) == sorted( + conformance.RUN_FILES + ) + for name in conformance.RUN_FILES: + assert pins.sha256(staged / "run" / name) == pins.RUN_SHA256[name] + assert pins.sha256(staged / "benchmark_specs.json") == pins.SPECS_SHA256 + recorded = json.loads((VERIFICATION / "definition_conformance.json").read_text()) + assert meta == recorded["run"]["law_classification"] + assert laws + + +def test_the_reference_system_is_built_from_staged_modules(): + expected = {Path(path).name: sha for path, sha in pins.FIXES_SHA256.items()} + for module in (conformance, sources): + with fenced(): + fix_dir = module._assemble_fixes() + assert {path.name: pins.sha256(path) for path in fix_dir.iterdir()} == expected + # definition_conformance records the hashes of the directory it built from. + assert conformance._assemble_fixes() == conformance._FIX_DIR + + +def test_publication_sources_reads_only_staged_inputs(tmp_path): + with fenced(): + run = sources.stage_inputs(tmp_path / "run") + scored, sha = sources._scored_cells_from_payload(run / "data.json.gz") + assert sorted(path.name for path in run.iterdir()) == sorted(sources.RUN_FILES) + assert sha == pins.RUN_SHA256["data.json.gz"] + assert len(scored) == sources.SCORED_CELLS + reference = pd.read_csv(run / "reference_outputs.csv") + exclusions = json.loads((run / "reference_exclusions.json").read_text()) + excluded = {(e["scenario_id"], e["variable"]) for e in exclusions["exclusions"]} + keys = set(zip(reference["scenario_id"], reference["variable"])) + assert scored == keys - excluded + + +def test_engine_probe_scores_the_pass_cells_not_202s(tmp_path, monkeypatch): + """The probe's reproduction set comes from the pass's exclusion record. + + #202 later excluded scenario_043's payroll_tax; read from the working tree, + the probe would drop it. The engine is stubbed to return the published + references, so this runs without policyengine-us. + """ + reference = pd.read_csv( + io.BytesIO( + pins.git_bytes( + pins.PASS_COMMIT, + f"{pins.RUN_PATH}/reference_outputs.csv", + pins.RUN_SHA256["reference_outputs.csv"], + ) + ) + ) + published = { + row.variable: float(row.value) + for row in reference[reference["scenario_id"] == "scenario_043"].itertuples() + } + engine = SimpleNamespace( + build_situation=conformance.build_situation, + _system=lambda: SimpleNamespace(variables={}), + _simulation=lambda situation: None, + _outputs=lambda sim, scenario, scored: {v: published[v] for v in scored}, + REPRODUCE_TOLERANCE=conformance.REPRODUCE_TOLERANCE, + ) + monkeypatch.setattr(probe, "_conformance", lambda: engine) + out = tmp_path / "probe.json" + monkeypatch.setattr( + sys, "argv", ["engine_probe", "scenario_043", "--out", str(out)] + ) + with fenced(): + probe.main() + committed = json.loads( + (VERIFICATION / "probes/co_sales_tax_refund_scenario_043.json").read_text() + ) + written = json.loads(out.read_text()) + assert "payroll_tax" in written["reproduction"] + assert { + variable: row["published"] for variable, row in written["reproduction"].items() + } == { + variable: row["published"] + for variable, row in committed["reproduction"].items() + } + assert written["people"] == committed["people"] + assert written["state"] == committed["state"] == "CO" + + +# --------------------------------------------------------------------------- +# Byte-for-byte regeneration (slow: the reference system, policyengine-us 2.15.17) +# --------------------------------------------------------------------------- + + +def _run(script: str, tmp_path: Path, *args: str) -> None: + """Run a script with the fence armed in every one of its processes.""" + result = subprocess.run( + [sys.executable, str(SCRIPTS / f"{script}.py"), *args], + cwd=ROOT, + env=_fenced_env(tmp_path / "fence"), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + + +def _identical_but(regenerated: Path, committed: Path, volatile: set[str]) -> dict: + """Require the files byte for byte, except the one line carrying each key in + ``volatile``; return the regenerated value of each of those keys.""" + new = regenerated.read_bytes().splitlines(keepends=True) + old = committed.read_bytes().splitlines(keepends=True) + assert len(new) == len(old), regenerated.name + line = re.compile(rb'^\s*"(?P[a-z_0-9]+)": (?P.*?),?\n$') + values = {} + for key in volatile: + lines = [ + i + for i, text in enumerate(old) + if re.match(rb'^\s*"' + key.encode() + rb'": ', text) + ] + assert len(lines) == 1, (committed.name, key) + values[key] = json.loads(line.match(new[lines[0]])["value"]) + differing = [i for i, (a, b) in enumerate(zip(new, old)) if a != b] + unexpected = [ + (i + 1, old[i][:120], new[i][:120]) + for i in differing + if (match := line.match(old[i])) is None + or match["key"].decode() not in volatile + ] + assert not unexpected, (regenerated.name, unexpected[:5]) + return values + + +@pytest.mark.slow +def test_definition_conformance_regenerates_byte_for_byte(tmp_path): + """Only the wall time and this script's own sha256 differ from the record. + + About two to four minutes. Run with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_adversary_inputs.py + """ + _run("definition_conformance", tmp_path, "--out-dir", str(tmp_path)) + assert (tmp_path / "definition_conformance.md").read_bytes() == ( + VERIFICATION / "definition_conformance.md" + ).read_bytes() + values = _identical_but( + tmp_path / "definition_conformance.json", + VERIFICATION / "definition_conformance.json", + {"seconds", "script_sha256"}, + ) + assert values["script_sha256"] == pins.sha256(SCRIPTS / "definition_conformance.py") + + +@pytest.mark.slow +def test_publication_sources_regenerates_byte_for_byte(tmp_path): + """Only the wall time differs from the record. About four to ten minutes.""" + _run("publication_sources", tmp_path, "--out-dir", str(tmp_path)) + assert (tmp_path / "publication_sources.md").read_bytes() == ( + VERIFICATION / "publication_sources.md" + ).read_bytes() + _identical_but( + tmp_path / "publication_sources.json", + VERIFICATION / "publication_sources.json", + {"seconds"}, + ) + + +def _probe_args(path: Path) -> list[str]: + """The arguments a committed probe records about itself.""" + record = json.loads(path.read_text()) + args = [record["scenario_id"], "--period", record["period"]] + args += ["--variables", *record["variables"]] + args += ["--parameters", *record["parameters"]] + for assignment in record.get("counterfactual", {}).get("set", []): + args += ["--set", assignment] + return args + + +@pytest.mark.slow +@pytest.mark.parametrize("path", PROBES, ids=[path.stem for path in PROBES]) +def test_every_probe_regenerates_byte_for_byte(path, tmp_path): + """About a minute each.""" + assert len(PROBES) == 8 + out = tmp_path / path.name + _run("engine_probe", tmp_path, *_probe_args(path), "--out", str(out)) + assert out.read_bytes() == path.read_bytes() diff --git a/tests/working_tree_fence.py b/tests/working_tree_fence.py new file mode 100644 index 00000000..a97c3c18 --- /dev/null +++ b/tests/working_tree_fence.py @@ -0,0 +1,91 @@ +"""An audit hook that fails any open of the working tree's copy of a pinned input. + +tests/test_reference_adversary_inputs.py arms it in-process around the +reference adversary's scripts, and in every process of a script it runs as a +subprocess, worker processes included: ``sitecustomize(directory)`` writes a +``sitecustomize.py`` that loads this file and calls ``install_from_env()``, so +putting ``directory`` first on PYTHONPATH and the fence's JSON in ``ENV`` arms +each interpreter as it starts. + +``exempt`` maps a forbidden file to the one directory whose code may open it: +policybench reads its own ``benchmark_specs.json`` (package data, which the +scripts treat as code), while a script that opened that file itself would trip +the fence. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +ENV = "REFERENCE_ADVERSARY_FENCE" +_STATE: dict = {"forbidden": (), "exempt": {}, "root": ""} +_INSTALLED = False + + +def _innermost_repository_frame() -> str: + """The file of the innermost calling frame that is repository code.""" + root = _STATE["root"] + os.sep + frame = sys._getframe(2) + while frame is not None: + name = os.path.abspath(frame.f_code.co_filename) + if name.startswith(root) and f"{os.sep}.venv{os.sep}" not in name: + return name + frame = frame.f_back + return "" + + +def _hook(event: str, args: tuple) -> None: + if event != "open" or not _STATE["forbidden"]: + return + target = args[0] + if not isinstance(target, (str, bytes, os.PathLike)): + return + path = os.path.abspath(os.fsdecode(target)) + if not any( + path == forbidden or path.startswith(forbidden + os.sep) + for forbidden in _STATE["forbidden"] + ): + return + owner = _STATE["exempt"].get(path) + if owner and _innermost_repository_frame().startswith(owner + os.sep): + return + raise PermissionError(f"opened the working tree's {path}") + + +def arm(forbidden, exempt: dict[str, str], root: str) -> None: + global _INSTALLED + if not _INSTALLED: + sys.addaudithook(_hook) + _INSTALLED = True + _STATE.update( + forbidden=tuple(os.path.abspath(path) for path in forbidden), + exempt={os.path.abspath(k): os.path.abspath(v) for k, v in exempt.items()}, + root=os.path.abspath(root), + ) + + +def disarm() -> None: + _STATE["forbidden"] = () + + +def install_from_env() -> None: + fence = os.environ.get(ENV) + if fence: + arm(**json.loads(fence)) + + +def sitecustomize(directory: Path) -> Path: + """Write a sitecustomize.py into ``directory`` that arms the fence from ENV.""" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sitecustomize.py").write_text( + "import importlib.util\n" + 'spec = importlib.util.spec_from_file_location("working_tree_fence", ' + f"{str(Path(__file__).resolve())!r})\n" + "module = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(module)\n" + "module.install_from_env()\n" + ) + return directory From 2829f6a8a9a5307f75d8409bc1247cdc116600be Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 14:51:28 -0400 Subject: [PATCH 4/7] Require the volatile lines to differ only in value, and refuse a symlinked --out _identical_but now requires the regenerated file to carry the same key, indentation and punctuation on each volatile line, so only that one JSON value may differ; a new test feeds it a changed value elsewhere, a renamed key, re-indentation, a changed list and a dropped line. The build_proposals refusal test adds a relative path and a symlink to the committed proposed_changes.json. Co-Authored-By: Claude Opus 5.5 --- tests/test_reference_adversary_inputs.py | 69 +++++++++++++++++++----- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/tests/test_reference_adversary_inputs.py b/tests/test_reference_adversary_inputs.py index dbe593f0..3d69f40e 100644 --- a/tests/test_reference_adversary_inputs.py +++ b/tests/test_reference_adversary_inputs.py @@ -530,12 +530,18 @@ def test_definition_conformance_refuses_another_law_table(tmp_path, monkeypatch) conformance.stage_inputs(tmp_path / "inputs") -def test_build_proposals_never_writes_the_committed_records(monkeypatch, capsys): +def test_build_proposals_never_writes_the_committed_records( + tmp_path, monkeypatch, capsys +): committed = (AUDIT / "proposed_changes.json").read_bytes() monkeypatch.setattr(proposals, "build", _never("build")) + link = tmp_path / "link.json" + link.symlink_to(AUDIT / "proposed_changes.json") for target in ( AUDIT / "proposed_changes.json", SCRIPTS / ".." / "proposed_changes.json", + Path(os.path.relpath(AUDIT / "proposed_changes.json")), + link, ): monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(target)]) with pytest.raises(SystemExit): @@ -673,33 +679,70 @@ def _run(script: str, tmp_path: Path, *args: str) -> None: assert result.returncode == 0, result.stderr[-4000:] +LINE = re.compile( + rb'^(?P\s*"(?P[a-z_0-9]+)": )(?P.*?)(?P,?\n)$' +) + + def _identical_but(regenerated: Path, committed: Path, volatile: set[str]) -> dict: - """Require the files byte for byte, except the one line carrying each key in - ``volatile``; return the regenerated value of each of those keys.""" + """Require the files byte for byte, except the value on the one line that + carries each key in ``volatile``; return each such regenerated value. + + The committed file must carry each volatile key on exactly one line, and the + regenerated file must carry the same key, indentation and punctuation on the + same line, so only that one JSON value may differ. + """ new = regenerated.read_bytes().splitlines(keepends=True) old = committed.read_bytes().splitlines(keepends=True) assert len(new) == len(old), regenerated.name - line = re.compile(rb'^\s*"(?P[a-z_0-9]+)": (?P.*?),?\n$') - values = {} + values, allowed = {}, set() for key in volatile: lines = [ i for i, text in enumerate(old) - if re.match(rb'^\s*"' + key.encode() + rb'": ', text) + if (match := LINE.match(text)) and match["key"].decode() == key ] - assert len(lines) == 1, (committed.name, key) - values[key] = json.loads(line.match(new[lines[0]])["value"]) - differing = [i for i, (a, b) in enumerate(zip(new, old)) if a != b] + assert len(lines) == 1, (committed.name, key, lines) + (index,) = lines + before, after = LINE.match(old[index]), LINE.match(new[index]) + assert after is not None, (regenerated.name, new[index][:120]) + assert (after["head"], after["tail"]) == (before["head"], before["tail"]) + values[key] = json.loads(after["value"]) + allowed.add(index) unexpected = [ - (i + 1, old[i][:120], new[i][:120]) - for i in differing - if (match := line.match(old[i])) is None - or match["key"].decode() not in volatile + (i + 1, b[:120], a[:120]) + for i, (a, b) in enumerate(zip(new, old)) + if a != b and i not in allowed ] assert not unexpected, (regenerated.name, unexpected[:5]) return values +def test_only_the_named_values_may_differ(tmp_path): + """The comparison is not vacuous: a changed value elsewhere, a moved key or a + changed key name on the volatile line all fail it.""" + committed = tmp_path / "committed.json" + committed.write_text('{\n "a": 1,\n "seconds": 2.5,\n "z": [1]\n}\n') + cases = { + '{\n "a": 1,\n "seconds": 9.1,\n "z": [1]\n}\n': True, + '{\n "a": 2,\n "seconds": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "minutes": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5,\n "z": [2]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5\n}\n': False, + } + for text, passes in cases.items(): + regenerated = tmp_path / "regenerated.json" + regenerated.write_text(text) + if passes: + assert _identical_but(regenerated, committed, {"seconds"}) == { + "seconds": 9.1 + } + else: + with pytest.raises(AssertionError): + _identical_but(regenerated, committed, {"seconds"}) + + @pytest.mark.slow def test_definition_conformance_regenerates_byte_for_byte(tmp_path): """Only the wall time and this script's own sha256 differ from the record. From b4f2aac8ec90f2e5d793c8b4dd8fd08d6b168ad5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 15:04:55 -0400 Subject: [PATCH 5/7] State the slow regenerations' measured time Co-Authored-By: Claude Opus 5.5 --- reference_audit/2026-10-05-reference-adversary/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index b2f3df44..4d8a7e5f 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -560,7 +560,7 @@ The tests regenerate every one of these outputs into scratch and require the com - `engine_probe.py`: all 8 probes, from the arguments each records. - `leaderboard_impact.py`: all 23 `verification/leaderboard_impact*` files. -The engine-side regenerations take about 15 minutes on a loaded machine, so they are marked slow and CI deselects them: +The engine-side regenerations took 21 minutes on a loaded machine, so they are marked slow and CI deselects them: ```bash OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_inputs.py tests/test_reference_adversary_impact.py From f6774fd190157f07962b194fef0aa060afcb521b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 20:20:45 -0400 Subject: [PATCH 6/7] Address the review: pin the definitions the engine helpers read, and stage before the engine Review of b4f2aac8 (GPT-6.1 Sol, hard tier) asked for these changes. - policybench loads benchmark_specs.json itself, as policybench.scenarios is imported, so the engine-side scripts computed with the checkout's definitions while recording the pinned copy's hash. Each process of definition_conformance.py, publication_sources.py and engine_probe.py, workers included, now stages the pinned definitions and points policybench at them (pass_inputs.use_staged_specs) before importing anything else from the package. The test fence no longer exempts the package's own read for these scripts. - publication_sources.py loses --from-facts, which built both reports from a saved file with no pin check. - The conventions and definitions are staged before the engine is imported, hooked or handed to a worker, not after. - Workers are started with the spawn method and an initializer that stages, so a forked worker cannot inherit the main process's system. - build_proposals.py creates --out exclusively, so a hard link to the committed proposed_changes.json or another checkout's copy is refused like any existing file. - leaderboard_impact.py keeps the caller's PYTHONPATH for its scoring child, and its slow regeneration now runs fenced in both processes. - Each script checks that the pass_inputs it imported is the one beside it. No evidence file, reference, exclusion or score changes. Co-Authored-By: Claude Opus 5.5 --- .../2026-10-05-reference-adversary/README.md | 15 +- .../scripts/build_proposals.py | 14 +- .../scripts/definition_conformance.py | 89 +++-- .../scripts/engine_probe.py | 17 +- .../scripts/leaderboard_impact.py | 12 +- .../scripts/pass_inputs.py | 42 ++- .../scripts/publication_sources.py | 85 +++-- tests/test_reference_adversary_impact.py | 65 +++- tests/test_reference_adversary_inputs.py | 334 ++++++++++++++++-- 9 files changed, 566 insertions(+), 107 deletions(-) diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index 4d8a7e5f..5cdf80ab 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -46,7 +46,7 @@ PolicyBench's diagnosis judge (`docs/audit.md`) explains why models miss. Its pr Inputs: -- The pass read the frozen run and its annotations as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Later commits rewrote several of the working tree's copies: release dashboard-data-20261006 (#202) the run's payload, exclusion record and annotations, and #204 one of the reference system's convention modules. So every script in `scripts/` reads the inputs below only through `scripts/pass_inputs.py`. It writes `git show :` to scratch and stops (SystemExit) unless the bytes match the pinned sha256, and each script stages everything it reads before it computes or writes anything. No script reads the working tree's copy. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. +- The pass read the frozen run and its annotations as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Later commits rewrote several of the working tree's copies: release dashboard-data-20261006 (#202) the run's payload, exclusion record and annotations, and #204 one of the reference system's convention modules. So every script in `scripts/` reads the inputs below only through `scripts/pass_inputs.py`. It writes `git show :` to scratch and stops (SystemExit) unless the bytes match the pinned sha256. Each script stages everything it reads before it computes or writes anything, and before it imports the engine. No script reads the working tree's copy. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. | Input | Commit | sha256 | Read by | |---|---|---|---| @@ -58,12 +58,13 @@ Inputs: | `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | | `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `8b4c0ca1` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | - | `policybench/benchmark_specs.json`, the output definitions | `8b4c0ca1` | `ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5` | `definition_conformance.py` | + | `policybench/benchmark_specs.json`, the output definitions | `8b4c0ca1` | `ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` | | `reference_audit/2026-10-05-payroll/program_classification.json`, #194's classification of state paid-leave programs | `049f4f09` | `282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a` | `definition_conformance.py` | | `reference_audit/2026-10-05-reference-adversary/proposed_changes.json` | `4db91b5f` | `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf` | `leaderboard_impact.py` | | `reference_audit/2026-10-05-reference-adversary/verification/definition_conformance.json` | `4db91b5f` | `05e503f1b291c505b90182395c4afb88844dcb45fb2952c004ca09d4273894a0` | `build_proposals.py` | -- The reference system is policyengine-us 2.15.17 plus `latest_final`, built from the convention modules below as `8b4c0ca1` held them. They are read by `definition_conformance.py`, `publication_sources.py` and `engine_probe.py`, and each worker process stages its own copy. `latest_final` loads `latest_conventions` (the nine `latest_c_*` modules) and `latest_md_local_output_scope`. #204 later rewrote `latest_alt_snap_mortgage_residence.py`, which `latest_final` does not load, but `verification/definition_conformance.json` records every module's sha256. +- `policybench` loads `benchmark_specs.json` itself, once per process: first as `policybench.scenarios` is imported, and every later lookup of an output's engine variable or aggregation uses that loaded copy. The three engine-side scripts therefore point the package at their staged copy (`pass_inputs.use_staged_specs`) before they import anything else from it, in the main process and in every worker. A checkout whose definitions have changed since the pass does not change what they compute. `leaderboard_impact.py` is the exception. It scores with `policybench analyze` in a child process and treats the definitions as part of the checkout's scoring code, which its published-reproduction check covers. +- The reference system is policyengine-us 2.15.17 plus `latest_final`, built from the convention modules below as `8b4c0ca1` held them. They are read by `definition_conformance.py`, `publication_sources.py` and `engine_probe.py`. The main process and every worker stage their own copy; workers are started with the spawn method, so none inherits the main process's system. `latest_final` loads `latest_conventions` (the nine `latest_c_*` modules) and `latest_md_local_output_scope`. #204 later rewrote `latest_alt_snap_mortgage_residence.py`, which `latest_final` does not load, but `verification/definition_conformance.json` records every module's sha256. | Module | Commit | sha256 | |---|---|---| @@ -89,8 +90,10 @@ Inputs: | `reference_audit/2026-09-28/fixes/latest_state_conventions.py` | `8b4c0ca1` | `56e62e399d20ba8c8d9c5dc9bc7ea6681e74d2b529fa38561dbc8e8b2a90b442` | - Who checks the pins: - - `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py` and `leaderboard_impact.py` stage every input above that they read, and stop on any sha256 mismatch before they compute or write anything. `pass_inputs.py` is the only file that holds a pin. - - `tests/test_reference_adversary_inputs.py` checks that each commit holds its pinned bytes, that the committed evidence records the same hashes, and that this table lists each pin with the scripts that stage it. It also checks that any changed byte is refused and that each script refuses #202's run before it computes or writes. An audit hook (`tests/working_tree_fence.py`) fails any open of the working tree's copy of a pinned input while the scripts stage and read their inputs, and in every process, workers included, of the slow regenerations below. `tests/test_reference_adversary_impact.py` checks the impact script the same way, from a checkout whose working-tree run and proposals are junk. + - `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py` and `leaderboard_impact.py` stage every input above that they read, and stop on any sha256 mismatch before they compute or write anything. The engine-side scripts also stop before they import the engine or start a worker. `pass_inputs.py` is the only file that holds a pin, and each script checks that the `pass_inputs` it imported is the one beside it. `publication_sources.py` has no mode that builds a report from a saved file. + - `tests/test_reference_adversary_inputs.py` checks that each commit holds its pinned bytes, that the committed evidence records the same hashes, and that this table lists each pin with the scripts that stage it. It also checks that any changed byte is refused, that each script refuses #202's run before it computes or writes, and that a wrong convention or definitions pin stops each script before the engine is imported. + - An audit hook (`tests/working_tree_fence.py`) fails any open of the working tree's copy of a pinned input, `benchmark_specs.json` included. It is armed while the scripts stage and read their inputs, and in every process, workers included, of the slow regenerations below. A copy of the package with changed definitions, put first on the path, does not change the engine variable the scripts' helpers look up. + - `tests/test_reference_adversary_impact.py` checks the impact script from a checkout whose working-tree run and proposals are junk. Its slow regeneration runs under the same hook in the script and in its scoring children, except that `policybench` may read its own `benchmark_specs.json` there. - `consensus-flags` and `adversary-prepare` read the payload and annotations at the paths they are given, and record the payload's path and sha256 in their output. `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. - Top 5 by the payload's `modelStats` order: gpt-6-sol, claude-opus-5.5, gpt-5.6-sol, gpt-6-luna and claude-sonnet-5.5. - `consensus_flags.json` holds the default run, and `consensus_flags_prototype.json` reproduces the 2026-10-05 prototype. @@ -550,7 +553,7 @@ uv run python $a/scripts/leaderboard_impact.py --scratch --out-dir /proposed_changes.json ``` -Each committed probe records its own arguments: `scenario_id`, `period`, the keys of `variables` and `parameters`, and `counterfactual.set`. `build_proposals.py` never writes the committed `proposed_changes.json`, which `leaderboard_impact.py` and the Haiku 5.5 release pin at `3a6e5920…`. It refuses that path for `--out`. +Each committed probe records its own arguments: `scenario_id`, `period`, the keys of `variables` and `parameters`, and `counterfactual.set`. `build_proposals.py` never writes the committed `proposed_changes.json`, which `leaderboard_impact.py` and the Haiku 5.5 release pin at `3a6e5920…`. It refuses that path for `--out`, and it creates `--out` exclusively, so it never writes any file that already exists (a hard link to the committed file, or another checkout's copy, included). The tests regenerate every one of these outputs into scratch and require the committed file byte for byte: diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py index 6bd9a9aa..04032b21 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py @@ -25,8 +25,9 @@ The committed proposed_changes.json is pinned at sha256 PROPOSED_SHA256 below: leaderboard_impact.py reads it at that hash, and the Haiku 5.5 release pins it too. So the -script never writes it. It writes to --out, which must be another path, and a regeneration is -checked by comparing the two: +script never writes it, nor any file that already exists: it creates --out, which must be a +new path (a hard link to the committed file, or another checkout's copy, is an existing file +and is refused like any other). A regeneration is checked by comparing the two: python reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py \\ --out /proposed_changes.json @@ -46,6 +47,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] # The committed records, which the script never writes. @@ -394,10 +398,14 @@ def main() -> None: f"{PROPOSED.relative_to(ROOT)} is pinned at sha256 {PROPOSED_SHA256}; " "write elsewhere and compare" ) + if out.exists() or out.is_symlink(): + parser.error(f"{out} exists; --out must be a new file") with tempfile.TemporaryDirectory(prefix="build_proposals_inputs_") as scratch: records = build(stage_inputs(Path(scratch))) out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(records, indent=2) + "\n") + # Exclusive creation: never writes through to a file that already exists. + with out.open("x") as handle: + handle.write(json.dumps(records, indent=2) + "\n") print(f"wrote {out}") diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py index a6205f24..e014796c 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py @@ -28,12 +28,15 @@ reference_audit/2026-10-05-payroll/program_classification.json), read from git. Every input is the pass's, staged from git by pass_inputs.py and checked against its -pinned sha256 before anything is computed: the frozen run's payload, references, -reference sidecar and scenarios and the output definitions (benchmark_specs.json) as -release dashboard-data-20260930 (8b4c0ca1) committed them, #194's table, and -latest_final with its parts as 8b4c0ca1 held them (each worker stages its own copy). -The working tree's run, which later releases rewrite, is never read. The run record -names each input by its repository path. +pinned sha256 before anything is computed and before the engine is imported: the +frozen run's payload, references, reference sidecar and scenarios and the output +definitions (benchmark_specs.json) as release dashboard-data-20260930 (8b4c0ca1) +committed them, #194's table, and latest_final with its parts as 8b4c0ca1 held them. +The main process and every worker (started with the spawn method, so none inherits +the main process's system) stage their own conventions and output definitions and +point policybench at the staged definitions. The working tree's run, which later +releases rewrite, is never read. The run record names each input by its repository +path. Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout needs 8b4c0ca1 and 049f4f09 in its history): @@ -53,6 +56,7 @@ import hashlib import importlib.util import json +import multiprocessing import shutil import sys import tempfile @@ -67,6 +71,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + ROOT = Path(__file__).resolve().parents[3] # The frozen run's files main reads, staged from git. RUN_FILES = ( @@ -113,33 +120,37 @@ ) _SYSTEM = None -# The directory this process's reference system was built from. +# This process's staged conventions (a directory) and output definitions (a file). _FIX_DIR = None +_SPECS = None def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads, - staged from git (pass_inputs.FIXES_SHA256).""" - global _FIX_DIR - target = Path(tempfile.mkdtemp(prefix="definition_conformance_fixes_")) - _FIX_DIR = pass_inputs.stage_fixes(target) - return _FIX_DIR +def stage_engine_inputs() -> Path: + """Stage this process's reference-system inputs, before the engine is imported. + + latest_final and its parts, plus the sales tax table the IRS module reads + (pass_inputs.FIXES_SHA256), go into a new directory, which is returned; the + output definitions go beside it and policybench is pointed at them. Each is + refused unless it matches its pin. The main process and every worker call this. + """ + global _FIX_DIR, _SPECS + scratch = Path(tempfile.mkdtemp(prefix="definition_conformance_engine_")) + atexit.register(shutil.rmtree, scratch, True) + fix_dir = pass_inputs.stage_fixes(scratch / "fixes") + specs = pass_inputs.stage_specs(scratch / Path(pass_inputs.SPECS_PATH).name) + pass_inputs.use_staged_specs(specs) + _FIX_DIR, _SPECS = fix_dir, specs + return fix_dir def stage_inputs(target: Path) -> Path: - """Stage what main reads besides the engine: the run's files under ``run/``, the - output definitions and #194's law table, each refused unless it matches its pin.""" + """Stage what main reads besides the reference system's inputs: the run's files + under ``run/`` and #194's law table, each refused unless it matches its pin.""" pass_inputs.stage_run(target / "run", RUN_FILES) - pass_inputs.git_input( - pass_inputs.PASS_COMMIT, - pass_inputs.SPECS_PATH, - pass_inputs.SPECS_SHA256, - target / Path(pass_inputs.SPECS_PATH).name, - ) pass_inputs.git_input( pass_inputs.LAW_COMMIT, pass_inputs.LAW_PATH, @@ -161,12 +172,27 @@ def _load_reform(fix_dir: Path): def _system(): global _SYSTEM if _SYSTEM is None: + # Staged and checked before the engine is imported. + fix_dir = _FIX_DIR or stage_engine_inputs() from policyengine_us import CountryTaxBenefitSystem - _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(_assemble_fixes())) + _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(fix_dir)) return _SYSTEM +def worker_pool(workers: int) -> ProcessPoolExecutor: + """A pool whose workers each stage their own reference-system inputs. + + Spawned, never forked: a forked worker would inherit the main process's built + system and policybench state instead of staging its own. + """ + return ProcessPoolExecutor( + max_workers=workers, + mp_context=multiprocessing.get_context("spawn"), + initializer=stage_engine_inputs, + ) + + def build_situation(scenario) -> dict: situation = scenario.to_pe_household() for person in situation["people"].values(): @@ -511,6 +537,15 @@ def main() -> None: parser.add_argument("--scenarios", nargs="*") args = parser.parse_args() + started = time.time() + # Every input is staged and checked before anything else: before policybench + # is imported (it reads its output definitions as it loads), before the + # engine is imported and before any worker starts. + staged = Path(tempfile.mkdtemp(prefix="definition_conformance_inputs_")) + atexit.register(shutil.rmtree, staged, True) + stage_inputs(staged) + stage_engine_inputs() + from policybench.definition_conformance import ( HOUSEHOLD_PROMPT_PHRASE, component_tree, @@ -519,13 +554,9 @@ def main() -> None: variable_graph_from_system, ) - started = time.time() out_dir = Path(args.out_dir) - staged = Path(tempfile.mkdtemp(prefix="definition_conformance_inputs_")) - atexit.register(shutil.rmtree, staged, True) - stage_inputs(staged) run = staged / "run" - specs_path = staged / Path(pass_inputs.SPECS_PATH).name + specs_path = _SPECS payload = json.loads(gzip.decompress((run / "data.json.gz").read_bytes())) meta = json.loads((run / "reference_outputs.csv.meta.json").read_text()) programs = list(meta["programs"]) @@ -635,7 +666,7 @@ def main() -> None: f"{len(scored)} scored cells, {len(node_variables)} node variables", flush=True, ) - with ProcessPoolExecutor(max_workers=args.workers) as pool: + with worker_pool(args.workers) as pool: results = list(pool.map(scenario_cells, jobs)) results.sort(key=lambda result: result["scenario_id"]) by_scenario = {result["scenario_id"]: result for result in results} diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py index 7c1bf483..da9d7789 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py @@ -15,11 +15,12 @@ verdicts; it reads no verdict and changes no reference. Every input is the pass's, staged from git by pass_inputs.py and checked against -its pinned sha256 before the reference system is built: the frozen run's -scenarios, references and exclusion record as release dashboard-data-20260930 -(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them. -The working tree's run, which later releases rewrite (#202 rewrote its exclusion -record), is never read. Each committed probe records its own arguments +its pinned sha256 before anything is computed and before the engine is imported: +the frozen run's scenarios, references and exclusion record and the output +definitions (benchmark_specs.json) as release dashboard-data-20260930 (8b4c0ca1) +committed them, and latest_final with its parts as 8b4c0ca1 held them. policybench +is pointed at the staged definitions. The working tree's run, which later +releases rewrite (#202 rewrote its exclusion record), is never read. Each committed probe records its own arguments (``scenario_id``, ``period``, the ``variables`` and ``parameters`` keys and ``counterfactual.set``), and tests/test_reference_adversary_inputs.py regenerates every probe from them. @@ -51,6 +52,9 @@ sys.path.insert(0, str(HERE)) import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) +if Path(pass_inputs.__file__).resolve().parent != HERE: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + YEAR = 2026 INSTANT = "2026-01-01" # The frozen run's files the probe reads, staged from git. @@ -192,6 +196,9 @@ def main() -> None: atexit.register(shutil.rmtree, run, True) stage_inputs(run) conformance = _conformance() + # The conventions and output definitions, before policybench or the engine + # is imported. + conformance.stage_engine_inputs() scenarios = pd.read_csv(run / "scenarios.csv") rows = scenarios[scenarios["scenario_id"] == args.scenario_id] if rows.empty: diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py index 1ba7c54e..3a47a0fb 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py @@ -57,6 +57,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] OUT_DIR = HERE / "verification" @@ -206,10 +209,17 @@ def stage(source: Path, target: Path, records: dict[str, list[dict]] | None) -> return target +def analyze_env() -> dict[str, str]: + """The scoring child's environment: this checkout first on PYTHONPATH, with the + caller's own entries kept after it.""" + path = os.pathsep.join(filter(None, [str(ROOT), os.environ.get("PYTHONPATH")])) + return dict(os.environ, PYTHONPATH=path, PYTHONDONTWRITEBYTECODE="1") + + def analyze(run_dir: Path) -> dict: out = run_dir / "analysis" dashboard = run_dir / "dashboard.json" - env = dict(os.environ, PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE="1") + env = analyze_env() subprocess.run( [ sys.executable, diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py index b39b37ee..356ad96e 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py @@ -9,8 +9,17 @@ these inputs only through ``git_input``: ``git show :`` into scratch, refused (SystemExit) unless the bytes match the pinned sha256, and read only from the staged copy. A script stages everything it reads before it computes or writes -anything. The code (this checkout's ``policybench`` package and the installed engine) -is not pinned here; each script's own reproduction check covers it. +anything, and before it imports the engine. The code (this checkout's ``policybench`` +package and the installed engine) is not pinned here; each script's own reproduction +check covers it. + +``policybench`` loads its output definitions (``benchmark_specs.json``) itself, once +per process: first at import of ``policybench.scenarios``, and every spec lookup then +uses that copy. Those are an input, so ``use_staged_specs`` points the package at the +staged copy; the engine-side scripts call it in every process, workers included, +before they import anything else from ``policybench``. leaderboard_impact.py does not: it scores with ``policybench analyze`` +in a child process and treats the definitions as part of the scoring code its +published-reproduction check covers. The README's Inputs table lists every pin and the scripts that read it, and tests/test_reference_adversary_inputs.py checks that the table and these pins agree. @@ -19,6 +28,7 @@ from __future__ import annotations import hashlib +import json import shutil import subprocess from collections.abc import Iterable @@ -183,3 +193,31 @@ def stage_fixes(target: Path) -> Path: for path, pinned in FIXES_SHA256.items(): git_input(PASS_COMMIT, path, pinned, target / Path(path).name) return target + + +def stage_specs(target: Path) -> Path: + """The output definitions as PASS_COMMIT holds them, written to ``target``.""" + return git_input(PASS_COMMIT, SPECS_PATH, SPECS_SHA256, target) + + +def use_staged_specs(path: Path) -> None: + """Make ``policybench`` read its output definitions from the staged ``path``. + + For this process only; call it before importing anything else from + ``policybench``, because ``policybench.scenarios`` derives module constants from + the definitions as it is imported. If the package has already loaded its + definitions (a long-lived interpreter, such as a test run), they must equal the + staged ones, or the script stops: constants derived from other definitions + cannot be recalled. + """ + from policybench import spec + + staged = json.loads(path.read_text(encoding="utf-8")) + if spec._raw_spec_data.cache_info().currsize and spec._raw_spec_data() != staged: + raise SystemExit( + "policybench already loaded output definitions that are not the pinned " + f"{SPECS_PATH}; run this script in a fresh interpreter" + ) + spec._spec_path = lambda: path + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py index 556e2267..289c5f42 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py @@ -59,12 +59,16 @@ not reproduce. Every input is the pass's, staged from git by pass_inputs.py and checked against its -pinned sha256 before any system is built: the frozen run's payload, references, -reference sidecar, exclusion record and scenarios as release dashboard-data-20260930 -(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them (each -worker stages its own copy). The working tree's run, which later releases rewrite -(#202 rewrote its payload and exclusion record), is never read. The report names each -input by its repository path. +pinned sha256 before anything is computed and before the engine is imported: the +frozen run's payload, references, reference sidecar, exclusion record and scenarios +and the output definitions (benchmark_specs.json) as release dashboard-data-20260930 +(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them. The +main process and every worker (started with the spawn method) stage their own +conventions and output definitions and point policybench at the staged definitions. +The working tree's run, which later releases rewrite (#202 rewrote its payload and +exclusion record), is never read. The report names each input by its repository +path. The script always runs the engine: it has no mode that builds a report from a +saved file. Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout needs 8b4c0ca1 in its history): @@ -83,6 +87,7 @@ import importlib.util import json import math +import multiprocessing import os import shutil import sys @@ -99,6 +104,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + # The frozen run's files main reads, staged from git. RUN_FILES = ( "data.json.gz", @@ -136,12 +144,20 @@ def _jsonable(value): return repr(value) -def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads, - staged from git (pass_inputs.FIXES_SHA256).""" - target = Path(tempfile.mkdtemp(prefix="publication_sources_fixes_")) - atexit.register(shutil.rmtree, target, True) - return pass_inputs.stage_fixes(target) +def stage_engine_inputs() -> Path: + """Stage this process's reference-system inputs, before the engine is imported. + + latest_final and its parts, plus the sales tax table the IRS module reads + (pass_inputs.FIXES_SHA256), go into a new directory, which is returned; the + output definitions go beside it and policybench is pointed at them. Each is + refused unless it matches its pin. The main process and every worker call this. + """ + scratch = Path(tempfile.mkdtemp(prefix="publication_sources_engine_")) + atexit.register(shutil.rmtree, scratch, True) + fix_dir = pass_inputs.stage_fixes(scratch / "fixes") + specs = pass_inputs.stage_specs(scratch / Path(pass_inputs.SPECS_PATH).name) + pass_inputs.use_staged_specs(specs) + return fix_dir def _load_reform(fix_dir: Path): @@ -334,10 +350,25 @@ def scale_at_instant(self, instant): def _init_worker() -> None: global _SYSTEM + # Staged and checked before the engine is imported. + fix_dir = stage_engine_inputs() install_recording() from policyengine_us import CountryTaxBenefitSystem - _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(_assemble_fixes())) + _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(fix_dir)) + + +def worker_pool(workers: int) -> ProcessPoolExecutor: + """A pool whose workers each stage their own reference-system inputs. + + Spawned, never forked: a forked worker would inherit the main process's + policybench and build-hook state. + """ + return ProcessPoolExecutor( + max_workers=workers, + mp_context=multiprocessing.get_context("spawn"), + initializer=_init_worker, + ) def _output_value(sim, scenario, variable: str) -> float: @@ -777,30 +808,16 @@ def main() -> None: parser.add_argument("--scenarios", nargs="*") parser.add_argument("--workers", type=int, default=6) parser.add_argument("--facts-out", help="also write the unclassified facts here") - parser.add_argument( - "--from-facts", - help="skip the engine: rebuild the report from a --facts-out file", - ) args = parser.parse_args() - if args.from_facts: - from policybench.publication_sources import build_report, dumps_report - - saved = json.loads(Path(args.from_facts).read_text()) - report, markdown = build_report( - saved["facts"], saved["other_instants"], saved["meta"] - ) - out_dir = Path(args.out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "publication_sources.json").write_text( - dumps_report(report) - ) - (out_dir / "publication_sources.md").write_text(markdown) - return started = time.time() + # Every input is staged and checked before anything else: before policybench + # is imported (it reads its output definitions as it loads), before the + # engine is imported or hooked and before any worker starts. run = Path(tempfile.mkdtemp(prefix="publication_sources_inputs_")) atexit.register(shutil.rmtree, run, True) stage_inputs(run) + fix_dir = stage_engine_inputs().resolve() install_build_hooks() BUILD["label"] = "plain" @@ -841,9 +858,7 @@ def main() -> None: (row["scenario_json"], variables, position % REVERSE_CHECK_EVERY == 0) ) - with ProcessPoolExecutor( - max_workers=min(args.workers, len(jobs)), initializer=_init_worker - ) as pool: + with worker_pool(min(args.workers, len(jobs))) as pool: futures = [pool.submit(record_scenario, job) for job in jobs] # Build both systems here while the workers run. @@ -853,7 +868,7 @@ def main() -> None: plain_system = policyengine_us.system.system BUILD["label"] = "reference" - BUILD["fix_dir"] = _assemble_fixes().resolve() + BUILD["fix_dir"] = fix_dir reference_system = CountryTaxBenefitSystem( reform=_load_reform(BUILD["fix_dir"]) ) diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py index 64022da4..87d09d11 100644 --- a/tests/test_reference_adversary_impact.py +++ b/tests/test_reference_adversary_impact.py @@ -30,6 +30,29 @@ sys.path.insert(0, str(SCRIPT.parent)) import pass_inputs as pins # noqa: E402 +from tests import working_tree_fence as fence # noqa: E402 + +# The working tree's copy of every input the script stages. The script scores with +# the checkout's policybench, which may read its own output definitions. +FENCE = { + "forbidden": [ + str(ROOT / path) + for path in (pins.RUN_PATH, pins.PROPOSALS_PATH, pins.SPECS_PATH) + ], + "exempt": {str(ROOT / pins.SPECS_PATH): str(ROOT / "policybench")}, + "root": str(ROOT), +} + + +def _fenced_env(directory: Path) -> dict[str, str]: + """A subprocess environment whose every interpreter is fenced.""" + return dict( + os.environ, + PYTHONPATH=f"{fence.sitecustomize(directory)}{os.pathsep}{ROOT}", + **{fence.ENV: json.dumps(FENCE)}, + ) + + # The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" @@ -195,12 +218,48 @@ def analyze(run_dir: Path) -> dict: assert summary["proposals"] == "proposed_changes.json" +READ = "import sys; print(len(open(sys.argv[1], 'rb').read()))" +LOAD = "from policybench import spec; print(len(spec._raw_spec_data()))" + + +def test_the_scoring_child_keeps_the_callers_path_and_so_the_fence( + tmp_path, monkeypatch +): + """analyze() used to replace PYTHONPATH, which dropped a caller's sitecustomize + from the scoring child. Under the fenced environment the slow test uses, a + child started as analyze() starts it cannot open the working tree's run, while + policybench can still load its own output definitions.""" + for name, value in _fenced_env(tmp_path / "fence").items(): + monkeypatch.setenv(name, value) + env = impact.analyze_env() + assert env["PYTHONPATH"].split(os.pathsep)[0] == str(impact.ROOT) + assert str(tmp_path / "fence") in env["PYTHONPATH"].split(os.pathsep) + + def child(code: str, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", code, *args], env=env, capture_output=True, text=True + ) + + for name in ("scenarios.csv", "reference_exclusions.json"): + refused = child(READ, str(ROOT / pins.RUN_PATH / name)) + assert refused.returncode != 0 + assert "opened the working tree's" in refused.stderr + assert child(READ, str(ROOT / pins.SPECS_PATH)).returncode != 0 + loaded = child(LOAD) + assert loaded.returncode == 0, loaded.stderr + # Unfenced, the same child reads the file: the refusals above were the fence's. + monkeypatch.delenv(fence.ENV) + env = impact.analyze_env() + assert child(READ, str(ROOT / pins.RUN_PATH / "scenarios.csv")).returncode == 0 + + @pytest.mark.slow def test_regeneration_reproduces_the_committed_evidence(tmp_path): """Score every variant for real and require the committed files byte for byte. - About five minutes of ``policybench analyze`` runs, so CI deselects it. Run it - with + The script and its scoring children run fenced: none may open the working + tree's run or proposals. About five minutes of ``policybench analyze`` runs, + so CI deselects it. Run it with OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ tests/test_reference_adversary_impact.py """ @@ -216,7 +275,7 @@ def test_regeneration_reproduces_the_committed_evidence(tmp_path): str(out), ], cwd=ROOT, - env=dict(os.environ, PYTHONPATH=str(ROOT)), + env=_fenced_env(tmp_path / "fence"), capture_output=True, text=True, ) diff --git a/tests/test_reference_adversary_inputs.py b/tests/test_reference_adversary_inputs.py index 3d69f40e..02dce9d8 100644 --- a/tests/test_reference_adversary_inputs.py +++ b/tests/test_reference_adversary_inputs.py @@ -12,9 +12,10 @@ committed evidence records the same hashes, and the README's Inputs table lists each pin with the scripts that stage it; * any other bytes are refused, and each script refuses before it computes or - writes anything; + writes anything and before it imports the engine; * no script opens the working tree's copy of a pinned input (an audit hook - fails any such open); + fails any such open), policybench's own read of its output definitions + included: the engine-side scripts point the package at their staged copy; * each script regenerates its committed outputs byte for byte. The engine-side regenerations need policyengine-us 2.15.17 and take minutes, so they are marked slow and CI deselects them; build_proposals.py's regeneration is fast. @@ -31,6 +32,7 @@ import json import os import re +import shutil import subprocess import sys import tempfile @@ -42,7 +44,7 @@ import pandas as pd import pytest -from hypothesis import given, settings +from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st ROOT = Path(__file__).resolve().parents[1] @@ -109,7 +111,8 @@ def _load(name: str): # A fence on the working tree's copy of every pinned input # --------------------------------------------------------------------------- -# The working tree's copies; policybench may read its own benchmark_specs.json. +# The working tree's copies, policybench's own benchmark_specs.json included: the +# scripts here point the package at a staged copy before it reads the file. FENCE = { "forbidden": [ str(ROOT / path) @@ -122,11 +125,29 @@ def _load(name: str): pins.CONFORMANCE_PATH, ) ], - "exempt": {str(ROOT / pins.SPECS_PATH): str(ROOT / "policybench")}, + "exempt": {}, "root": str(ROOT), } +@pytest.fixture(autouse=True) +def package_specs(): + """Start each test with policybench's output definitions unloaded, and point + the package back at its own file afterwards: the engine-side scripts repoint + it at their staged copy.""" + from policybench import spec + + original = spec._spec_path + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() + try: + yield spec + finally: + spec._spec_path = original + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() + + @contextmanager def fenced(): """Inside, opening the working tree's copy of any pinned input raises.""" @@ -148,7 +169,7 @@ def _fenced_env(directory: Path) -> dict[str, str]: ) -def test_the_fence_catches_a_working_tree_read(): +def test_the_fence_catches_a_working_tree_read(package_specs): """The hook is not vacuous: each way the scripts read files trips it.""" run = ROOT / pins.RUN_PATH with fenced(): @@ -160,13 +181,9 @@ def test_the_fence_catches_a_working_tree_read(): (ROOT / MODULE_204).read_text() with pytest.raises(PermissionError, match="working tree"): (ROOT / pins.CONFORMANCE_PATH).read_bytes() - # Only policybench's own code may read its output definitions. + # policybench's own read of its output definitions trips it too. with pytest.raises(PermissionError, match="working tree"): - (ROOT / pins.SPECS_PATH).read_bytes() - from policybench import spec - - spec._raw_spec_data.cache_clear() - assert spec._raw_spec_data() + package_specs._raw_spec_data() # git show reads the object store, not the working tree. assert pins.git_bytes( pins.PASS_COMMIT, MODULE_204, pins.FIXES_SHA256[MODULE_204] @@ -288,11 +305,11 @@ def record(commit, path, pinned, target): else: SCRIPT_MODULES[script].stage_inputs(tmp_path / script) if script in ("definition_conformance", "publication_sources"): - SCRIPT_MODULES[script]._assemble_fixes() + SCRIPT_MODULES[script].stage_engine_inputs() if script == "engine_probe": # The probe builds its reference system with definition_conformance's - # builder, which stages the convention modules. - conformance._assemble_fixes() + # builder, which stages the conventions and output definitions. + conformance.stage_engine_inputs() return seen @@ -407,7 +424,13 @@ def _shown(data: bytes) -> SimpleNamespace: ) -@settings(max_examples=60, deadline=None) +@settings( + max_examples=60, + deadline=None, + # The autouse fixture only resets policybench's spec cache, which this + # property never touches. + suppress_health_check=[HealthCheck.function_scoped_fixture], +) @given(_edits()) def test_any_changed_byte_in_any_pinned_input_is_refused_unwritten(edit): """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" @@ -453,7 +476,7 @@ def test_the_reference_system_refuses_other_convention_modules( """#204's rewrite of a convention module (on 4db91b5f) is refused.""" monkeypatch.setattr(pins, attribute, value) with pytest.raises(SystemExit, match=re.escape(message)): - SCRIPT_MODULES[script]._assemble_fixes() + SCRIPT_MODULES[script].stage_engine_inputs() def _never(name: str): @@ -471,6 +494,15 @@ def _argv(script: str, out: Path) -> list[str]: return [script, "--out-dir", str(out)] +# What each script calls only after every input is staged. +ENGINE_ENTRY_POINTS = ( + "_system", + "stage_engine_inputs", + "install_build_hooks", + "install_recording", + "worker_pool", + "_conformance", +) # The first input each script stages that #202 rewrote. FIRST_REWRITTEN = { "definition_conformance": "data.json.gz", @@ -487,7 +519,7 @@ def test_a_mismatched_input_stops_the_script_before_it_computes_or_writes( """Pointed at #202's run, each script refuses before any engine work or write.""" module = SCRIPT_MODULES[script] monkeypatch.setattr(pins, "PASS_COMMIT", RELEASE_20261006_COMMIT) - for name in ("_system", "_assemble_fixes", "install_build_hooks", "_conformance"): + for name in ENGINE_ENTRY_POINTS: if hasattr(module, name): monkeypatch.setattr(module, name, _never(name)) monkeypatch.setattr(module, "build", _never("build"), raising=False) @@ -505,6 +537,121 @@ def test_a_mismatched_input_stops_the_script_before_it_computes_or_writes( assert not out.exists() +def _bad_pin(monkeypatch, which: str) -> None: + """Make a late-staged pin wrong: a convention module or the definitions.""" + if which == "conventions": + last = list(pins.FIXES_SHA256)[-1] + monkeypatch.setattr(pins, "FIXES_SHA256", {**pins.FIXES_SHA256, last: "0" * 64}) + else: + monkeypatch.setattr(pins, "SPECS_SHA256", "0" * 64) + + +def _no_engine(monkeypatch) -> None: + """Any import of the engine from here on fails with ImportError.""" + for name in ("policyengine_us", "policyengine_core"): + monkeypatch.setitem(sys.modules, name, None) + + +@pytest.mark.parametrize("which", ["conventions", "definitions"]) +@pytest.mark.parametrize( + "script", ["definition_conformance", "publication_sources", "engine_probe"] +) +def test_a_late_staged_input_stops_each_main_before_the_engine( + script, which, tmp_path, monkeypatch +): + """The reference system's inputs are staged last. A wrong pin there still stops + the script before it imports the engine, hooks it, starts a worker or writes.""" + module = SCRIPT_MODULES[script] + _bad_pin(monkeypatch, which) + _no_engine(monkeypatch) + for name in ("_system", "install_build_hooks", "install_recording", "worker_pool"): + if hasattr(module, name): + monkeypatch.setattr(module, name, _never(name)) + if script == "engine_probe": + builder = SimpleNamespace( + stage_engine_inputs=conformance.stage_engine_inputs, + _system=_never("_system"), + _simulation=_never("_simulation"), + build_situation=_never("build_situation"), + ) + monkeypatch.setattr(probe, "_conformance", lambda: builder) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv(script, out)) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + module.main() + assert not out.exists() + + +@pytest.mark.parametrize("which", ["conventions", "definitions"]) +def test_a_worker_stages_its_inputs_before_it_imports_the_engine(which, monkeypatch): + """The paths a worker takes: definition_conformance's lazy system builder and + publication_sources's initializer both check the pins first.""" + _bad_pin(monkeypatch, which) + _no_engine(monkeypatch) + monkeypatch.setattr(conformance, "_SYSTEM", None) + monkeypatch.setattr(conformance, "_FIX_DIR", None) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + conformance._system() + monkeypatch.setattr(sources, "install_recording", _never("install_recording")) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + sources._init_worker() + + +def test_workers_are_spawned_and_stage_their_own_inputs(): + """Never forked: a forked worker would inherit the main process's system and + policybench state instead of staging its own. Each worker's initializer stages.""" + for module, initializer in ( + (conformance, conformance.stage_engine_inputs), + (sources, sources._init_worker), + ): + pool = module.worker_pool(1) + try: + assert pool._mp_context.get_start_method() == "spawn" + assert pool._initializer is initializer + finally: + pool.shutdown() + + +def test_publication_sources_has_no_mode_that_skips_staging(tmp_path, monkeypatch): + """--from-facts built both reports from a saved file with no pin check.""" + for name in ("stage_inputs", *ENGINE_ENTRY_POINTS): + if hasattr(sources, name): + monkeypatch.setattr(sources, name, _never(name)) + facts = tmp_path / "facts.json" + facts.write_text(json.dumps({"facts": [], "other_instants": [], "meta": {}})) + out = tmp_path / "out" + monkeypatch.setattr( + sys, + "argv", + ["publication_sources", "--out-dir", str(out), "--from-facts", str(facts)], + ) + with pytest.raises(SystemExit) as refused: + sources.main() + assert refused.value.code == 2 + assert not out.exists() + + +def test_a_script_refuses_another_module_named_pass_inputs(tmp_path): + """A pass_inputs already imported from elsewhere would win over the scripts' + directory; each script checks where its pins came from.""" + driver = tmp_path / "driver.py" + driver.write_text( + "import runpy, sys, types\n" + "fake = types.ModuleType('pass_inputs')\n" + "fake.__file__ = '/elsewhere/pass_inputs.py'\n" + "sys.modules['pass_inputs'] = fake\n" + "runpy.run_path(sys.argv[1], run_name='not_main')\n" + ) + for path in sorted(SCRIPTS.glob("*.py")): + if "import pass_inputs" not in path.read_text(): + continue + result = subprocess.run( + [sys.executable, str(driver), str(path)], capture_output=True, text=True + ) + assert result.returncode != 0, path.name + assert "/elsewhere/pass_inputs.py, not this script's" in result.stderr + + @pytest.mark.parametrize( ("attribute", "value", "message"), [ @@ -554,6 +701,46 @@ def test_build_proposals_never_writes_the_committed_records( assert hashlib.sha256(committed).hexdigest() == pins.PROPOSALS_SHA256 +def test_build_proposals_writes_only_a_new_file(tmp_path, monkeypatch, capsys): + """A hard link to the committed records, another checkout's copy and a dangling + symlink are existing paths, and no existing path is written.""" + committed = AUDIT / "proposed_changes.json" + before = committed.read_bytes() + monkeypatch.setattr(proposals, "build", _never("build")) + hard = tmp_path / "hard.json" + os.link(committed, hard) + other = tmp_path / "checkout" / pins.PROPOSALS_PATH + other.parent.mkdir(parents=True) + other.write_bytes(before) + dangling = tmp_path / "dangling.json" + dangling.symlink_to(tmp_path / "nowhere.json") + for target in (hard, other, dangling): + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(target)]) + with pytest.raises(SystemExit): + proposals.main() + assert "--out must be a new file" in capsys.readouterr().err + assert committed.read_bytes() == hard.read_bytes() == other.read_bytes() == before + assert not (tmp_path / "nowhere.json").exists() + + +def test_build_proposals_creates_its_output_exclusively(tmp_path, monkeypatch): + """A file that appears at --out after the check is not written through: here + it is a hard link to the committed records, created while the script builds.""" + committed = AUDIT / "proposed_changes.json" + before = committed.read_bytes() + out = tmp_path / "proposed_changes.json" + + def build(inputs): + os.link(committed, out) + return {"root_causes": {}} + + monkeypatch.setattr(proposals, "build", build) + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(out)]) + with pytest.raises(FileExistsError): + proposals.main() + assert committed.read_bytes() == out.read_bytes() == before + + # --------------------------------------------------------------------------- # The working tree is never read # --------------------------------------------------------------------------- @@ -583,20 +770,120 @@ def test_definition_conformance_reads_only_staged_inputs(tmp_path): ) for name in conformance.RUN_FILES: assert pins.sha256(staged / "run" / name) == pins.RUN_SHA256[name] - assert pins.sha256(staged / "benchmark_specs.json") == pins.SPECS_SHA256 recorded = json.loads((VERIFICATION / "definition_conformance.json").read_text()) assert meta == recorded["run"]["law_classification"] assert laws -def test_the_reference_system_is_built_from_staged_modules(): +def test_the_reference_system_is_built_from_staged_inputs(package_specs): + """Conventions and output definitions are staged, and policybench then reads + the staged definitions: with the working tree fenced off, its lookups work.""" + from policybench.ground_truth import _pe_variable_for_output + expected = {Path(path).name: sha for path, sha in pins.FIXES_SHA256.items()} + pinned = json.loads( + pins.git_bytes(pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256) + ) for module in (conformance, sources): with fenced(): - fix_dir = module._assemble_fixes() + fix_dir = module.stage_engine_inputs() + assert package_specs._raw_spec_data() == pinned + assert _pe_variable_for_output("snap", "us") == "snap" assert {path.name: pins.sha256(path) for path in fix_dir.iterdir()} == expected - # definition_conformance records the hashes of the directory it built from. - assert conformance._assemble_fixes() == conformance._FIX_DIR + assert pins.sha256(package_specs._spec_path()) == pins.SPECS_SHA256 + # definition_conformance records the hashes of what it staged. + assert conformance.stage_engine_inputs() == conformance._FIX_DIR + assert pins.sha256(conformance._SPECS) == pins.SPECS_SHA256 + + +def _mutated_specs(target: Path) -> tuple[Path, str]: + """The pinned definitions with the US ``snap`` output's engine variable changed.""" + data = json.loads( + pins.git_bytes(pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256) + ) + outputs = data["specs"]["policybench"]["countries"]["us"] + (snap,) = [output for output in outputs if output["id"] == "snap"] + assert snap["pe_variable"] == "snap" + snap["pe_variable"] = "not_the_pass_variable" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(data)) + return target, "not_the_pass_variable" + + +def test_staged_definitions_replace_a_checkouts_changed_ones(tmp_path, package_specs): + """A checkout whose benchmark_specs.json maps an output to another engine + variable does not change what the scripts compute: lookups read the pin.""" + from policybench.ground_truth import _pe_variable_for_output + + mutated, variable = _mutated_specs(tmp_path / "benchmark_specs.json") + package_specs._spec_path = lambda: mutated + assert _pe_variable_for_output("snap", "us") == variable + package_specs._raw_spec_data.cache_clear() + package_specs.get_benchmark_spec.cache_clear() + conformance.stage_engine_inputs() + assert _pe_variable_for_output("snap", "us") == "snap" + + +def test_definitions_already_loaded_from_elsewhere_are_refused(tmp_path, package_specs): + """Constants derived from other definitions cannot be recalled, so a process + that already loaded them is refused, not silently repointed.""" + mutated, _ = _mutated_specs(tmp_path / "benchmark_specs.json") + package_specs._spec_path = lambda: mutated + package_specs._raw_spec_data() + staged = pins.stage_specs(tmp_path / "staged.json") + with pytest.raises(SystemExit, match="fresh interpreter"): + pins.use_staged_specs(staged) + assert package_specs._spec_path() == mutated + + +SPECS_DRIVER = """ +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[1]) +import pass_inputs + +staged = pass_inputs.stage_specs(Path(sys.argv[2])) +if sys.argv[3] == "staged": + pass_inputs.use_staged_specs(staged) +from policybench import scenarios, spec # scenarios derives constants as it loads +from policybench.ground_truth import _pe_variable_for_output + +print(spec.__file__) +print(_pe_variable_for_output("snap", "us")) +""" + + +def test_a_mutated_checkout_specs_file_never_reaches_the_engine_helpers(tmp_path): + """End to end in fresh interpreters: with a copy of the package whose + benchmark_specs.json is changed first on the path, the helpers the scripts + call return the pinned mapping once the staged definitions are in use, and + the changed one otherwise.""" + package = tmp_path / "checkout" / "policybench" + shutil.copytree( + ROOT / "policybench", package, ignore=shutil.ignore_patterns("__pycache__") + ) + _, variable = _mutated_specs(package / "benchmark_specs.json") + driver = tmp_path / "driver.py" + driver.write_text(SPECS_DRIVER) + seen = {} + for mode in ("staged", "checkout"): + result = subprocess.run( + [ + sys.executable, + str(driver), + str(SCRIPTS), + str(tmp_path / f"{mode}.json"), + mode, + ], + env=dict(os.environ, PYTHONPATH=str(package.parent)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-2000:] + module, seen[mode] = result.stdout.split() + assert Path(module).parent == package + assert seen == {"staged": "snap", "checkout": variable} def test_publication_sources_reads_only_staged_inputs(tmp_path): @@ -634,6 +921,7 @@ def test_engine_probe_scores_the_pass_cells_not_202s(tmp_path, monkeypatch): for row in reference[reference["scenario_id"] == "scenario_043"].itertuples() } engine = SimpleNamespace( + stage_engine_inputs=conformance.stage_engine_inputs, build_situation=conformance.build_situation, _system=lambda: SimpleNamespace(variables={}), _simulation=lambda situation: None, From 609d5a7f3ee630dd59cd32dc1889713032e7c4ce Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 20:53:46 -0400 Subject: [PATCH 7/7] State the slow regenerations' measured range Co-Authored-By: Claude Opus 5.5 --- reference_audit/2026-10-05-reference-adversary/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index 5cdf80ab..f5d4efe3 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -563,7 +563,7 @@ The tests regenerate every one of these outputs into scratch and require the com - `engine_probe.py`: all 8 probes, from the arguments each records. - `leaderboard_impact.py`: all 23 `verification/leaderboard_impact*` files. -The engine-side regenerations took 21 minutes on a loaded machine, so they are marked slow and CI deselects them: +The engine-side regenerations took 21 to 27 minutes on a heavily loaded machine, so they are marked slow and CI deselects them: ```bash OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_inputs.py tests/test_reference_adversary_impact.py