diff --git a/reference_audit/2026-10-05-reference-adversary/README.md b/reference_audit/2026-10-05-reference-adversary/README.md index 26e0b045..f5d4efe3 100644 --- a/reference_audit/2026-10-05-reference-adversary/README.md +++ b/reference_audit/2026-10-05-reference-adversary/README.md @@ -46,7 +46,55 @@ PolicyBench's diagnosis judge (`docs/audit.md`) explains why models miss. Its pr Inputs: -- Payload: `data.json.gz` of release dashboard-data-20260930, as committed at `8b4c0ca1` (#187), sha256 `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18`. Release dashboard-data-20261006 (#202) later rewrote the working tree's payload and annotations; the tests and the Reproduce steps read the pass's inputs from `8b4c0ca1`. +- The pass read the frozen run and its annotations as release dashboard-data-20260930 committed them at `8b4c0ca1` (#187). Later commits rewrote several of the working tree's copies: release dashboard-data-20261006 (#202) the run's payload, exclusion record and annotations, and #204 one of the reference system's convention modules. So every script in `scripts/` reads the inputs below only through `scripts/pass_inputs.py`. It writes `git show :` to scratch and stops (SystemExit) unless the bytes match the pinned sha256. Each script stages everything it reads before it computes or writes anything, and before it imports the engine. No script reads the working tree's copy. `` is `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace`. + + | Input | Commit | sha256 | Read by | + |---|---|---|---| + | `/data.json.gz`, the payload | `8b4c0ca1` | `1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18` | `consensus-flags`, `adversary-prepare`, `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/predictions.csv.gz` | `8b4c0ca1` | `ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12` | `leaderboard_impact.py` | + | `/reference_outputs.csv` | `8b4c0ca1` | `e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/reference_outputs.csv.meta.json` | `8b4c0ca1` | `816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b` | `definition_conformance.py`, `publication_sources.py`, `leaderboard_impact.py` | + | `/reference_exclusions.json` | `8b4c0ca1` | `bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2` | `publication_sources.py`, `engine_probe.py`, `build_proposals.py`, `leaderboard_impact.py` | + | `/scenarios.csv` | `8b4c0ca1` | `71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `leaderboard_impact.py` | + | `/scenarios.csv.meta.json` | `8b4c0ca1` | `03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb` | `leaderboard_impact.py` | + | `annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv` | `8b4c0ca1` | `6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4` | `adversary-prepare` | + | `policybench/benchmark_specs.json`, the output definitions | `8b4c0ca1` | `ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5` | `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` | + | `reference_audit/2026-10-05-payroll/program_classification.json`, #194's classification of state paid-leave programs | `049f4f09` | `282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a` | `definition_conformance.py` | + | `reference_audit/2026-10-05-reference-adversary/proposed_changes.json` | `4db91b5f` | `3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf` | `leaderboard_impact.py` | + | `reference_audit/2026-10-05-reference-adversary/verification/definition_conformance.json` | `4db91b5f` | `05e503f1b291c505b90182395c4afb88844dcb45fb2952c004ca09d4273894a0` | `build_proposals.py` | + +- `policybench` loads `benchmark_specs.json` itself, once per process: first as `policybench.scenarios` is imported, and every later lookup of an output's engine variable or aggregation uses that loaded copy. The three engine-side scripts therefore point the package at their staged copy (`pass_inputs.use_staged_specs`) before they import anything else from it, in the main process and in every worker. A checkout whose definitions have changed since the pass does not change what they compute. `leaderboard_impact.py` is the exception. It scores with `policybench analyze` in a child process and treats the definitions as part of the checkout's scoring code, which its published-reproduction check covers. +- The reference system is policyengine-us 2.15.17 plus `latest_final`, built from the convention modules below as `8b4c0ca1` held them. They are read by `definition_conformance.py`, `publication_sources.py` and `engine_probe.py`. The main process and every worker stage their own copy; workers are started with the spawn method, so none inherits the main process's system. `latest_final` loads `latest_conventions` (the nine `latest_c_*` modules) and `latest_md_local_output_scope`. #204 later rewrote `latest_alt_snap_mortgage_residence.py`, which `latest_final` does not load, but `verification/definition_conformance.json` records every module's sha256. + + | Module | Commit | sha256 | + |---|---|---| + | `reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json` | `8b4c0ca1` | `7cf93799aad6a8c3a6fc70e7df4acc6043202a1ecc083f2c3101f76b87a85eae` | + | `reference_audit/2026-09-28/fixes/alt_conventions_r25.py` | `8b4c0ca1` | `0f77fdd44e758581f7e563b0f7b70c4d4359a833b08a4ac45a614c380938775e` | + | `reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py` | `8b4c0ca1` | `3e71bab1db867988544e6f87f6dda4dc7c719d51320fbbb7558cd574a0374f6e` | + | `reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py` | `8b4c0ca1` | `d8a314093466caf5fe0a226f1060acba2e4364c4b87befbd80639ebf41cc4a2c` | + | `reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py` | `8b4c0ca1` | `00cd13886a8c3d03c4e84944d35f7e8a1448af26838e8bc0761b914b887c70cd` | + | `reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py` | `8b4c0ca1` | `5e6538a903112243ad85051ccba26b6133bdfbd5b9e5ba3dfa81ba690c061b32` | + | `reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py` | `8b4c0ca1` | `f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2` | + | `reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py` | `8b4c0ca1` | `39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5` | + | `reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py` | `8b4c0ca1` | `7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e` | + | `reference_audit/2026-09-28/fixes/latest_c_md_2026.py` | `8b4c0ca1` | `8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76` | + | `reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py` | `8b4c0ca1` | `787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e` | + | `reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py` | `8b4c0ca1` | `02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522` | + | `reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py` | `8b4c0ca1` | `0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916` | + | `reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py` | `8b4c0ca1` | `3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd` | + | `reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py` | `8b4c0ca1` | `4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce` | + | `reference_audit/2026-09-28/fixes/latest_conventions.py` | `8b4c0ca1` | `4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb` | + | `reference_audit/2026-09-28/fixes/latest_final.py` | `8b4c0ca1` | `dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e` | + | `reference_audit/2026-09-28/fixes/latest_map_stated_hours.py` | `8b4c0ca1` | `b210e780a31cfa0a3a0da2c49a8068ee14f00029b3ed39c00be67a1e917eda21` | + | `reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py` | `8b4c0ca1` | `53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd` | + | `reference_audit/2026-09-28/fixes/latest_state_conventions.py` | `8b4c0ca1` | `56e62e399d20ba8c8d9c5dc9bc7ea6681e74d2b529fa38561dbc8e8b2a90b442` | + +- Who checks the pins: + - `definition_conformance.py`, `publication_sources.py`, `engine_probe.py`, `build_proposals.py` and `leaderboard_impact.py` stage every input above that they read, and stop on any sha256 mismatch before they compute or write anything. The engine-side scripts also stop before they import the engine or start a worker. `pass_inputs.py` is the only file that holds a pin, and each script checks that the `pass_inputs` it imported is the one beside it. `publication_sources.py` has no mode that builds a report from a saved file. + - `tests/test_reference_adversary_inputs.py` checks that each commit holds its pinned bytes, that the committed evidence records the same hashes, and that this table lists each pin with the scripts that stage it. It also checks that any changed byte is refused, that each script refuses #202's run before it computes or writes, and that a wrong convention or definitions pin stops each script before the engine is imported. + - An audit hook (`tests/working_tree_fence.py`) fails any open of the working tree's copy of a pinned input, `benchmark_specs.json` included. It is armed while the scripts stage and read their inputs, and in every process, workers included, of the slow regenerations below. A copy of the package with changed definitions, put first on the path, does not change the engine variable the scripts' helpers look up. + - `tests/test_reference_adversary_impact.py` checks the impact script from a checkout whose working-tree run and proposals are junk. Its slow regeneration runs under the same hook in the script and in its scoring children, except that `policybench` may read its own `benchmark_specs.json` there. + - `consensus-flags` and `adversary-prepare` read the payload and annotations at the paths they are given, and record the payload's path and sha256 in their output. `tests/test_consensus.py` and `tests/test_reference_adversary.py` read the payload and the explanations from `8b4c0ca1`. - Top 5 by the payload's `modelStats` order: gpt-6-sol, claude-opus-5.5, gpt-5.6-sol, gpt-6-luna and claude-sonnet-5.5. - `consensus_flags.json` holds the default run, and `consensus_flags_prototype.json` reproduces the 2026-10-05 prototype. - `verification/flags_table.md` lists all 61 cells, each with its cluster, its exact-model count and whether the prototype flagged it. @@ -286,7 +334,7 @@ The root cause is a gap in scenario generation and definition scope, not an engi ## Leaderboard impact -`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. +`scripts/leaderboard_impact.py` copies the frozen run to scratch, applies each alternative in `proposed_changes.json`, and scores every copy with `policybench analyze`, the command the freeze runs. It takes the run and the proposals from git at their pinned commits (Inputs above), not from the working tree. The unchanged copy reproduces the published scoring: every `modelStats` field (except the cost and latency fields the freeze overlays), plus `programStats`, `heatmap`, `globalWeights` and `failureModes` exactly. Full tables are in `verification/leaderboard_impact.json` and `verification/leaderboard_impact_*_{models,programs}.csv`. | Variant | Scored cells | Exact-rate change across models | Leader (gpt-6-sol) exact | Exact-rank changes | |---|---:|---|---|---| @@ -441,13 +489,22 @@ The pass's own expectations are not part of either ruling (`status.not_ruled`). | `verification/leaderboard_impact*` | Leaderboard impact of each alternative | | `verification/pytest_*.txt` | Test runs: after the rebase, the rolling-pool change, and the final run (391 passed) | | `proposed_changes.json` | Exclusion and regeneration records per root cause, and the rulings | -| `scripts/` | The scripts that wrote the files above | +| `scripts/` | The scripts that wrote the files above, and `pass_inputs.py`, which pins and stages their inputs | -The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. +The code is in `policybench/consensus.py`, `reference_adversary.py`, `definition_conformance.py` and `publication_sources.py`, with CLI commands in `policybench/cli.py`. The runners are `scripts/run_reference_adversary_{claude,codex}.sh`, and `docs/audit.md` has the pipeline. The diagnosis judge's prompt still claims that earlier audits' bugs "were fixed before this run", which the frozen run's 28 unfixed engine-defect exclusions contradict. Removing it changes every judge prompt and breaks byte-identical carry-over of existing verdicts, so it waits for a versioned judge template, which a separate PR adds. Tests are in `tests/test_consensus.py`, `test_reference_adversary.py`, `test_reference_adversary_inputs.py`, `test_reference_adversary_impact.py`, `test_reference_adversary_runner.py`, `test_definition_conformance.py`, `test_publication_sources.py` and `test_audit.py`. ## Reproduce -`/data.json.gz` and `annotations//` must hold the pass's inputs as commit `8b4c0ca1` (release dashboard-data-20260930) committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. A worktree at `8b4c0ca1` has them, or `git show 8b4c0ca1:` each file. +The scripts in `scripts/` stage the pass's inputs from git (the tables under [Flag parameters](#flag-parameters)), so they run from any checkout that has `8b4c0ca1`, `049f4f09` and `4db91b5f` in its history, whatever its working tree's run, convention modules and records hold. The code they run is the checkout's. A shallow clone needs `git fetch --unshallow`. Each script's docstring gives its command. + +The CLI steps read the payload and annotations at the paths they are given and record the payload's path in their output (`consensus_flags.json` `source`). So they need the working tree's run and annotations as `8b4c0ca1` committed them, not today's, which release dashboard-data-20261006 (#202) rewrote. Restore both from `8b4c0ca1` first, and put today's back when done: + +```bash +run=paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace +git restore --source 8b4c0ca1 -- "$run" "annotations/${run##*/}" +# ... the CLI steps below ... +git restore -- "$run" "annotations/${run##*/}" +``` ```bash uv run policybench consensus-flags --payload /data.json.gz \ @@ -480,4 +537,34 @@ uv run python reference_audit/2026-10-05-reference-adversary/scripts/search_expo Preparing `runs/claude` today reproduces its `cases.jsonl` and derivations byte for byte. Each stage-1 prompt then differs from the committed one in one line, because the source rule now asks for `blocked_domains`. The re-judge's preparation reproduces `runs/claude-rejudge`'s inputs byte for byte. -The scripts in `scripts/` regenerate the verification files; each script's docstring gives its command. `definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment. +`definition_conformance.py`, `publication_sources.py`, `engine_probe.py` and `leaderboard_impact.py` need the policyengine-us 2.15.17 environment, which `uv sync --locked --extra dev --python 3.12` provides. Write their output to scratch and compare it with the committed files: + +```bash +a=reference_audit/2026-10-05-reference-adversary +export OPENBLAS_NUM_THREADS=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. +uv run python $a/scripts/definition_conformance.py --out-dir +uv run python $a/scripts/publication_sources.py --out-dir +uv run python $a/scripts/engine_probe.py scenario_043 \ + --variables co_sales_tax_refund co_refundable_credits co_eitc co_tabor_cash_back state_refundable_credits \ + --parameters gov.states.co.tax.income.credits.sales_tax_refund.amount.flat_amount_enabled \ + gov.states.co.tax.income.credits.sales_tax_refund.amount.amount \ + --out /co_sales_tax_refund_scenario_043.json +uv run python $a/scripts/leaderboard_impact.py --scratch --out-dir /impact +uv run python $a/scripts/build_proposals.py --out /proposed_changes.json +``` + +Each committed probe records its own arguments: `scenario_id`, `period`, the keys of `variables` and `parameters`, and `counterfactual.set`. `build_proposals.py` never writes the committed `proposed_changes.json`, which `leaderboard_impact.py` and the Haiku 5.5 release pin at `3a6e5920…`. It refuses that path for `--out`, and it creates `--out` exclusively, so it never writes any file that already exists (a hard link to the committed file, or another checkout's copy, included). + +The tests regenerate every one of these outputs into scratch and require the committed file byte for byte: + +- `build_proposals.py`: `proposed_changes.json`, in the fast suite. +- `definition_conformance.py`: both files, except two lines of `definition_conformance.json`: `run.seconds`, the wall time, and `run.script_sha256`, the script's own sha256. The committed file records `33ce4c3e…`, the script as #200 merged it; the test requires a regeneration to record the current script's hash. +- `publication_sources.py`: both files, except `meta.seconds`. +- `engine_probe.py`: all 8 probes, from the arguments each records. +- `leaderboard_impact.py`: all 23 `verification/leaderboard_impact*` files. + +The engine-side regenerations took 21 to 27 minutes on a heavily loaded machine, so they are marked slow and CI deselects them: + +```bash +OPENBLAS_NUM_THREADS=1 uv run pytest -m slow tests/test_reference_adversary_inputs.py tests/test_reference_adversary_impact.py +``` diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py index aa32e595..04032b21 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py @@ -1,4 +1,4 @@ -"""Write proposed_changes.json: the exclusion and regeneration records this pass proposes. +"""Build proposed_changes.json: the exclusion and regeneration records this pass proposes. Verification (verification/independent/*.md) found four engine defects behind four scored references and one definition-scope ambiguity behind four more. Each root cause gets an @@ -17,22 +17,47 @@ from verification/definition_conformance.json (the engine with the dependent as the head of their own tax unit). - python reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py +Every input is the pass's, staged from git by pass_inputs.py and checked against its pinned +sha256 before anything is built: the frozen run's references and exclusion record as release +dashboard-data-20260930 (8b4c0ca1) committed them, and verification/definition_conformance.json +as #200 merged it (4db91b5f). The working tree's run, which later releases rewrite (#202 +rewrote its exclusion record), is never read. + +The committed proposed_changes.json is pinned at sha256 PROPOSED_SHA256 below: +leaderboard_impact.py reads it at that hash, and the Haiku 5.5 release pins it too. So the +script never writes it, nor any file that already exists: it creates --out, which must be a +new path (a hard link to the committed file, or another checkout's copy, is an existing file +and is refused like any other). A regeneration is checked by comparing the two: + + python reference_audit/2026-10-05-reference-adversary/scripts/build_proposals.py \\ + --out /proposed_changes.json + cmp /proposed_changes.json \\ + reference_audit/2026-10-05-reference-adversary/proposed_changes.json """ from __future__ import annotations +import argparse import csv import json +import sys +import tempfile from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -OUT = HERE / "proposed_changes.json" +# The committed records, which the script never writes. +PROPOSED = HERE / "proposed_changes.json" +PROPOSED_SHA256 = pass_inputs.PROPOSALS_SHA256 +# The frozen run's files the script reads, staged from git. +RUN_FILES = ("reference_outputs.csv", "reference_exclusions.json") +CONFORMANCE_NAME = Path(pass_inputs.CONFORMANCE_PATH).name DRAFTED = "2026-10-06" ENGINE = "policyengine-us 2.15.17" STATE_TAX = "state_income_tax_before_refundable_credits" @@ -215,23 +240,36 @@ } -def frozen_references() -> dict[tuple[str, str], float]: - with (RUN / "reference_outputs.csv").open() as handle: +def stage_inputs(target: Path) -> Path: + """Stage the run's files under ``target/run`` and the conformance scan beside them, + each refused unless it matches its pin.""" + pass_inputs.stage_run(target / "run", RUN_FILES) + pass_inputs.git_input( + pass_inputs.RECORDS_COMMIT, + pass_inputs.CONFORMANCE_PATH, + pass_inputs.CONFORMANCE_SHA256, + target / CONFORMANCE_NAME, + ) + return target + + +def frozen_references(run: Path) -> dict[tuple[str, str], float]: + with (run / "reference_outputs.csv").open() as handle: return { (row["scenario_id"], row["variable"]): float(row["value"]) for row in csv.DictReader(handle) } -def excluded() -> set[tuple[str, str]]: - record = json.loads((RUN / "reference_exclusions.json").read_text()) +def excluded(run: Path) -> set[tuple[str, str]]: + record = json.loads((run / "reference_exclusions.json").read_text()) return {(e["scenario_id"], e["variable"]) for e in record["exclusions"]} -def household_scope_values() -> dict[tuple[str, str], float]: +def household_scope_values(inputs: Path) -> dict[tuple[str, str], float]: """Reference plus the dependent's own return, from the conformance scan.""" - scan = json.loads((HERE / "verification/definition_conformance.json").read_text()) - frozen = frozen_references() + scan = json.loads((inputs / CONFORMANCE_NAME).read_text()) + frozen = frozen_references(inputs / "run") values = {} for row in scan["household_scope_records"]: for output in (STATE_TAX, FED_TAX): @@ -240,10 +278,11 @@ def household_scope_values() -> dict[tuple[str, str], float]: return values -def main() -> None: - frozen = frozen_references() - already = excluded() - scope = household_scope_values() +def build(inputs: Path) -> dict: + """The records, from the inputs ``stage_inputs`` staged under ``inputs``.""" + frozen = frozen_references(inputs / "run") + already = excluded(inputs / "run") + scope = household_scope_values(inputs) out = { "schema_version": 1, "status": { @@ -342,8 +381,32 @@ def main() -> None: "exclusions": exclusions, "regenerations": regenerations, } - OUT.write_text(json.dumps(out, indent=2) + "\n") - print(f"wrote {OUT.relative_to(ROOT)}") + return out + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument( + "--out", + required=True, + help="where to write the records; never the committed proposed_changes.json", + ) + args = parser.parse_args() + out = Path(args.out) + if out.resolve() == PROPOSED.resolve(): + parser.error( + f"{PROPOSED.relative_to(ROOT)} is pinned at sha256 {PROPOSED_SHA256}; " + "write elsewhere and compare" + ) + if out.exists() or out.is_symlink(): + parser.error(f"{out} exists; --out must be a new file") + with tempfile.TemporaryDirectory(prefix="build_proposals_inputs_") as scratch: + records = build(stage_inputs(Path(scratch))) + out.parent.mkdir(parents=True, exist_ok=True) + # Exclusive creation: never writes through to a file that already exists. + with out.open("x") as handle: + handle.write(json.dumps(records, indent=2) + "\n") + print(f"wrote {out}") if __name__ == "__main__": diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py index 7a1e344c..e014796c 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/definition_conformance.py @@ -27,7 +27,19 @@ PolicyEngine/policybench#194 (commit 049f4f09, reference_audit/2026-10-05-payroll/program_classification.json), read from git. -Run from a policybench checkout with the policyengine-us 2.15.17 venv: +Every input is the pass's, staged from git by pass_inputs.py and checked against its +pinned sha256 before anything is computed and before the engine is imported: the +frozen run's payload, references, reference sidecar and scenarios and the output +definitions (benchmark_specs.json) as release dashboard-data-20260930 (8b4c0ca1) +committed them, #194's table, and latest_final with its parts as 8b4c0ca1 held them. +The main process and every worker (started with the spawn method, so none inherits +the main process's system) stage their own conventions and output definitions and +point policybench at the staged definitions. The working tree's run, which later +releases rewrite, is never read. The run record names each input by its repository +path. + +Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout +needs 8b4c0ca1 and 049f4f09 in its history): PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ /.venv-pe21517/bin/python \\ @@ -38,13 +50,14 @@ from __future__ import annotations import argparse +import atexit import copy import gzip import hashlib import importlib.util import json +import multiprocessing import shutil -import subprocess import sys import tempfile import time @@ -55,16 +68,20 @@ import numpy as np import pandas as pd +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + ROOT = Path(__file__).resolve().parents[3] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +# The frozen run's files main reads, staged from git. +RUN_FILES = ( + "data.json.gz", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "scenarios.csv", ) -PAYLOAD = RUN / "data.json.gz" -PAYLOAD_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" -AUDIT_0928 = ROOT / "reference_audit/2026-09-28" -AUDIT_0922 = ROOT / "reference_audit/2026-09-22" -SPECS = ROOT / "policybench/benchmark_specs.json" YEAR = 2026 PERIOD = str(YEAR) INSTANT = "2026-01-01" @@ -72,9 +89,10 @@ TOLERANCE = 0.005 # sweep.py's rename for the 1.755.4 harness; sweep_latest.py kept it on 2.15.17. RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} -LAW_COMMIT = "049f4f09" -LAW_PATH = "reference_audit/2026-10-05-payroll/program_classification.json" -LAW_SOURCE = f"PolicyEngine/policybench#194 @ {LAW_COMMIT}:{LAW_PATH}" +LAW_SOURCE = ( + f"PolicyEngine/policybench#194 @ {pass_inputs.LAW_COMMIT[:8]}:" + f"{pass_inputs.LAW_PATH}" +) # Gross income sources the engine counts as earned (irs_gross_income reads # irs_employment_income, itself max(0, employment_income - pre_tax_contributions)). EARNED_SOURCES = ( @@ -102,18 +120,43 @@ ) _SYSTEM = None +# This process's staged conventions (a directory) and output definitions (a file). +_FIX_DIR = None +_SPECS = None def _sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads.""" - target = Path(tempfile.mkdtemp(prefix="definition_conformance_fixes_")) - for path in (AUDIT_0928 / "fixes").glob("*.py"): - shutil.copy2(path, target / path.name) - shutil.copy2(AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json", target) +def stage_engine_inputs() -> Path: + """Stage this process's reference-system inputs, before the engine is imported. + + latest_final and its parts, plus the sales tax table the IRS module reads + (pass_inputs.FIXES_SHA256), go into a new directory, which is returned; the + output definitions go beside it and policybench is pointed at them. Each is + refused unless it matches its pin. The main process and every worker call this. + """ + global _FIX_DIR, _SPECS + scratch = Path(tempfile.mkdtemp(prefix="definition_conformance_engine_")) + atexit.register(shutil.rmtree, scratch, True) + fix_dir = pass_inputs.stage_fixes(scratch / "fixes") + specs = pass_inputs.stage_specs(scratch / Path(pass_inputs.SPECS_PATH).name) + pass_inputs.use_staged_specs(specs) + _FIX_DIR, _SPECS = fix_dir, specs + return fix_dir + + +def stage_inputs(target: Path) -> Path: + """Stage what main reads besides the reference system's inputs: the run's files + under ``run/`` and #194's law table, each refused unless it matches its pin.""" + pass_inputs.stage_run(target / "run", RUN_FILES) + pass_inputs.git_input( + pass_inputs.LAW_COMMIT, + pass_inputs.LAW_PATH, + pass_inputs.LAW_SHA256, + target / Path(pass_inputs.LAW_PATH).name, + ) return target @@ -129,12 +172,27 @@ def _load_reform(fix_dir: Path): def _system(): global _SYSTEM if _SYSTEM is None: + # Staged and checked before the engine is imported. + fix_dir = _FIX_DIR or stage_engine_inputs() from policyengine_us import CountryTaxBenefitSystem - _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(_assemble_fixes())) + _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(fix_dir)) return _SYSTEM +def worker_pool(workers: int) -> ProcessPoolExecutor: + """A pool whose workers each stage their own reference-system inputs. + + Spawned, never forked: a forked worker would inherit the main process's built + system and policybench state instead of staging its own. + """ + return ProcessPoolExecutor( + max_workers=workers, + mp_context=multiprocessing.get_context("spawn"), + initializer=stage_engine_inputs, + ) + + def build_situation(scenario) -> dict: situation = scenario.to_pe_household() for person in situation["people"].values(): @@ -347,16 +405,9 @@ def scenario_cells(job: dict) -> dict: } -def law_classifications() -> tuple[dict, dict]: - """Variable -> cited classification, from the payroll audit's program table.""" - try: - raw = subprocess.run( - ["git", "-C", str(ROOT), "show", f"{LAW_COMMIT}:{LAW_PATH}"], - check=True, - capture_output=True, - ).stdout - except (subprocess.CalledProcessError, FileNotFoundError) as error: - return {}, {"source": LAW_SOURCE, "available": False, "error": str(error)} +def law_classifications(path: Path) -> tuple[dict, dict]: + """Variable -> cited classification, from the payroll audit's staged program table.""" + raw = path.read_bytes() table = json.loads(raw) classifications = {} entries = list(table.get("programs", [])) + list( @@ -486,6 +537,15 @@ def main() -> None: parser.add_argument("--scenarios", nargs="*") args = parser.parse_args() + started = time.time() + # Every input is staged and checked before anything else: before policybench + # is imported (it reads its output definitions as it loads), before the + # engine is imported and before any worker starts. + staged = Path(tempfile.mkdtemp(prefix="definition_conformance_inputs_")) + atexit.register(shutil.rmtree, staged, True) + stage_inputs(staged) + stage_engine_inputs() + from policybench.definition_conformance import ( HOUSEHOLD_PROMPT_PHRASE, component_tree, @@ -494,21 +554,20 @@ def main() -> None: variable_graph_from_system, ) - started = time.time() out_dir = Path(args.out_dir) - if _sha256(PAYLOAD) != PAYLOAD_SHA256: - raise SystemExit(f"{PAYLOAD} does not match its pinned sha256") - payload = json.loads(gzip.decompress(PAYLOAD.read_bytes())) - meta = json.loads((RUN / "reference_outputs.csv.meta.json").read_text()) + run = staged / "run" + specs_path = _SPECS + payload = json.loads(gzip.decompress((run / "data.json.gz").read_bytes())) + meta = json.loads((run / "reference_outputs.csv.meta.json").read_text()) programs = list(meta["programs"]) specs = [ spec - for spec in json.loads(SPECS.read_text())["specs"]["policybench"]["countries"][ - "us" - ] + for spec in json.loads(specs_path.read_text())["specs"]["policybench"][ + "countries" + ]["us"] if spec["id"] in programs ] - reference = pd.read_csv(RUN / "reference_outputs.csv") + reference = pd.read_csv(run / "reference_outputs.csv") indexed = reference.set_index(["scenario_id", "variable"])["value"] scored = _scored_cells(payload) if args.scenarios: @@ -532,7 +591,7 @@ def main() -> None: graph = variable_graph_from_system( system, INSTANT, roots=[spec["pe_variable"] for spec in specs] ) - laws, law_meta = law_classifications() + laws, law_meta = law_classifications(staged / Path(pass_inputs.LAW_PATH).name) # A first, cell-free pass names the variables whose values measure a rule's # effect when that is not the component itself. @@ -583,7 +642,7 @@ def main() -> None: if graph[spec["pe_variable"]].entity == "tax_unit" } - scenarios = pd.read_csv(RUN / "scenarios.csv") + scenarios = pd.read_csv(run / "scenarios.csv") scored_by_scenario: dict[str, list[str]] = {} for scenario_id, output_id in scored: scored_by_scenario.setdefault(scenario_id, []).append(output_id) @@ -607,7 +666,7 @@ def main() -> None: f"{len(scored)} scored cells, {len(node_variables)} node variables", flush=True, ) - with ProcessPoolExecutor(max_workers=args.workers) as pool: + with worker_pool(args.workers) as pool: results = list(pool.map(scenario_cells, jobs)) results.sort(key=lambda result: result["scenario_id"]) by_scenario = {result["scenario_id"]: result for result in results} @@ -660,7 +719,6 @@ def main() -> None: ) ) script = Path(__file__).resolve() - fixes = sorted((AUDIT_0928 / "fixes").glob("*.py")) report["run"] = { "script": str(script.relative_to(ROOT)), "script_sha256": _sha256(script), @@ -670,15 +728,15 @@ def main() -> None: "reference_system": "policyengine-us 2.15.17 + latest_final " "(reference_audit/2026-09-28/fixes)", "fixes_sha256": { - str(path.relative_to(ROOT)): _sha256(path) - for path in fixes + [AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json"] + path: _sha256(_FIX_DIR / Path(path).name) + for path in pass_inputs.FIXES_SHA256 }, "instant": INSTANT, - "payload": str(PAYLOAD.relative_to(ROOT)), - "payload_sha256": PAYLOAD_SHA256, - "reference_outputs_sha256": _sha256(RUN / "reference_outputs.csv"), - "scenarios_sha256": _sha256(RUN / "scenarios.csv"), - "benchmark_specs_sha256": _sha256(SPECS), + "payload": f"{pass_inputs.RUN_PATH}/data.json.gz", + "payload_sha256": _sha256(run / "data.json.gz"), + "reference_outputs_sha256": _sha256(run / "reference_outputs.csv"), + "scenarios_sha256": _sha256(run / "scenarios.csv"), + "benchmark_specs_sha256": _sha256(specs_path), "household_prompt_phrase": HOUSEHOLD_PROMPT_PHRASE, "household_prompt_in_every_scenario": household_prompt, "reproduction": { diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py index da568751..da9d7789 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py @@ -14,6 +14,17 @@ It is the engine-evidence probe for the reference adversary's non-holding verdicts; it reads no verdict and changes no reference. +Every input is the pass's, staged from git by pass_inputs.py and checked against +its pinned sha256 before anything is computed and before the engine is imported: +the frozen run's scenarios, references and exclusion record and the output +definitions (benchmark_specs.json) as release dashboard-data-20260930 (8b4c0ca1) +committed them, and latest_final with its parts as 8b4c0ca1 held them. policybench +is pointed at the staged definitions. The working tree's run, which later +releases rewrite (#202 rewrote its exclusion record), is never read. Each committed probe records its own arguments +(``scenario_id``, ``period``, the ``variables`` and ``parameters`` keys and +``counterfactual.set``), and tests/test_reference_adversary_inputs.py regenerates +every probe from them. + OPENBLAS_NUM_THREADS=1 PYTHONPATH=. <2.15.17 venv>/bin/python \\ reference_audit/2026-10-05-reference-adversary/scripts/engine_probe.py \\ scenario_018 --variables az_standard_deduction az_taxable_income \\ @@ -24,10 +35,13 @@ from __future__ import annotations import argparse +import atexit import copy import importlib.util import json +import shutil import sys +import tempfile from importlib.metadata import version from pathlib import Path @@ -35,8 +49,22 @@ import pandas as pd HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +if Path(pass_inputs.__file__).resolve().parent != HERE: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + YEAR = 2026 INSTANT = "2026-01-01" +# The frozen run's files the probe reads, staged from git. +RUN_FILES = ("scenarios.csv", "reference_outputs.csv", "reference_exclusions.json") + + +def stage_inputs(target: Path) -> Path: + """Stage the run's files the probe reads under ``target``, each refused unless it + matches its pin.""" + return pass_inputs.stage_run(target, RUN_FILES) def _conformance(): @@ -164,8 +192,14 @@ def main() -> None: parser.add_argument("--out") args = parser.parse_args() + run = Path(tempfile.mkdtemp(prefix="engine_probe_inputs_")) + atexit.register(shutil.rmtree, run, True) + stage_inputs(run) conformance = _conformance() - scenarios = pd.read_csv(conformance.RUN / "scenarios.csv") + # The conventions and output definitions, before policybench or the engine + # is imported. + conformance.stage_engine_inputs() + scenarios = pd.read_csv(run / "scenarios.csv") rows = scenarios[scenarios["scenario_id"] == args.scenario_id] if rows.empty: raise SystemExit(f"no {args.scenario_id} in the frozen run") @@ -176,8 +210,8 @@ def main() -> None: system = conformance._system() sim = conformance._simulation(situation) - reference = pd.read_csv(conformance.RUN / "reference_outputs.csv") - exclusions = json.loads((conformance.RUN / "reference_exclusions.json").read_text()) + reference = pd.read_csv(run / "reference_outputs.csv") + exclusions = json.loads((run / "reference_exclusions.json").read_text()) excluded = { (record["scenario_id"], record["variable"]) for record in ( diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py index d728c5ee..3a47a0fb 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py @@ -1,9 +1,19 @@ """Score the frozen run as published and under each alternative in proposed_changes.json. -Copies the frozen run (predictions, references, scenarios, exclusions) to scratch -directories and never writes the snapshot. Each copy is scored with -``python -m policybench.cli analyze``, the command the freeze runs (the method of -PolicyEngine/policybench#191 and #194): +Reads the pass's inputs from git, never from the working tree: the frozen run +(payload, predictions, references, scenarios, exclusions) as commit PASS_COMMIT +(release dashboard-data-20260930, #187) holds it, and proposed_changes.json as +RECORDS_COMMIT (#200) holds it, both pinned in pass_inputs.py. Later releases +rewrite the working tree's run (dashboard-data-20261006, #202, rewrote its payload +and exclusions), and scoring those would regenerate different evidence into this +pass's verification files. Each file is staged under ``/pass_inputs`` and +must match its pinned sha256, or the script stops before scoring anything. The +scoring code is the checkout's; ``published`` below stops the script if that code +no longer scores the pinned run as the pinned payload records. + +Copies the staged run to scratch directories and never writes the snapshot. Each +copy is scored with ``python -m policybench.cli analyze``, the command the freeze +runs (the method of PolicyEngine/policybench#191 and #194): ``published`` Unchanged. It must reproduce the published payload's scoring: every modelStats field @@ -24,13 +34,17 @@ PYTHONPATH= /bin/python \\ reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py \\ --scratch + +The checkout needs PASS_COMMIT and RECORDS_COMMIT in its history (a shallow +clone needs ``git fetch --unshallow``). ``--out-dir`` writes the verification +files elsewhere; tests/test_reference_adversary_impact.py regenerates them that +way and requires the committed ones byte for byte. """ from __future__ import annotations import argparse import gzip -import hashlib import json import os import shutil @@ -40,14 +54,17 @@ import pandas as pd +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + HERE = Path(__file__).resolve().parents[1] ROOT = HERE.parents[1] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" -) -PROPOSALS = HERE / "proposed_changes.json" OUT_DIR = HERE / "verification" +PROPOSALS_NAME = Path(pass_inputs.PROPOSALS_PATH).name +# Copied into each variant and scored; data.json.gz is only compared against. RUN_FILES = ( "predictions.csv.gz", "reference_outputs.csv", @@ -70,17 +87,27 @@ RANKED = ("exact", "within1pct", "score") -def sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() +def stage_inputs(target: Path) -> Path: + """Stage the pass's run bundle under ``target/run`` and its proposals beside it.""" + if target.exists(): + shutil.rmtree(target) + pass_inputs.stage_run(target / "run") + pass_inputs.git_input( + pass_inputs.RECORDS_COMMIT, + pass_inputs.PROPOSALS_PATH, + pass_inputs.PROPOSALS_SHA256, + target / PROPOSALS_NAME, + ) + return target -def proposals() -> dict: - return json.loads(PROPOSALS.read_text())["root_causes"] +def proposals(path: Path) -> dict: + return json.loads(path.read_text())["root_causes"] -def variants() -> dict[str, dict[str, list[dict]]]: - """Each variant's exclusion and regeneration records.""" - causes = proposals() +def variants(path: Path) -> dict[str, dict[str, list[dict]]]: + """Each variant's exclusion and regeneration records, from ``path``'s proposals.""" + causes = proposals(path) plan: dict[str, dict[str, list[dict]]] = {} for cause, spec in causes.items(): plan[f"exclude:{cause}"] = {"exclude": spec["exclusions"], "regenerate": []} @@ -140,12 +167,13 @@ def always_zero(run_dir: Path) -> dict[str, float]: } -def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: +def stage(source: Path, target: Path, records: dict[str, list[dict]] | None) -> Path: + """Copy the staged run at ``source`` to ``target`` and apply ``records``.""" if target.exists(): shutil.rmtree(target) target.mkdir(parents=True) for name in RUN_FILES: - shutil.copy2(RUN / name, target / name) + shutil.copy2(source / name, target / name) if not records: return target if records["exclude"]: @@ -165,7 +193,7 @@ def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: raise SystemExit(f"{key}: frozen value differs from the reference CSV") reference.at[row, "value"] = repr(float(change["regenerated_value"])) reference.to_csv(target / "reference_outputs.csv", index=False) - before = (RUN / "reference_outputs.csv").read_text().splitlines() + before = (source / "reference_outputs.csv").read_text().splitlines() after = (target / "reference_outputs.csv").read_text().splitlines() changed = [i for i, (a, b) in enumerate(zip(before, after)) if a != b] if len(before) != len(after) or len(changed) != len(changes): @@ -174,15 +202,24 @@ def stage(target: Path, records: dict[str, list[dict]] | None) -> Path: ) meta_path = target / "reference_outputs.csv.meta.json" meta = json.loads(meta_path.read_text()) - meta["reference_csv_sha256"] = sha256(target / "reference_outputs.csv") + meta["reference_csv_sha256"] = pass_inputs.sha256( + target / "reference_outputs.csv" + ) meta_path.write_text(json.dumps(meta, indent=2) + "\n") return target +def analyze_env() -> dict[str, str]: + """The scoring child's environment: this checkout first on PYTHONPATH, with the + caller's own entries kept after it.""" + path = os.pathsep.join(filter(None, [str(ROOT), os.environ.get("PYTHONPATH")])) + return dict(os.environ, PYTHONPATH=path, PYTHONDONTWRITEBYTECODE="1") + + def analyze(run_dir: Path) -> dict: out = run_dir / "analysis" dashboard = run_dir / "dashboard.json" - env = dict(os.environ, PYTHONPATH=str(ROOT), PYTHONDONTWRITEBYTECODE="1") + env = analyze_env() subprocess.run( [ sys.executable, @@ -200,7 +237,7 @@ def analyze(run_dir: Path) -> dict: "--app-data-output", str(dashboard), "--reference-digest", - sha256(run_dir / "reference_outputs.csv"), + pass_inputs.sha256(run_dir / "reference_outputs.csv"), ], cwd=ROOT, env=env, @@ -211,8 +248,9 @@ def analyze(run_dir: Path) -> dict: return payload["countries"]["us"] -def check_reproduces_published(base: dict) -> None: - published = json.loads(gzip.decompress((RUN / "data.json.gz").read_bytes())) +def check_reproduces_published(base: dict, source: Path) -> None: + """Stop unless ``base`` scores as the staged ``source/data.json.gz`` does.""" + published = json.loads(gzip.decompress((source / "data.json.gz").read_bytes())) for key in EXACT_PAYLOAD_KEYS: if published[key] != base[key]: raise SystemExit(f"unchanged copy does not reproduce published {key}") @@ -331,27 +369,36 @@ def main() -> None: parser.add_argument( "--only", nargs="*", help="score only these variants (default: all)" ) + parser.add_argument( + "--out-dir", + default=str(OUT_DIR), + help="where to write the verification files (default: %(default)s)", + ) args = parser.parse_args() scratch = Path(args.scratch).resolve() - # stage() deletes and rewrites scratch/; never inside the repository. + out_dir = Path(args.out_dir) + # stage() and stage_inputs() delete and rewrite scratch/; never inside + # the repository. if scratch == ROOT.resolve() or scratch.is_relative_to(ROOT.resolve()): parser.error(f"--scratch must be outside the repository ({ROOT})") - plan = variants() + inputs = stage_inputs(scratch / "pass_inputs") + source = inputs / "run" + plan = variants(inputs / PROPOSALS_NAME) unknown = sorted(set(args.only or ()) - set(plan)) if unknown: parser.error(f"unknown variant(s) {unknown}; choose from {sorted(plan)}") - base_dir = stage(scratch / "published", None) + base_dir = stage(source, scratch / "published", None) base = analyze(base_dir) - check_reproduces_published(base) + check_reproduces_published(base, source) zero_published = always_zero(base_dir) - OUT_DIR.mkdir(parents=True, exist_ok=True) - summary: dict = {"published_reproduced": True, "proposals": PROPOSALS.name} + out_dir.mkdir(parents=True, exist_ok=True) + summary: dict = {"published_reproduced": True, "proposals": PROPOSALS_NAME} if args.only: plan = {name: plan[name] for name in args.only} for name, records in plan.items(): - run_dir = stage(scratch / name.replace(":", "__"), records) + run_dir = stage(source, scratch / name.replace(":", "__"), records) other = analyze(run_dir) models = compare(base, other) programs = program_rows(base, other) @@ -363,8 +410,8 @@ def main() -> None: } ) slug = name.replace(":", "_") - models.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_models.csv") - programs.to_csv(OUT_DIR / f"leaderboard_impact_{slug}_programs.csv") + models.to_csv(out_dir / f"leaderboard_impact_{slug}_models.csv") + programs.to_csv(out_dir / f"leaderboard_impact_{slug}_programs.csv") summary[name] = summarize(models, base, other, zero, cells) summary[name]["programs"] = programs.round(6).to_dict(orient="index") with pd.option_context("display.width", 250, "display.max_rows", 100): @@ -385,7 +432,7 @@ def main() -> None: ].round(4) ) print(programs.round(4)) - (OUT_DIR / "leaderboard_impact.json").write_text( + (out_dir / "leaderboard_impact.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(json.dumps({k: v for k, v in summary.items() if k != "proposals"}, indent=2)[:4000]) diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py new file mode 100644 index 00000000..356ad96e --- /dev/null +++ b/reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py @@ -0,0 +1,223 @@ +"""The reference adversary's inputs, staged from git and checked against pinned sha256s. + +The pass read the frozen run as commit PASS_COMMIT (release dashboard-data-20260930, +#187) committed it, and built its reference system (policyengine-us 2.15.17 plus +``latest_final``) from the convention modules as that commit held them. Later commits +rewrite the working tree's copies: release dashboard-data-20261006 (#202) rewrote the +run's payload and exclusion record, and #204 rewrote +``latest_alt_snap_mortgage_residence.py``. So every script in this directory reads +these inputs only through ``git_input``: ``git show :`` into scratch, +refused (SystemExit) unless the bytes match the pinned sha256, and read only from the +staged copy. A script stages everything it reads before it computes or writes +anything, and before it imports the engine. The code (this checkout's ``policybench`` +package and the installed engine) is not pinned here; each script's own reproduction +check covers it. + +``policybench`` loads its output definitions (``benchmark_specs.json``) itself, once +per process: first at import of ``policybench.scenarios``, and every spec lookup then +uses that copy. Those are an input, so ``use_staged_specs`` points the package at the +staged copy; the engine-side scripts call it in every process, workers included, +before they import anything else from ``policybench``. leaderboard_impact.py does not: it scores with ``policybench analyze`` +in a child process and treats the definitions as part of the scoring code its +published-reproduction check covers. + +The README's Inputs table lists every pin and the scripts that read it, and +tests/test_reference_adversary_inputs.py checks that the table and these pins agree. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from collections.abc import Iterable +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +# Release dashboard-data-20260930 (#187): the frozen run, the output definitions and +# the reference system's convention modules, as the pass read them. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +RUN_PATH = ( + "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +RUN_SHA256 = { + "data.json.gz": "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18", + "predictions.csv.gz": ( + "ca2c4c48c7fd3e680c9c61a7380ecfcb60ce95f913c5c363762e023949d8ad12" + ), + "reference_outputs.csv": ( + "e8bbba8fd3e90f78e7c0e83df06227bc1c94563e92f7405fe12be853a30b2466" + ), + "reference_outputs.csv.meta.json": ( + "816fef53c452d8520a321bc12bc29b28da1e7956a06818e5ec13d7fc7b371a4b" + ), + "reference_exclusions.json": ( + "bf4e6a249aeee01d0b71f5834ef7a35c4bab2266d2c59d0e81b12a0da44281c2" + ), + "scenarios.csv": "71b16212f0c0b3e5d13d8694ce57e362c23248665806c4d6dea7b23ef472858a", + "scenarios.csv.meta.json": ( + "03a66e90b86e9bd0cc77f27520784bd581777762f749675dc716e24c1b8eaebb" + ), +} +SPECS_PATH = "policybench/benchmark_specs.json" +SPECS_SHA256 = "ce233f8cbb0549b33469b5929fc6df3cd5d064c727529afafe26ff2da7970cc5" +# latest_final and every module beside it, plus the sales tax table +# latest_c_irs_sales_tax_2025 reads; staged flat into one directory, as the +# reference builder copied them. +FIXES_SHA256 = { + "reference_audit/2026-09-22/fixes/r19_irs_sales_tax_2025.json": ( + "7cf93799aad6a8c3a6fc70e7df4acc6043202a1ecc083f2c3101f76b87a85eae" + ), + "reference_audit/2026-09-28/fixes/alt_conventions_r25.py": ( + "0f77fdd44e758581f7e563b0f7b70c4d4359a833b08a4ac45a614c380938775e" + ), + "reference_audit/2026-09-28/fixes/latest_alt_r02_ira_219g.py": ( + "3e71bab1db867988544e6f87f6dda4dc7c719d51320fbbb7558cd574a0374f6e" + ), + "reference_audit/2026-09-28/fixes/latest_alt_salt_refund_no_prior_benefit.py": ( + "d8a314093466caf5fe0a226f1060acba2e4364c4b87befbd80639ebf41cc4a2c" + ), + "reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py": ( + "00cd13886a8c3d03c4e84944d35f7e8a1448af26838e8bc0761b914b887c70cd" + ), + "reference_audit/2026-09-28/fixes/latest_alt_unlisted_hours_40.py": ( + "5e6538a903112243ad85051ccba26b6133bdfbd5b9e5ba3dfa81ba690c061b32" + ), + "reference_audit/2026-09-28/fixes/latest_c_ca_hold_2025.py": ( + "f19a47b9583032a0602a31e52f995eee824a427a76e87a9d4257d6682a016fa2" + ), + "reference_audit/2026-09-28/fixes/latest_c_id_hold_2025.py": ( + "39fb99292691600cd273a1c6ef3c90847453e21e4d7b872603bcaac71727f8a5" + ), + "reference_audit/2026-09-28/fixes/latest_c_irs_sales_tax_2025.py": ( + "7f7fd235b29c45d1d5cceded0c3391aceb38740518dfda770e3aebcc3003228e" + ), + "reference_audit/2026-09-28/fixes/latest_c_md_2026.py": ( + "8111c77ffd2e43d434d083994a7e8506ba42461a0845b33ac47ce0df988c2c76" + ), + "reference_audit/2026-09-28/fixes/latest_c_mi_published_2026.py": ( + "787cf837422e0bdf331a8e01a6fa8e8f2e3b78c35c248a4da83f3b2dbfa9ef9e" + ), + "reference_audit/2026-09-28/fixes/latest_c_mn_published_2026.py": ( + "02a8ccc1fce71485fc9411d4a55f33226de4c8de09cf6375f99866b79da70522" + ), + "reference_audit/2026-09-28/fixes/latest_c_mo_published_2026.py": ( + "0474524c83a7eb0c8440abf06bbeee38d4394e08260464d0dacb195ed3a8d916" + ), + "reference_audit/2026-09-28/fixes/latest_c_snap_hold_fy2026.py": ( + "3a8709ef36582af2248e460136db618fd319d490b7aeed94adbd6143ad9c07dd" + ), + "reference_audit/2026-09-28/fixes/latest_c_wi_published_2026.py": ( + "4edf4c714c05d300e6fa32c4b56879aad8aa0de1d9ddd9a77987eefe3b06acce" + ), + "reference_audit/2026-09-28/fixes/latest_conventions.py": ( + "4155ed4a0be72c907815b33e2b3603cf0b5ea3eca351ad7509f8bd9ea5bb8ddb" + ), + "reference_audit/2026-09-28/fixes/latest_final.py": ( + "dbbdd228b99933c6f336af97970821c25c863a18606285e8d6d9ffb3bc9af78e" + ), + "reference_audit/2026-09-28/fixes/latest_map_stated_hours.py": ( + "b210e780a31cfa0a3a0da2c49a8068ee14f00029b3ed39c00be67a1e917eda21" + ), + "reference_audit/2026-09-28/fixes/latest_md_local_output_scope.py": ( + "53a6de3cd9b149467e5d86a5bb1fcc258e859478a7c16b7d3df951c713e423dd" + ), + "reference_audit/2026-09-28/fixes/latest_state_conventions.py": ( + "56e62e399d20ba8c8d9c5dc9bc7ea6681e74d2b529fa38561dbc8e8b2a90b442" + ), +} + +# PolicyEngine/policybench#194's cited classification of state paid-leave programs. +LAW_COMMIT = "049f4f091397e9c3ec2d9b21f3acbeb9c07c9764" +LAW_PATH = "reference_audit/2026-10-05-payroll/program_classification.json" +LAW_SHA256 = "282cbf25ce9110a3c5c51873b7ca0e89f1f5ed38592b0c5b51d393f08000089a" + +# The pass's own records as #200 merged them. +RECORDS_COMMIT = "4db91b5f10581947f60a07f899e0ec2867b0405e" +PROPOSALS_PATH = "reference_audit/2026-10-05-reference-adversary/proposed_changes.json" +PROPOSALS_SHA256 = "3a6e5920a2d02e94e1df52c95c1def2739f3eb21b1fb67fc74a7d68219a749bf" +CONFORMANCE_PATH = ( + "reference_audit/2026-10-05-reference-adversary/verification/" + "definition_conformance.json" +) +CONFORMANCE_SHA256 = "05e503f1b291c505b90182395c4afb88844dcb45fb2952c004ca09d4273894a0" + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def git_bytes(commit: str, path: str, pinned: str) -> bytes: + """``path`` as ``commit`` holds it, refusing any bytes but the pinned ones.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + ) + if result.returncode: + raise SystemExit( + f"cannot read {path} at {commit[:12]}; fetch full history " + f"(git fetch --unshallow): {result.stderr.decode().strip()}" + ) + digest = hashlib.sha256(result.stdout).hexdigest() + if digest != pinned: + raise SystemExit( + f"{commit[:12]}:{path} has sha256 {digest}, not the pinned {pinned}" + ) + return result.stdout + + +def git_input(commit: str, path: str, pinned: str, target: Path) -> Path: + """Write ``path`` as ``commit`` holds it to ``target``, refusing any other bytes.""" + data = git_bytes(commit, path, pinned) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + return target + + +def stage_run(target: Path, names: Iterable[str] | None = None) -> Path: + """The frozen run's files ``names`` (default: all) as PASS_COMMIT holds them, in an + empty ``target``.""" + if target.exists(): + shutil.rmtree(target) + for name in RUN_SHA256 if names is None else names: + git_input(PASS_COMMIT, f"{RUN_PATH}/{name}", RUN_SHA256[name], target / name) + return target + + +def stage_fixes(target: Path) -> Path: + """``latest_final``, its parts and the sales tax table, flat in an empty ``target``.""" + if target.exists(): + shutil.rmtree(target) + for path, pinned in FIXES_SHA256.items(): + git_input(PASS_COMMIT, path, pinned, target / Path(path).name) + return target + + +def stage_specs(target: Path) -> Path: + """The output definitions as PASS_COMMIT holds them, written to ``target``.""" + return git_input(PASS_COMMIT, SPECS_PATH, SPECS_SHA256, target) + + +def use_staged_specs(path: Path) -> None: + """Make ``policybench`` read its output definitions from the staged ``path``. + + For this process only; call it before importing anything else from + ``policybench``, because ``policybench.scenarios`` derives module constants from + the definitions as it is imported. If the package has already loaded its + definitions (a long-lived interpreter, such as a test run), they must equal the + staged ones, or the script stops: constants derived from other definitions + cannot be recalled. + """ + from policybench import spec + + staged = json.loads(path.read_text(encoding="utf-8")) + if spec._raw_spec_data.cache_info().currsize and spec._raw_spec_data() != staged: + raise SystemExit( + "policybench already loaded output definitions that are not the pinned " + f"{SPECS_PATH}; run this script in a fresh interpreter" + ) + spec._spec_path = lambda: path + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() diff --git a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py index fec7d92d..289c5f42 100644 --- a/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py +++ b/reference_audit/2026-10-05-reference-adversary/scripts/publication_sources.py @@ -58,7 +58,20 @@ recorded per-output simulations, and refuses to report if any scored reference does not reproduce. -Run from a policybench checkout with the policyengine-us 2.15.17 venv: +Every input is the pass's, staged from git by pass_inputs.py and checked against its +pinned sha256 before anything is computed and before the engine is imported: the +frozen run's payload, references, reference sidecar, exclusion record and scenarios +and the output definitions (benchmark_specs.json) as release dashboard-data-20260930 +(8b4c0ca1) committed them, and latest_final with its parts as 8b4c0ca1 held them. The +main process and every worker (started with the spawn method) stage their own +conventions and output definitions and point policybench at the staged definitions. +The working tree's run, which later releases rewrite (#202 rewrote its payload and +exclusion record), is never read. The report names each input by its repository +path. The script always runs the engine: it has no mode that builds a report from a +saved file. + +Run from a policybench checkout with the policyengine-us 2.15.17 venv (the checkout +needs 8b4c0ca1 in its history): PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ /.venv-pe21517/bin/python \\ @@ -74,6 +87,7 @@ import importlib.util import json import math +import multiprocessing import os import shutil import sys @@ -87,18 +101,24 @@ import numpy as np import pandas as pd -ROOT = Path(__file__).resolve().parents[3] -RUN = ( - ROOT - / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pass_inputs # noqa: E402 (the pass's pinned inputs, beside this script) + +if Path(pass_inputs.__file__).resolve().parent != Path(__file__).resolve().parent: + raise SystemExit(f"pass_inputs is {pass_inputs.__file__}, not this script's") + +# The frozen run's files main reads, staged from git. +RUN_FILES = ( + "data.json.gz", + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", ) -AUDIT_0928 = ROOT / "reference_audit/2026-09-28" -AUDIT_0922 = ROOT / "reference_audit/2026-09-22" YEAR = 2026 TOLERANCE = 1e-3 BINARY_SUFFIXES = ("_eligible",) RENAME = {"partnership_se_income": "partnership_self_employment_net_earnings"} -PAYLOAD_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" SCORED_CELLS = 1928 SKIP_PREFIXES = ("gov.abolitions.",) SCALE_COMPONENTS = ("threshold", "rate", "amount", "average_rate", "base") @@ -124,14 +144,20 @@ def _jsonable(value): return repr(value) -def _assemble_fixes() -> Path: - """latest_final and its parts, plus the sales tax table the IRS module reads.""" - target = Path(tempfile.mkdtemp(prefix="publication_sources_fixes_")) - atexit.register(shutil.rmtree, target, True) - for path in (AUDIT_0928 / "fixes").glob("*.py"): - shutil.copy2(path, target / path.name) - shutil.copy2(AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json", target) - return target +def stage_engine_inputs() -> Path: + """Stage this process's reference-system inputs, before the engine is imported. + + latest_final and its parts, plus the sales tax table the IRS module reads + (pass_inputs.FIXES_SHA256), go into a new directory, which is returned; the + output definitions go beside it and policybench is pointed at them. Each is + refused unless it matches its pin. The main process and every worker call this. + """ + scratch = Path(tempfile.mkdtemp(prefix="publication_sources_engine_")) + atexit.register(shutil.rmtree, scratch, True) + fix_dir = pass_inputs.stage_fixes(scratch / "fixes") + specs = pass_inputs.stage_specs(scratch / Path(pass_inputs.SPECS_PATH).name) + pass_inputs.use_staged_specs(specs) + return fix_dir def _load_reform(fix_dir: Path): @@ -324,10 +350,25 @@ def scale_at_instant(self, instant): def _init_worker() -> None: global _SYSTEM + # Staged and checked before the engine is imported. + fix_dir = stage_engine_inputs() install_recording() from policyengine_us import CountryTaxBenefitSystem - _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(_assemble_fixes())) + _SYSTEM = CountryTaxBenefitSystem(reform=_load_reform(fix_dir)) + + +def worker_pool(workers: int) -> ProcessPoolExecutor: + """A pool whose workers each stage their own reference-system inputs. + + Spawned, never forked: a forked worker would inherit the main process's + policybench and build-hook state. + """ + return ProcessPoolExecutor( + max_workers=workers, + mp_context=multiprocessing.get_context("spawn"), + initializer=_init_worker, + ) def _output_value(sim, scenario, variable: str) -> float: @@ -736,13 +777,19 @@ def _parameter_fact(ref: Tree, plain: Tree, name: str, instant: str) -> dict: # --------------------------------------------------------------------------- -def _scored_cells_from_payload() -> tuple[set, str]: +def stage_inputs(target: Path) -> Path: + """Stage the run's files main reads under ``target``, each refused unless it + matches its pin.""" + return pass_inputs.stage_run(target, RUN_FILES) + + +def _scored_cells_from_payload(path: Path) -> tuple[set, str]: from policybench.consensus import file_sha256, load_us_payload - path = RUN / "data.json.gz" sha = file_sha256(path) - if sha != PAYLOAD_SHA256: - raise SystemExit(f"{path}: sha256 {sha} is not the frozen {PAYLOAD_SHA256}") + pinned = pass_inputs.RUN_SHA256["data.json.gz"] + if sha != pinned: + raise SystemExit(f"{path}: sha256 {sha} is not the frozen {pinned}") payload = load_us_payload(path) scored = set() for sid, outputs in payload["scenarioPredictions"].items(): @@ -761,27 +808,16 @@ def main() -> None: parser.add_argument("--scenarios", nargs="*") parser.add_argument("--workers", type=int, default=6) parser.add_argument("--facts-out", help="also write the unclassified facts here") - parser.add_argument( - "--from-facts", - help="skip the engine: rebuild the report from a --facts-out file", - ) args = parser.parse_args() - if args.from_facts: - from policybench.publication_sources import build_report, dumps_report - - saved = json.loads(Path(args.from_facts).read_text()) - report, markdown = build_report( - saved["facts"], saved["other_instants"], saved["meta"] - ) - out_dir = Path(args.out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "publication_sources.json").write_text( - dumps_report(report) - ) - (out_dir / "publication_sources.md").write_text(markdown) - return started = time.time() + # Every input is staged and checked before anything else: before policybench + # is imported (it reads its output definitions as it loads), before the + # engine is imported or hooked and before any worker starts. + run = Path(tempfile.mkdtemp(prefix="publication_sources_inputs_")) + atexit.register(shutil.rmtree, run, True) + stage_inputs(run) + fix_dir = stage_engine_inputs().resolve() install_build_hooks() BUILD["label"] = "plain" @@ -789,24 +825,24 @@ def main() -> None: from policybench.spec import expand_programs_for_scenario out_dir = Path(args.out_dir) - meta = json.loads((RUN / "reference_outputs.csv.meta.json").read_text()) + meta = json.loads((run / "reference_outputs.csv.meta.json").read_text()) programs = meta["programs"] if isinstance(programs, str): programs = ast.literal_eval(programs) - reference = pd.read_csv(RUN / "reference_outputs.csv") + reference = pd.read_csv(run / "reference_outputs.csv") indexed = reference.set_index(["scenario_id", "variable"])["value"] - exclusions = json.loads((RUN / "reference_exclusions.json").read_text())[ + exclusions = json.loads((run / "reference_exclusions.json").read_text())[ "exclusions" ] excluded = {(e["scenario_id"], e["variable"]) for e in exclusions} - scored, payload_sha = _scored_cells_from_payload() + scored, payload_sha = _scored_cells_from_payload(run / "data.json.gz") by_exclusion = {key for key in indexed.index if key not in excluded} if scored != by_exclusion or len(scored) != SCORED_CELLS: raise SystemExit( f"scored cells: payload {len(scored)}, reference rows minus exclusions " f"{len(by_exclusion)}, expected {SCORED_CELLS}" ) - scenarios = pd.read_csv(RUN / "scenarios.csv") + scenarios = pd.read_csv(run / "scenarios.csv") if args.scenarios: scenarios = scenarios[scenarios["scenario_id"].isin(args.scenarios)] @@ -822,9 +858,7 @@ def main() -> None: (row["scenario_json"], variables, position % REVERSE_CHECK_EVERY == 0) ) - with ProcessPoolExecutor( - max_workers=min(args.workers, len(jobs)), initializer=_init_worker - ) as pool: + with worker_pool(min(args.workers, len(jobs))) as pool: futures = [pool.submit(record_scenario, job) for job in jobs] # Build both systems here while the workers run. @@ -834,7 +868,7 @@ def main() -> None: plain_system = policyengine_us.system.system BUILD["label"] = "reference" - BUILD["fix_dir"] = _assemble_fixes().resolve() + BUILD["fix_dir"] = fix_dir reference_system = CountryTaxBenefitSystem( reform=_load_reform(BUILD["fix_dir"]) ) @@ -988,9 +1022,9 @@ def main() -> None: "policyengine_core": version("policyengine-core"), "reference_system": "policyengine-us 2.15.17 + latest_final " "(reference_audit/2026-09-28/fixes)", - "payload": str((RUN / "data.json.gz").relative_to(ROOT)), + "payload": f"{pass_inputs.RUN_PATH}/data.json.gz", "payload_sha256": payload_sha, - "reference_outputs": str((RUN / "reference_outputs.csv").relative_to(ROOT)), + "reference_outputs": f"{pass_inputs.RUN_PATH}/reference_outputs.csv", "scenarios": int(len(results)), "outputs_recomputed": checked, "scored_cells_recomputed": n_scored, diff --git a/tests/test_reference_adversary_impact.py b/tests/test_reference_adversary_impact.py new file mode 100644 index 00000000..87d09d11 --- /dev/null +++ b/tests/test_reference_adversary_impact.py @@ -0,0 +1,300 @@ +"""The reference adversary's leaderboard impact comes from the pass's pinned inputs. + +``reference_audit/2026-10-05-reference-adversary/scripts/leaderboard_impact.py`` +reads the frozen run as PASS_COMMIT (release dashboard-data-20260930, #187) +committed it and the proposals as #200 merged them, each checked against its +sha256 by the shared ``scripts/pass_inputs.py``. It never reads the working +tree's run, which later releases rewrite (#202 did). The pins themselves, and the +refusal of any changed byte, are tested in tests/test_reference_adversary_inputs.py. +CI checks out full history, so git holds every pinned input. +""" + +from __future__ import annotations + +import difflib +import gzip +import importlib.util +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" +SCRIPT = AUDIT / "scripts/leaderboard_impact.py" +VERIFICATION = AUDIT / "verification" +sys.path.insert(0, str(SCRIPT.parent)) +import pass_inputs as pins # noqa: E402 + +from tests import working_tree_fence as fence # noqa: E402 + +# The working tree's copy of every input the script stages. The script scores with +# the checkout's policybench, which may read its own output definitions. +FENCE = { + "forbidden": [ + str(ROOT / path) + for path in (pins.RUN_PATH, pins.PROPOSALS_PATH, pins.SPECS_PATH) + ], + "exempt": {str(ROOT / pins.SPECS_PATH): str(ROOT / "policybench")}, + "root": str(ROOT), +} + + +def _fenced_env(directory: Path) -> dict[str, str]: + """A subprocess environment whose every interpreter is fenced.""" + return dict( + os.environ, + PYTHONPATH=f"{fence.sitecustomize(directory)}{os.pathsep}{ROOT}", + **{fence.ENV: json.dumps(FENCE)}, + ) + + +# The pins tests/test_consensus.py and tests/test_reference_adversary.py hold. +PASS_COMMIT = "8b4c0ca146bb6f66deba6ce24009d49d70d92df2" +FROZEN_SHA256 = "1e029aaa87d1dfbd2ceee88419599a919dd7c9d4aba78a308ec48d008d54ae18" +# Release dashboard-data-20261006 (#202) rewrote the run's payload and exclusions. +RELEASE_20261006_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = ( + "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d" +) +COMMITTED = sorted(path.name for path in VERIFICATION.glob("leaderboard_impact*")) + + +def _load_script(): + spec = importlib.util.spec_from_file_location("adversary_impact", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +impact = _load_script() + + +def _tampered_checkout(path: Path) -> Path: + """A checkout sharing this repository's objects whose working tree holds junk. + + Its run directory and proposals are not the pass's inputs, so a script that + read them instead of git would fail or regenerate different evidence. + """ + subprocess.run( + ["git", "clone", "--quiet", "--shared", "--no-checkout", str(ROOT), str(path)], + check=True, + ) + for name in pins.RUN_SHA256: + junk = path / pins.RUN_PATH / name + junk.parent.mkdir(parents=True, exist_ok=True) + junk.write_bytes(b"not the pass's input\n") + proposals = path / pins.PROPOSALS_PATH + proposals.parent.mkdir(parents=True, exist_ok=True) + proposals.write_text('{"root_causes": {}}\n') + return path + + +def _argv(monkeypatch, tmp_path: Path) -> Path: + out = tmp_path / "out" + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + ) + return out + + +def test_the_pins_cover_every_file_the_script_reads(tmp_path): + assert pins.PASS_COMMIT == PASS_COMMIT + assert pins.RUN_SHA256["data.json.gz"] == FROZEN_SHA256 + assert set(pins.RUN_SHA256) == {*impact.RUN_FILES, "data.json.gz"} + + inputs = impact.stage_inputs(tmp_path / "pass_inputs") + staged = sorted( + path.relative_to(inputs).as_posix() + for path in inputs.rglob("*") + if path.is_file() + ) + assert staged == sorted( + [f"run/{name}" for name in pins.RUN_SHA256] + [impact.PROPOSALS_NAME] + ) + for name, pinned in pins.RUN_SHA256.items(): + assert pins.sha256(inputs / "run" / name) == pinned + assert pins.sha256(inputs / impact.PROPOSALS_NAME) == pins.PROPOSALS_SHA256 + # The README's frozen run: 46 models, 1,928 scored cells. + payload = json.loads(gzip.decompress((inputs / "run/data.json.gz").read_bytes())) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + (inputs / "stray").write_text("left over\n") + impact.stage_inputs(inputs) + assert not (inputs / "stray").exists() + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ( + "PASS_COMMIT", + RELEASE_20261006_COMMIT, + f"data.json.gz has sha256 {RELEASE_20261006_SHA256}, " + f"not the pinned {FROZEN_SHA256}", + ), + ("PROPOSALS_SHA256", "0" * 64, f"not the pinned {'0' * 64}"), + ("PASS_COMMIT", "0" * 40, "cannot read"), + ], +) +def test_staged_inputs_refuse_anything_but_the_pinned_bytes( + tmp_path, monkeypatch, attribute, value, message +): + monkeypatch.setattr(pins, attribute, value) + with pytest.raises(SystemExit, match=re.escape(message)): + impact.stage_inputs(tmp_path / "pass_inputs") + + +def test_a_refused_input_stops_the_script_before_it_scores(tmp_path, monkeypatch): + def analyze(run_dir): + raise AssertionError(f"scored {run_dir} after its inputs were refused") + + monkeypatch.setattr(pins, "PASS_COMMIT", RELEASE_20261006_COMMIT) + monkeypatch.setattr(impact, "analyze", analyze) + out = _argv(monkeypatch, tmp_path) + with pytest.raises(SystemExit, match=f"not the pinned {FROZEN_SHA256}"): + impact.main() + assert not out.exists() + + +def test_the_script_scores_only_the_staged_inputs(tmp_path, monkeypatch): + """Run from a checkout whose working-tree run is junk, every variant still + starts from the pinned bytes and the full set of evidence files is written.""" + checkout = _tampered_checkout(tmp_path / "checkout") + monkeypatch.setattr(pins, "ROOT", checkout) + monkeypatch.setattr(impact, "ROOT", checkout) + scored: dict[str, dict[str, str]] = {} + published: dict = {} + + def analyze(run_dir: Path) -> dict: + scored[run_dir.name] = { + name: pins.sha256(run_dir / name) for name in impact.RUN_FILES + } + if not published: + payload = run_dir.parent / "pass_inputs/run/data.json.gz" + published.update(json.loads(gzip.decompress(payload.read_bytes()))) + return published + + monkeypatch.setattr(impact, "analyze", analyze) + monkeypatch.setattr( + impact, "always_zero", lambda run_dir: {"exact": 0.0, "within1pct": 0.0} + ) + out = _argv(monkeypatch, tmp_path) + impact.main() + + pinned = {name: pins.RUN_SHA256[name] for name in impact.RUN_FILES} + assert scored.pop("published") == pinned + exclusions = {"reference_exclusions.json"} + regenerations = {"reference_outputs.csv", "reference_outputs.csv.meta.json"} + expected = { + "recommended": exclusions | regenerations, + "exclude_all": exclusions, + } + assert set(scored) >= set(expected) + for variant, digests in scored.items(): + changed = {name for name in pinned if digests[name] != pinned[name]} + if variant.startswith("exclude__"): + assert changed == exclusions, variant + elif variant.startswith("regenerate__"): + assert changed == regenerations, variant + else: + assert changed == expected[variant], variant + assert sorted(path.name for path in out.iterdir()) == COMMITTED + summary = json.loads((out / "leaderboard_impact.json").read_text()) + assert summary["published_reproduced"] is True + assert summary["proposals"] == "proposed_changes.json" + + +READ = "import sys; print(len(open(sys.argv[1], 'rb').read()))" +LOAD = "from policybench import spec; print(len(spec._raw_spec_data()))" + + +def test_the_scoring_child_keeps_the_callers_path_and_so_the_fence( + tmp_path, monkeypatch +): + """analyze() used to replace PYTHONPATH, which dropped a caller's sitecustomize + from the scoring child. Under the fenced environment the slow test uses, a + child started as analyze() starts it cannot open the working tree's run, while + policybench can still load its own output definitions.""" + for name, value in _fenced_env(tmp_path / "fence").items(): + monkeypatch.setenv(name, value) + env = impact.analyze_env() + assert env["PYTHONPATH"].split(os.pathsep)[0] == str(impact.ROOT) + assert str(tmp_path / "fence") in env["PYTHONPATH"].split(os.pathsep) + + def child(code: str, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", code, *args], env=env, capture_output=True, text=True + ) + + for name in ("scenarios.csv", "reference_exclusions.json"): + refused = child(READ, str(ROOT / pins.RUN_PATH / name)) + assert refused.returncode != 0 + assert "opened the working tree's" in refused.stderr + assert child(READ, str(ROOT / pins.SPECS_PATH)).returncode != 0 + loaded = child(LOAD) + assert loaded.returncode == 0, loaded.stderr + # Unfenced, the same child reads the file: the refusals above were the fence's. + monkeypatch.delenv(fence.ENV) + env = impact.analyze_env() + assert child(READ, str(ROOT / pins.RUN_PATH / "scenarios.csv")).returncode == 0 + + +@pytest.mark.slow +def test_regeneration_reproduces_the_committed_evidence(tmp_path): + """Score every variant for real and require the committed files byte for byte. + + The script and its scoring children run fenced: none may open the working + tree's run or proposals. About five minutes of ``policybench analyze`` runs, + so CI deselects it. Run it with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_adversary_impact.py + """ + assert len(COMMITTED) == 23 + out = tmp_path / "out" + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--scratch", + str(tmp_path / "scratch"), + "--out-dir", + str(out), + ], + cwd=ROOT, + env=_fenced_env(tmp_path / "fence"), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + assert sorted(path.name for path in out.iterdir()) == COMMITTED + differing = [ + name + for name in COMMITTED + if (out / name).read_bytes() != (VERIFICATION / name).read_bytes() + ] + if differing: + first = differing[0] + diff = difflib.unified_diff( + (VERIFICATION / first).read_text().splitlines(), + (out / first).read_text().splitlines(), + f"committed/{first}", + f"regenerated/{first}", + lineterm="", + ) + pytest.fail( + f"{len(differing)} files differ: {differing}\n" + "\n".join(list(diff)[:60]) + ) diff --git a/tests/test_reference_adversary_inputs.py b/tests/test_reference_adversary_inputs.py new file mode 100644 index 00000000..02dce9d8 --- /dev/null +++ b/tests/test_reference_adversary_inputs.py @@ -0,0 +1,1086 @@ +"""The reference adversary's scripts read only the pass's pinned inputs. + +``reference_audit/2026-10-05-reference-adversary/scripts/pass_inputs.py`` pins +every input the pass read, by commit and sha256: the frozen run as release +dashboard-data-20260930 (8b4c0ca1) committed it, the output definitions and the +reference system's convention modules as that commit held them, #194's law +table, and the pass's own records as #200 merged them. Later commits rewrote +the working tree's copies (#202 the run's payload and exclusion record, #204 +one convention module). These tests check that + +* the pins are what the pass read: each commit holds those bytes, the + committed evidence records the same hashes, and the README's Inputs table + lists each pin with the scripts that stage it; +* any other bytes are refused, and each script refuses before it computes or + writes anything and before it imports the engine; +* no script opens the working tree's copy of a pinned input (an audit hook + fails any such open), policybench's own read of its output definitions + included: the engine-side scripts point the package at their staged copy; +* each script regenerates its committed outputs byte for byte. The engine-side + regenerations need policyengine-us 2.15.17 and take minutes, so they are + marked slow and CI deselects them; build_proposals.py's regeneration is fast. + +CI checks out full history, so git holds every pinned input. +""" + +from __future__ import annotations + +import gzip +import hashlib +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from functools import cache +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit/2026-10-05-reference-adversary" +SCRIPTS = AUDIT / "scripts" +VERIFICATION = AUDIT / "verification" +PROBES = sorted((VERIFICATION / "probes").glob("*.json")) +sys.path.insert(0, str(SCRIPTS)) +import pass_inputs as pins # noqa: E402 + +from tests import working_tree_fence as fence # noqa: E402 + +# Release dashboard-data-20261006 (#202): the run's payload and exclusion record +# as it rewrote them. +RELEASE_20261006_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" +RELEASE_20261006_SHA256 = { + "data.json.gz": "b1da3eae058c1a511f5579bc8235c822cf6862e33caf4840004bd0cf0f34c41d", + "reference_exclusions.json": ( + "92741dfd047298eacb79a0dadeb7e4965e3c6f986beab0341b12ff8f5b023815" + ), +} +# latest_alt_snap_mortgage_residence.py as #204 rewrote it (held at 4db91b5f). +MODULE_204 = "reference_audit/2026-09-28/fixes/latest_alt_snap_mortgage_residence.py" +MODULE_204_SHA256 = "5e701f8fa1b31bfeed882a0bc5160faa4ee1d2b5a67974f9d9dc60e6f6638053" +# Every pin: (commit, path, sha256). +PINNED = ( + [ + (pins.PASS_COMMIT, f"{pins.RUN_PATH}/{name}", sha) + for name, sha in pins.RUN_SHA256.items() + ] + + [(pins.PASS_COMMIT, path, sha) for path, sha in pins.FIXES_SHA256.items()] + + [ + (pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256), + (pins.LAW_COMMIT, pins.LAW_PATH, pins.LAW_SHA256), + (pins.RECORDS_COMMIT, pins.PROPOSALS_PATH, pins.PROPOSALS_SHA256), + (pins.RECORDS_COMMIT, pins.CONFORMANCE_PATH, pins.CONFORMANCE_SHA256), + ] +) + + +def _load(name: str): + spec = importlib.util.spec_from_file_location( + f"reference_adversary_{name}", SCRIPTS / f"{name}.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +conformance = _load("definition_conformance") +sources = _load("publication_sources") +probe = _load("engine_probe") +proposals = _load("build_proposals") +impact = _load("leaderboard_impact") +SCRIPT_MODULES = { + "definition_conformance": conformance, + "publication_sources": sources, + "engine_probe": probe, + "build_proposals": proposals, +} + + +# --------------------------------------------------------------------------- +# A fence on the working tree's copy of every pinned input +# --------------------------------------------------------------------------- + +# The working tree's copies, policybench's own benchmark_specs.json included: the +# scripts here point the package at a staged copy before it reads the file. +FENCE = { + "forbidden": [ + str(ROOT / path) + for path in ( + pins.RUN_PATH, + *pins.FIXES_SHA256, + pins.SPECS_PATH, + pins.LAW_PATH, + pins.PROPOSALS_PATH, + pins.CONFORMANCE_PATH, + ) + ], + "exempt": {}, + "root": str(ROOT), +} + + +@pytest.fixture(autouse=True) +def package_specs(): + """Start each test with policybench's output definitions unloaded, and point + the package back at its own file afterwards: the engine-side scripts repoint + it at their staged copy.""" + from policybench import spec + + original = spec._spec_path + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() + try: + yield spec + finally: + spec._spec_path = original + spec._raw_spec_data.cache_clear() + spec.get_benchmark_spec.cache_clear() + + +@contextmanager +def fenced(): + """Inside, opening the working tree's copy of any pinned input raises.""" + fence.arm(**FENCE) + try: + yield + finally: + fence.disarm() + + +def _fenced_env(directory: Path) -> dict[str, str]: + """A subprocess environment whose every interpreter, workers included, is fenced.""" + return dict( + os.environ, + PYTHONPATH=f"{fence.sitecustomize(directory)}{os.pathsep}{ROOT}", + PYTHONDONTWRITEBYTECODE="1", + OPENBLAS_NUM_THREADS="1", + **{fence.ENV: json.dumps(FENCE)}, + ) + + +def test_the_fence_catches_a_working_tree_read(package_specs): + """The hook is not vacuous: each way the scripts read files trips it.""" + run = ROOT / pins.RUN_PATH + with fenced(): + with pytest.raises(PermissionError, match="working tree"): + pd.read_csv(run / "scenarios.csv") + with pytest.raises(PermissionError, match="working tree"): + gzip.open(run / "data.json.gz").read() + with pytest.raises(PermissionError, match="working tree"): + (ROOT / MODULE_204).read_text() + with pytest.raises(PermissionError, match="working tree"): + (ROOT / pins.CONFORMANCE_PATH).read_bytes() + # policybench's own read of its output definitions trips it too. + with pytest.raises(PermissionError, match="working tree"): + package_specs._raw_spec_data() + # git show reads the object store, not the working tree. + assert pins.git_bytes( + pins.PASS_COMMIT, MODULE_204, pins.FIXES_SHA256[MODULE_204] + ) + assert (run / "scenarios.csv").read_bytes() + + +FENCE_PROBE = """ +import sys +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + + +def read(path): + return len(Path(path).read_bytes()) + + +if __name__ == "__main__": + with ProcessPoolExecutor(1) as pool: + print(pool.submit(read, sys.argv[1]).result()) +""" + + +@pytest.mark.parametrize("where", ["main", "worker"]) +def test_the_fence_reaches_every_process_of_a_subprocess(where, tmp_path): + """Armed through sitecustomize, it fails a read in the main process and in a + spawned worker alike; unarmed, the same read succeeds.""" + script = tmp_path / "read.py" + if where == "main": + script.write_text("import sys\nprint(len(open(sys.argv[1], 'rb').read()))\n") + else: + script.write_text(FENCE_PROBE) + target = str(ROOT / pins.RUN_PATH / "scenarios.csv") + env = _fenced_env(tmp_path / "fence") + fenced_run = subprocess.run( + [sys.executable, str(script), target], env=env, capture_output=True, text=True + ) + assert fenced_run.returncode != 0 + assert "opened the working tree's" in fenced_run.stderr + env.pop(fence.ENV) + plain = subprocess.run( + [sys.executable, str(script), target], env=env, capture_output=True, text=True + ) + assert plain.returncode == 0, plain.stderr + assert int(plain.stdout) > 0 + + +# --------------------------------------------------------------------------- +# The pins are what the pass read +# --------------------------------------------------------------------------- + + +def test_every_pin_is_what_its_commit_holds(): + for commit, path, pinned in PINNED: + data = pins.git_bytes(commit, path, pinned) + assert hashlib.sha256(data).hexdigest() == pinned, path + # The README's frozen run: 46 models, 1,928 scored cells. + payload = json.loads( + gzip.decompress( + pins.git_bytes( + pins.PASS_COMMIT, + f"{pins.RUN_PATH}/data.json.gz", + pins.RUN_SHA256["data.json.gz"], + ) + ) + ) + assert len(payload["modelStats"]) == 46 + assert {row["n"] for row in payload["modelStats"]} == {1928} + + +def test_the_committed_evidence_records_the_pins(): + """Differential: the hashes the pass's own outputs recorded equal the pins.""" + run = json.loads((VERIFICATION / "definition_conformance.json").read_text())["run"] + assert run["payload"] == f"{pins.RUN_PATH}/data.json.gz" + assert run["payload_sha256"] == pins.RUN_SHA256["data.json.gz"] + assert run["reference_outputs_sha256"] == pins.RUN_SHA256["reference_outputs.csv"] + assert run["scenarios_sha256"] == pins.RUN_SHA256["scenarios.csv"] + assert run["benchmark_specs_sha256"] == pins.SPECS_SHA256 + assert run["fixes_sha256"] == pins.FIXES_SHA256 + assert run["law_classification"]["sha256"] == pins.LAW_SHA256 + assert run["law_classification"]["source"] == conformance.LAW_SOURCE + + meta = json.loads((VERIFICATION / "publication_sources.json").read_text())["meta"] + assert meta["payload"] == f"{pins.RUN_PATH}/data.json.gz" + assert meta["payload_sha256"] == pins.RUN_SHA256["data.json.gz"] + assert meta["reference_outputs"] == f"{pins.RUN_PATH}/reference_outputs.csv" + + flags = json.loads((AUDIT / "consensus_flags.json").read_text()) + assert flags["source"] == f"{pins.RUN_PATH}/data.json.gz" + assert flags["source_sha256"] == pins.RUN_SHA256["data.json.gz"] + + +def test_later_commits_hold_other_bytes_than_the_pins(): + """Why the pins exist: #202 rewrote the run's payload and exclusion record and + #204 a convention module, so reading those commits' copies is refused.""" + for name, sha in RELEASE_20261006_SHA256.items(): + with pytest.raises(SystemExit, match=f"has sha256 {sha}, not the pinned"): + pins.git_bytes( + RELEASE_20261006_COMMIT, + f"{pins.RUN_PATH}/{name}", + pins.RUN_SHA256[name], + ) + with pytest.raises(SystemExit, match=f"has sha256 {MODULE_204_SHA256}, not"): + pins.git_bytes(pins.RECORDS_COMMIT, MODULE_204, pins.FIXES_SHA256[MODULE_204]) + + +def _staged(script: str, tmp_path: Path) -> set[tuple[str, str]]: + """Every (commit, path) a script stages, recorded as git_input is called.""" + seen: set[tuple[str, str]] = set() + original = pins.git_input + + def record(commit, path, pinned, target): + seen.add((commit, path)) + return original(commit, path, pinned, target) + + with mock.patch.object(pins, "git_input", record): + if script == "leaderboard_impact": + impact.stage_inputs(tmp_path / script) + else: + SCRIPT_MODULES[script].stage_inputs(tmp_path / script) + if script in ("definition_conformance", "publication_sources"): + SCRIPT_MODULES[script].stage_engine_inputs() + if script == "engine_probe": + # The probe builds its reference system with definition_conformance's + # builder, which stages the conventions and output definitions. + conformance.stage_engine_inputs() + return seen + + +def _readme_inputs() -> dict[str, tuple[str, str, set[str]]]: + """README Inputs rows: path -> (commit, sha256, readers).""" + rows = {} + pattern = re.compile( + r"^\s*\| `(?P[^`]+)`[^|]* \| `(?P[0-9a-f]{8})` \| " + r"`(?P[0-9a-f]{64})` \|(?: (?P[^|]*) \|)?$" + ) + for line in (AUDIT / "README.md").read_text().splitlines(): + match = pattern.match(line) + if match is None: + continue + path = match["path"].replace("", pins.RUN_PATH) + readers = set(re.findall(r"`([^`]+)`", match["readers"] or "")) + assert path not in rows, f"README lists {path} twice" + rows[path] = (match["commit"], match["sha"], readers) + return rows + + +def test_the_readme_lists_each_pin_and_the_scripts_that_stage_it(tmp_path): + rows = _readme_inputs() + staged = { + script: _staged(script, tmp_path) + for script in [*SCRIPT_MODULES, "leaderboard_impact"] + } + readers: dict[tuple[str, str], set[str]] = {} + for script, inputs in staged.items(): + for key in inputs: + readers.setdefault(key, set()).add(f"{script}.py") + fixes = set(pins.FIXES_SHA256) + for commit, path, pinned in PINNED: + assert path in rows, f"README's Inputs table omits {path}" + row_commit, row_sha, row_readers = rows.pop(path) + assert commit.startswith(row_commit), path + assert row_sha == pinned, path + expected = readers.get((commit, path), set()) + assert expected, f"no script stages {path}" + if path in fixes: + # The convention modules' table names their readers once, above it. + assert row_readers == set(), path + continue + scripts = {reader for reader in row_readers if reader.endswith(".py")} + assert scripts == expected, path + # Rows left over are the CLI's inputs, which no script here reads. + for path, (commit, sha, row_readers) in rows.items(): + assert not any(reader.endswith(".py") for reader in row_readers), path + assert ( + hashlib.sha256( + subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path}"], + capture_output=True, + check=True, + ).stdout + ).hexdigest() + == sha + ), path + fix_readers = { + script + for script, inputs in staged.items() + if inputs & {(pins.PASS_COMMIT, path) for path in fixes} + } + assert fix_readers == { + "definition_conformance", + "publication_sources", + "engine_probe", + } + text = (AUDIT / "README.md").read_text() + assert ( + "read by `definition_conformance.py`, `publication_sources.py` and " + "`engine_probe.py`" in text + ) + + +def test_no_script_holds_its_own_pin_or_names_the_run(): + """pass_inputs.py is the only place a pin or the run's path is written.""" + hexdigest = re.compile(r"\b[0-9a-f]{64}\b") + for path in sorted(SCRIPTS.glob("*.py")): + if path.name == "pass_inputs.py": + continue + text = path.read_text() + assert not hexdigest.search(text), path.name + assert "paper/snapshot" not in text, path.name + + +# --------------------------------------------------------------------------- +# Any other bytes are refused +# --------------------------------------------------------------------------- + + +@cache +def _pinned_bytes(index: int) -> bytes: + commit, path, pinned = PINNED[index] + return pins.git_bytes(commit, path, pinned) + + +@st.composite +def _edits(draw) -> tuple[int, str, int, int]: + """One pinned input and one change to its bytes: flip, cut or extend.""" + index = draw(st.integers(0, len(PINNED) - 1)) + kind = draw(st.sampled_from(["flip", "truncate", "append"])) + position = draw(st.integers(0, len(_pinned_bytes(index)) - 1)) + return index, kind, position, draw(st.integers(1, 255)) + + +def _shown(data: bytes) -> SimpleNamespace: + return SimpleNamespace( + run=lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=data, stderr=b"" + ) + ) + + +@settings( + max_examples=60, + deadline=None, + # The autouse fixture only resets policybench's spec cache, which this + # property never touches. + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(_edits()) +def test_any_changed_byte_in_any_pinned_input_is_refused_unwritten(edit): + """Invariant: git_input writes a file only if its bytes are the pinned bytes.""" + index, kind, position, byte = edit + commit, path, pinned = PINNED[index] + original = _pinned_bytes(index) + if kind == "flip": + changed = bytearray(original) + changed[position] ^= byte + tampered = bytes(changed) + elif kind == "truncate": + tampered = original[:position] + else: + tampered = original + bytes([byte]) + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / Path(path).name + with mock.patch.object(pins, "subprocess", _shown(tampered)): + with pytest.raises(SystemExit, match=f"not the pinned {pinned}"): + pins.git_input(commit, path, pinned, target) + assert not target.exists() + # The unchanged bytes pass the same check. + with mock.patch.object(pins, "subprocess", _shown(original)): + pins.git_input(commit, path, pinned, target) + assert target.read_bytes() == original + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ( + "PASS_COMMIT", + pins.RECORDS_COMMIT, + f"latest_alt_snap_mortgage_residence.py has sha256 {MODULE_204_SHA256}, " + f"not the pinned {pins.FIXES_SHA256[MODULE_204]}", + ), + ("PASS_COMMIT", "0" * 40, "cannot read"), + ], +) +@pytest.mark.parametrize("script", ["definition_conformance", "publication_sources"]) +def test_the_reference_system_refuses_other_convention_modules( + script, attribute, value, message, monkeypatch +): + """#204's rewrite of a convention module (on 4db91b5f) is refused.""" + monkeypatch.setattr(pins, attribute, value) + with pytest.raises(SystemExit, match=re.escape(message)): + SCRIPT_MODULES[script].stage_engine_inputs() + + +def _never(name: str): + def called(*args, **kwargs): + raise AssertionError(f"{name} ran after an input was refused") + + return called + + +def _argv(script: str, out: Path) -> list[str]: + if script == "engine_probe": + return [script, "scenario_043", "--out", str(out / "probe.json")] + if script == "build_proposals": + return [script, "--out", str(out / "proposed_changes.json")] + return [script, "--out-dir", str(out)] + + +# What each script calls only after every input is staged. +ENGINE_ENTRY_POINTS = ( + "_system", + "stage_engine_inputs", + "install_build_hooks", + "install_recording", + "worker_pool", + "_conformance", +) +# The first input each script stages that #202 rewrote. +FIRST_REWRITTEN = { + "definition_conformance": "data.json.gz", + "publication_sources": "data.json.gz", + "engine_probe": "reference_exclusions.json", + "build_proposals": "reference_exclusions.json", +} + + +@pytest.mark.parametrize("script", sorted(SCRIPT_MODULES)) +def test_a_mismatched_input_stops_the_script_before_it_computes_or_writes( + script, tmp_path, monkeypatch +): + """Pointed at #202's run, each script refuses before any engine work or write.""" + module = SCRIPT_MODULES[script] + monkeypatch.setattr(pins, "PASS_COMMIT", RELEASE_20261006_COMMIT) + for name in ENGINE_ENTRY_POINTS: + if hasattr(module, name): + monkeypatch.setattr(module, name, _never(name)) + monkeypatch.setattr(module, "build", _never("build"), raising=False) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv(script, out)) + name = FIRST_REWRITTEN[script] + with pytest.raises( + SystemExit, + match=re.escape( + f"{name} has sha256 {RELEASE_20261006_SHA256[name]}, " + f"not the pinned {pins.RUN_SHA256[name]}" + ), + ): + module.main() + assert not out.exists() + + +def _bad_pin(monkeypatch, which: str) -> None: + """Make a late-staged pin wrong: a convention module or the definitions.""" + if which == "conventions": + last = list(pins.FIXES_SHA256)[-1] + monkeypatch.setattr(pins, "FIXES_SHA256", {**pins.FIXES_SHA256, last: "0" * 64}) + else: + monkeypatch.setattr(pins, "SPECS_SHA256", "0" * 64) + + +def _no_engine(monkeypatch) -> None: + """Any import of the engine from here on fails with ImportError.""" + for name in ("policyengine_us", "policyengine_core"): + monkeypatch.setitem(sys.modules, name, None) + + +@pytest.mark.parametrize("which", ["conventions", "definitions"]) +@pytest.mark.parametrize( + "script", ["definition_conformance", "publication_sources", "engine_probe"] +) +def test_a_late_staged_input_stops_each_main_before_the_engine( + script, which, tmp_path, monkeypatch +): + """The reference system's inputs are staged last. A wrong pin there still stops + the script before it imports the engine, hooks it, starts a worker or writes.""" + module = SCRIPT_MODULES[script] + _bad_pin(monkeypatch, which) + _no_engine(monkeypatch) + for name in ("_system", "install_build_hooks", "install_recording", "worker_pool"): + if hasattr(module, name): + monkeypatch.setattr(module, name, _never(name)) + if script == "engine_probe": + builder = SimpleNamespace( + stage_engine_inputs=conformance.stage_engine_inputs, + _system=_never("_system"), + _simulation=_never("_simulation"), + build_situation=_never("build_situation"), + ) + monkeypatch.setattr(probe, "_conformance", lambda: builder) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv(script, out)) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + module.main() + assert not out.exists() + + +@pytest.mark.parametrize("which", ["conventions", "definitions"]) +def test_a_worker_stages_its_inputs_before_it_imports_the_engine(which, monkeypatch): + """The paths a worker takes: definition_conformance's lazy system builder and + publication_sources's initializer both check the pins first.""" + _bad_pin(monkeypatch, which) + _no_engine(monkeypatch) + monkeypatch.setattr(conformance, "_SYSTEM", None) + monkeypatch.setattr(conformance, "_FIX_DIR", None) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + conformance._system() + monkeypatch.setattr(sources, "install_recording", _never("install_recording")) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + sources._init_worker() + + +def test_workers_are_spawned_and_stage_their_own_inputs(): + """Never forked: a forked worker would inherit the main process's system and + policybench state instead of staging its own. Each worker's initializer stages.""" + for module, initializer in ( + (conformance, conformance.stage_engine_inputs), + (sources, sources._init_worker), + ): + pool = module.worker_pool(1) + try: + assert pool._mp_context.get_start_method() == "spawn" + assert pool._initializer is initializer + finally: + pool.shutdown() + + +def test_publication_sources_has_no_mode_that_skips_staging(tmp_path, monkeypatch): + """--from-facts built both reports from a saved file with no pin check.""" + for name in ("stage_inputs", *ENGINE_ENTRY_POINTS): + if hasattr(sources, name): + monkeypatch.setattr(sources, name, _never(name)) + facts = tmp_path / "facts.json" + facts.write_text(json.dumps({"facts": [], "other_instants": [], "meta": {}})) + out = tmp_path / "out" + monkeypatch.setattr( + sys, + "argv", + ["publication_sources", "--out-dir", str(out), "--from-facts", str(facts)], + ) + with pytest.raises(SystemExit) as refused: + sources.main() + assert refused.value.code == 2 + assert not out.exists() + + +def test_a_script_refuses_another_module_named_pass_inputs(tmp_path): + """A pass_inputs already imported from elsewhere would win over the scripts' + directory; each script checks where its pins came from.""" + driver = tmp_path / "driver.py" + driver.write_text( + "import runpy, sys, types\n" + "fake = types.ModuleType('pass_inputs')\n" + "fake.__file__ = '/elsewhere/pass_inputs.py'\n" + "sys.modules['pass_inputs'] = fake\n" + "runpy.run_path(sys.argv[1], run_name='not_main')\n" + ) + for path in sorted(SCRIPTS.glob("*.py")): + if "import pass_inputs" not in path.read_text(): + continue + result = subprocess.run( + [sys.executable, str(driver), str(path)], capture_output=True, text=True + ) + assert result.returncode != 0, path.name + assert "/elsewhere/pass_inputs.py, not this script's" in result.stderr + + +@pytest.mark.parametrize( + ("attribute", "value", "message"), + [ + ("CONFORMANCE_SHA256", "0" * 64, f"not the pinned {'0' * 64}"), + ("RECORDS_COMMIT", pins.PASS_COMMIT, "cannot read"), + ], +) +def test_build_proposals_refuses_another_conformance_scan( + attribute, value, message, tmp_path, monkeypatch +): + monkeypatch.setattr(pins, attribute, value) + monkeypatch.setattr(proposals, "build", _never("build")) + out = tmp_path / "out" + monkeypatch.setattr(sys, "argv", _argv("build_proposals", out)) + with pytest.raises(SystemExit, match=re.escape(message)): + proposals.main() + assert not out.exists() + + +def test_definition_conformance_refuses_another_law_table(tmp_path, monkeypatch): + monkeypatch.setattr(pins, "LAW_SHA256", "0" * 64) + with pytest.raises(SystemExit, match=f"not the pinned {'0' * 64}"): + conformance.stage_inputs(tmp_path / "inputs") + + +def test_build_proposals_never_writes_the_committed_records( + tmp_path, monkeypatch, capsys +): + committed = (AUDIT / "proposed_changes.json").read_bytes() + monkeypatch.setattr(proposals, "build", _never("build")) + link = tmp_path / "link.json" + link.symlink_to(AUDIT / "proposed_changes.json") + for target in ( + AUDIT / "proposed_changes.json", + SCRIPTS / ".." / "proposed_changes.json", + Path(os.path.relpath(AUDIT / "proposed_changes.json")), + link, + ): + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(target)]) + with pytest.raises(SystemExit): + proposals.main() + assert "is pinned at sha256" in capsys.readouterr().err + monkeypatch.setattr(sys, "argv", ["build_proposals"]) + with pytest.raises(SystemExit): + proposals.main() + assert (AUDIT / "proposed_changes.json").read_bytes() == committed + assert hashlib.sha256(committed).hexdigest() == pins.PROPOSALS_SHA256 + + +def test_build_proposals_writes_only_a_new_file(tmp_path, monkeypatch, capsys): + """A hard link to the committed records, another checkout's copy and a dangling + symlink are existing paths, and no existing path is written.""" + committed = AUDIT / "proposed_changes.json" + before = committed.read_bytes() + monkeypatch.setattr(proposals, "build", _never("build")) + hard = tmp_path / "hard.json" + os.link(committed, hard) + other = tmp_path / "checkout" / pins.PROPOSALS_PATH + other.parent.mkdir(parents=True) + other.write_bytes(before) + dangling = tmp_path / "dangling.json" + dangling.symlink_to(tmp_path / "nowhere.json") + for target in (hard, other, dangling): + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(target)]) + with pytest.raises(SystemExit): + proposals.main() + assert "--out must be a new file" in capsys.readouterr().err + assert committed.read_bytes() == hard.read_bytes() == other.read_bytes() == before + assert not (tmp_path / "nowhere.json").exists() + + +def test_build_proposals_creates_its_output_exclusively(tmp_path, monkeypatch): + """A file that appears at --out after the check is not written through: here + it is a hard link to the committed records, created while the script builds.""" + committed = AUDIT / "proposed_changes.json" + before = committed.read_bytes() + out = tmp_path / "proposed_changes.json" + + def build(inputs): + os.link(committed, out) + return {"root_causes": {}} + + monkeypatch.setattr(proposals, "build", build) + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(out)]) + with pytest.raises(FileExistsError): + proposals.main() + assert committed.read_bytes() == out.read_bytes() == before + + +# --------------------------------------------------------------------------- +# The working tree is never read +# --------------------------------------------------------------------------- + + +def test_build_proposals_regenerates_the_committed_records_from_git( + tmp_path, monkeypatch +): + """The whole script, with every working-tree input fenced off, writes the + committed proposed_changes.json byte for byte.""" + out = tmp_path / "proposed_changes.json" + monkeypatch.setattr(sys, "argv", ["build_proposals", "--out", str(out)]) + with fenced(): + proposals.main() + assert out.read_bytes() == (AUDIT / "proposed_changes.json").read_bytes() + assert pins.sha256(out) == pins.PROPOSALS_SHA256 + + +def test_definition_conformance_reads_only_staged_inputs(tmp_path): + with fenced(): + staged = conformance.stage_inputs(tmp_path / "inputs") + laws, meta = conformance.law_classifications( + staged / "program_classification.json" + ) + assert sorted(path.name for path in (staged / "run").iterdir()) == sorted( + conformance.RUN_FILES + ) + for name in conformance.RUN_FILES: + assert pins.sha256(staged / "run" / name) == pins.RUN_SHA256[name] + recorded = json.loads((VERIFICATION / "definition_conformance.json").read_text()) + assert meta == recorded["run"]["law_classification"] + assert laws + + +def test_the_reference_system_is_built_from_staged_inputs(package_specs): + """Conventions and output definitions are staged, and policybench then reads + the staged definitions: with the working tree fenced off, its lookups work.""" + from policybench.ground_truth import _pe_variable_for_output + + expected = {Path(path).name: sha for path, sha in pins.FIXES_SHA256.items()} + pinned = json.loads( + pins.git_bytes(pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256) + ) + for module in (conformance, sources): + with fenced(): + fix_dir = module.stage_engine_inputs() + assert package_specs._raw_spec_data() == pinned + assert _pe_variable_for_output("snap", "us") == "snap" + assert {path.name: pins.sha256(path) for path in fix_dir.iterdir()} == expected + assert pins.sha256(package_specs._spec_path()) == pins.SPECS_SHA256 + # definition_conformance records the hashes of what it staged. + assert conformance.stage_engine_inputs() == conformance._FIX_DIR + assert pins.sha256(conformance._SPECS) == pins.SPECS_SHA256 + + +def _mutated_specs(target: Path) -> tuple[Path, str]: + """The pinned definitions with the US ``snap`` output's engine variable changed.""" + data = json.loads( + pins.git_bytes(pins.PASS_COMMIT, pins.SPECS_PATH, pins.SPECS_SHA256) + ) + outputs = data["specs"]["policybench"]["countries"]["us"] + (snap,) = [output for output in outputs if output["id"] == "snap"] + assert snap["pe_variable"] == "snap" + snap["pe_variable"] = "not_the_pass_variable" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(data)) + return target, "not_the_pass_variable" + + +def test_staged_definitions_replace_a_checkouts_changed_ones(tmp_path, package_specs): + """A checkout whose benchmark_specs.json maps an output to another engine + variable does not change what the scripts compute: lookups read the pin.""" + from policybench.ground_truth import _pe_variable_for_output + + mutated, variable = _mutated_specs(tmp_path / "benchmark_specs.json") + package_specs._spec_path = lambda: mutated + assert _pe_variable_for_output("snap", "us") == variable + package_specs._raw_spec_data.cache_clear() + package_specs.get_benchmark_spec.cache_clear() + conformance.stage_engine_inputs() + assert _pe_variable_for_output("snap", "us") == "snap" + + +def test_definitions_already_loaded_from_elsewhere_are_refused(tmp_path, package_specs): + """Constants derived from other definitions cannot be recalled, so a process + that already loaded them is refused, not silently repointed.""" + mutated, _ = _mutated_specs(tmp_path / "benchmark_specs.json") + package_specs._spec_path = lambda: mutated + package_specs._raw_spec_data() + staged = pins.stage_specs(tmp_path / "staged.json") + with pytest.raises(SystemExit, match="fresh interpreter"): + pins.use_staged_specs(staged) + assert package_specs._spec_path() == mutated + + +SPECS_DRIVER = """ +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[1]) +import pass_inputs + +staged = pass_inputs.stage_specs(Path(sys.argv[2])) +if sys.argv[3] == "staged": + pass_inputs.use_staged_specs(staged) +from policybench import scenarios, spec # scenarios derives constants as it loads +from policybench.ground_truth import _pe_variable_for_output + +print(spec.__file__) +print(_pe_variable_for_output("snap", "us")) +""" + + +def test_a_mutated_checkout_specs_file_never_reaches_the_engine_helpers(tmp_path): + """End to end in fresh interpreters: with a copy of the package whose + benchmark_specs.json is changed first on the path, the helpers the scripts + call return the pinned mapping once the staged definitions are in use, and + the changed one otherwise.""" + package = tmp_path / "checkout" / "policybench" + shutil.copytree( + ROOT / "policybench", package, ignore=shutil.ignore_patterns("__pycache__") + ) + _, variable = _mutated_specs(package / "benchmark_specs.json") + driver = tmp_path / "driver.py" + driver.write_text(SPECS_DRIVER) + seen = {} + for mode in ("staged", "checkout"): + result = subprocess.run( + [ + sys.executable, + str(driver), + str(SCRIPTS), + str(tmp_path / f"{mode}.json"), + mode, + ], + env=dict(os.environ, PYTHONPATH=str(package.parent)), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-2000:] + module, seen[mode] = result.stdout.split() + assert Path(module).parent == package + assert seen == {"staged": "snap", "checkout": variable} + + +def test_publication_sources_reads_only_staged_inputs(tmp_path): + with fenced(): + run = sources.stage_inputs(tmp_path / "run") + scored, sha = sources._scored_cells_from_payload(run / "data.json.gz") + assert sorted(path.name for path in run.iterdir()) == sorted(sources.RUN_FILES) + assert sha == pins.RUN_SHA256["data.json.gz"] + assert len(scored) == sources.SCORED_CELLS + reference = pd.read_csv(run / "reference_outputs.csv") + exclusions = json.loads((run / "reference_exclusions.json").read_text()) + excluded = {(e["scenario_id"], e["variable"]) for e in exclusions["exclusions"]} + keys = set(zip(reference["scenario_id"], reference["variable"])) + assert scored == keys - excluded + + +def test_engine_probe_scores_the_pass_cells_not_202s(tmp_path, monkeypatch): + """The probe's reproduction set comes from the pass's exclusion record. + + #202 later excluded scenario_043's payroll_tax; read from the working tree, + the probe would drop it. The engine is stubbed to return the published + references, so this runs without policyengine-us. + """ + reference = pd.read_csv( + io.BytesIO( + pins.git_bytes( + pins.PASS_COMMIT, + f"{pins.RUN_PATH}/reference_outputs.csv", + pins.RUN_SHA256["reference_outputs.csv"], + ) + ) + ) + published = { + row.variable: float(row.value) + for row in reference[reference["scenario_id"] == "scenario_043"].itertuples() + } + engine = SimpleNamespace( + stage_engine_inputs=conformance.stage_engine_inputs, + build_situation=conformance.build_situation, + _system=lambda: SimpleNamespace(variables={}), + _simulation=lambda situation: None, + _outputs=lambda sim, scenario, scored: {v: published[v] for v in scored}, + REPRODUCE_TOLERANCE=conformance.REPRODUCE_TOLERANCE, + ) + monkeypatch.setattr(probe, "_conformance", lambda: engine) + out = tmp_path / "probe.json" + monkeypatch.setattr( + sys, "argv", ["engine_probe", "scenario_043", "--out", str(out)] + ) + with fenced(): + probe.main() + committed = json.loads( + (VERIFICATION / "probes/co_sales_tax_refund_scenario_043.json").read_text() + ) + written = json.loads(out.read_text()) + assert "payroll_tax" in written["reproduction"] + assert { + variable: row["published"] for variable, row in written["reproduction"].items() + } == { + variable: row["published"] + for variable, row in committed["reproduction"].items() + } + assert written["people"] == committed["people"] + assert written["state"] == committed["state"] == "CO" + + +# --------------------------------------------------------------------------- +# Byte-for-byte regeneration (slow: the reference system, policyengine-us 2.15.17) +# --------------------------------------------------------------------------- + + +def _run(script: str, tmp_path: Path, *args: str) -> None: + """Run a script with the fence armed in every one of its processes.""" + result = subprocess.run( + [sys.executable, str(SCRIPTS / f"{script}.py"), *args], + cwd=ROOT, + env=_fenced_env(tmp_path / "fence"), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-4000:] + + +LINE = re.compile( + rb'^(?P\s*"(?P[a-z_0-9]+)": )(?P.*?)(?P,?\n)$' +) + + +def _identical_but(regenerated: Path, committed: Path, volatile: set[str]) -> dict: + """Require the files byte for byte, except the value on the one line that + carries each key in ``volatile``; return each such regenerated value. + + The committed file must carry each volatile key on exactly one line, and the + regenerated file must carry the same key, indentation and punctuation on the + same line, so only that one JSON value may differ. + """ + new = regenerated.read_bytes().splitlines(keepends=True) + old = committed.read_bytes().splitlines(keepends=True) + assert len(new) == len(old), regenerated.name + values, allowed = {}, set() + for key in volatile: + lines = [ + i + for i, text in enumerate(old) + if (match := LINE.match(text)) and match["key"].decode() == key + ] + assert len(lines) == 1, (committed.name, key, lines) + (index,) = lines + before, after = LINE.match(old[index]), LINE.match(new[index]) + assert after is not None, (regenerated.name, new[index][:120]) + assert (after["head"], after["tail"]) == (before["head"], before["tail"]) + values[key] = json.loads(after["value"]) + allowed.add(index) + unexpected = [ + (i + 1, b[:120], a[:120]) + for i, (a, b) in enumerate(zip(new, old)) + if a != b and i not in allowed + ] + assert not unexpected, (regenerated.name, unexpected[:5]) + return values + + +def test_only_the_named_values_may_differ(tmp_path): + """The comparison is not vacuous: a changed value elsewhere, a moved key or a + changed key name on the volatile line all fail it.""" + committed = tmp_path / "committed.json" + committed.write_text('{\n "a": 1,\n "seconds": 2.5,\n "z": [1]\n}\n') + cases = { + '{\n "a": 1,\n "seconds": 9.1,\n "z": [1]\n}\n': True, + '{\n "a": 2,\n "seconds": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "minutes": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5,\n "z": [1]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5,\n "z": [2]\n}\n': False, + '{\n "a": 1,\n "seconds": 2.5\n}\n': False, + } + for text, passes in cases.items(): + regenerated = tmp_path / "regenerated.json" + regenerated.write_text(text) + if passes: + assert _identical_but(regenerated, committed, {"seconds"}) == { + "seconds": 9.1 + } + else: + with pytest.raises(AssertionError): + _identical_but(regenerated, committed, {"seconds"}) + + +@pytest.mark.slow +def test_definition_conformance_regenerates_byte_for_byte(tmp_path): + """Only the wall time and this script's own sha256 differ from the record. + + About two to four minutes. Run with + OPENBLAS_NUM_THREADS=1 uv run pytest -m slow \\ + tests/test_reference_adversary_inputs.py + """ + _run("definition_conformance", tmp_path, "--out-dir", str(tmp_path)) + assert (tmp_path / "definition_conformance.md").read_bytes() == ( + VERIFICATION / "definition_conformance.md" + ).read_bytes() + values = _identical_but( + tmp_path / "definition_conformance.json", + VERIFICATION / "definition_conformance.json", + {"seconds", "script_sha256"}, + ) + assert values["script_sha256"] == pins.sha256(SCRIPTS / "definition_conformance.py") + + +@pytest.mark.slow +def test_publication_sources_regenerates_byte_for_byte(tmp_path): + """Only the wall time differs from the record. About four to ten minutes.""" + _run("publication_sources", tmp_path, "--out-dir", str(tmp_path)) + assert (tmp_path / "publication_sources.md").read_bytes() == ( + VERIFICATION / "publication_sources.md" + ).read_bytes() + _identical_but( + tmp_path / "publication_sources.json", + VERIFICATION / "publication_sources.json", + {"seconds"}, + ) + + +def _probe_args(path: Path) -> list[str]: + """The arguments a committed probe records about itself.""" + record = json.loads(path.read_text()) + args = [record["scenario_id"], "--period", record["period"]] + args += ["--variables", *record["variables"]] + args += ["--parameters", *record["parameters"]] + for assignment in record.get("counterfactual", {}).get("set", []): + args += ["--set", assignment] + return args + + +@pytest.mark.slow +@pytest.mark.parametrize("path", PROBES, ids=[path.stem for path in PROBES]) +def test_every_probe_regenerates_byte_for_byte(path, tmp_path): + """About a minute each.""" + assert len(PROBES) == 8 + out = tmp_path / path.name + _run("engine_probe", tmp_path, *_probe_args(path), "--out", str(out)) + assert out.read_bytes() == path.read_bytes() diff --git a/tests/working_tree_fence.py b/tests/working_tree_fence.py new file mode 100644 index 00000000..a97c3c18 --- /dev/null +++ b/tests/working_tree_fence.py @@ -0,0 +1,91 @@ +"""An audit hook that fails any open of the working tree's copy of a pinned input. + +tests/test_reference_adversary_inputs.py arms it in-process around the +reference adversary's scripts, and in every process of a script it runs as a +subprocess, worker processes included: ``sitecustomize(directory)`` writes a +``sitecustomize.py`` that loads this file and calls ``install_from_env()``, so +putting ``directory`` first on PYTHONPATH and the fence's JSON in ``ENV`` arms +each interpreter as it starts. + +``exempt`` maps a forbidden file to the one directory whose code may open it: +policybench reads its own ``benchmark_specs.json`` (package data, which the +scripts treat as code), while a script that opened that file itself would trip +the fence. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +ENV = "REFERENCE_ADVERSARY_FENCE" +_STATE: dict = {"forbidden": (), "exempt": {}, "root": ""} +_INSTALLED = False + + +def _innermost_repository_frame() -> str: + """The file of the innermost calling frame that is repository code.""" + root = _STATE["root"] + os.sep + frame = sys._getframe(2) + while frame is not None: + name = os.path.abspath(frame.f_code.co_filename) + if name.startswith(root) and f"{os.sep}.venv{os.sep}" not in name: + return name + frame = frame.f_back + return "" + + +def _hook(event: str, args: tuple) -> None: + if event != "open" or not _STATE["forbidden"]: + return + target = args[0] + if not isinstance(target, (str, bytes, os.PathLike)): + return + path = os.path.abspath(os.fsdecode(target)) + if not any( + path == forbidden or path.startswith(forbidden + os.sep) + for forbidden in _STATE["forbidden"] + ): + return + owner = _STATE["exempt"].get(path) + if owner and _innermost_repository_frame().startswith(owner + os.sep): + return + raise PermissionError(f"opened the working tree's {path}") + + +def arm(forbidden, exempt: dict[str, str], root: str) -> None: + global _INSTALLED + if not _INSTALLED: + sys.addaudithook(_hook) + _INSTALLED = True + _STATE.update( + forbidden=tuple(os.path.abspath(path) for path in forbidden), + exempt={os.path.abspath(k): os.path.abspath(v) for k, v in exempt.items()}, + root=os.path.abspath(root), + ) + + +def disarm() -> None: + _STATE["forbidden"] = () + + +def install_from_env() -> None: + fence = os.environ.get(ENV) + if fence: + arm(**json.loads(fence)) + + +def sitecustomize(directory: Path) -> Path: + """Write a sitecustomize.py into ``directory`` that arms the fence from ENV.""" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sitecustomize.py").write_text( + "import importlib.util\n" + 'spec = importlib.util.spec_from_file_location("working_tree_fence", ' + f"{str(Path(__file__).resolve())!r})\n" + "module = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(module)\n" + "module.install_from_env()\n" + ) + return directory