diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6bc6f9..fd5132ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: # pypdf + pyyaml are NOT optional extras: the HMT lane's strongest # honesty guarantee (every registry title and figure anchor verbatim # in the real PDF) runs as a test, and a bare env silently skipped it. - - run: uv run --with pytest --with hypothesis --with pypdf --with pyyaml --with numpy --with psutil pytest tests/ -q + - run: uv run --with pytest --with hypothesis --with pypdf --with pyyaml --with numpy --with psutil --with rfc8785==0.1.4 --with jsonschema pytest tests/ -q env: PYTHONPATH: . - run: git diff --exit-code diff --git a/data/uk/replay_variants.json b/data/uk/replay_variants.json new file mode 100644 index 00000000..60743ba8 --- /dev/null +++ b/data/uk/replay_variants.json @@ -0,0 +1,197 @@ +{ + "default_variant": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "schema_version": 1, + "variants": [ + { + "certification": { + "certified_for_model_version": "2.89.2", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "source": "policyengine.py 5.0.2 packaged bundle manifest; docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json" + }, + "data_years": [ + 2023, + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "data_years_evidence": { + "dataset_time_period": [ + "2023" + ], + "engine_extension": "policyengine_uk 2.89.2 data/economic_assumptions.py extend_single_year_dataset end_year=2030", + "simulation_dataset_years": [ + 2023, + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "source": "docs/uk_replay/YEARS.md" + }, + "executor": [ + "local", + "modal" + ], + "identity": { + "dataset": { + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "size_bytes": 1315880118 + }, + "materialization": "hf_cache_mirror", + "packaged_bundle_manifest_sha256": "255dbb893f455d078a6bcece87399f0a22478f56bc4c6e526b758c4fd5b4db24", + "packages": { + "policyengine": { + "record_sha256": "8d5eec548a5ce206a8fe59ef018542c701e26c0f927351f92339a7a57591b990", + "version": "5.0.2" + }, + "policyengine-core": { + "record_sha256": "103128a1c2a3397268b65e33b83edddfa8e406a4eb64fb6f4ff2310db421c770", + "version": "3.27.1" + }, + "policyengine-uk": { + "record_sha256": "4628bbb8ad80eec4624fc4c18214c186d1405098fa8cf40894a4437e071e1acc", + "version": "2.89.2" + } + }, + "python": "3.12", + "release_bundle": { + "bundle_id": "uk-5.0.2", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "certified_data_build_id": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "compatibility_basis": "built_with_model_package", + "country_id": "uk", + "data_build_fingerprint": null, + "data_build_model_git_sha": null, + "data_build_model_version": "2.89.2", + "data_package": "populace-data", + "data_version": "0.1.0", + "default_dataset": "populace_uk_2023", + "default_dataset_uri": "hf://policyengine/populace-uk-private/populace_uk_2023.h5@populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "model_package": "policyengine-uk", + "model_version": "2.89.2", + "policyengine_version": "5.0.2" + }, + "requirements_sha256": "450e64b173b66333027ac36e056e19928d23df1cbe2690e462bab09b31ddff9d" + }, + "identity_sha256": "3f1a33f31ac70a8f0beac2373b6408c8be3366128ec0d0e3c1b716de3880003c", + "layout": "legacy_events", + "note": "The bundle the track 1 replays were computed against. Its 1.3GB populace dataset needs Modal's 32GiB containers for full events.", + "requirements": { + "path": "docs/uk_replay/requirements.txt", + "sha256": "450e64b173b66333027ac36e056e19928d23df1cbe2690e462bab09b31ddff9d" + }, + "result_root": "results/uk/events", + "status": "certified", + "variant_id": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" + }, + { + "certification": { + "certified_for_model_version": "2.125.1", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_id": "policyengine-uk-data-1.58.0", + "source": "policyengine.py 6.2.6 packaged bundle manifest (PolicyEngine/policyengine.py#555, 481940eca44fc75e41fbf9de4ff454e0a0d7c0e0)" + }, + "data_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "data_years_evidence": { + "dataset_time_period": [ + "2024" + ], + "engine_extension": "policyengine_uk 2.125.1 data/economic_assumptions.py:53-56 extend_single_year_dataset end_year=2030", + "simulation_dataset_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "source": "Measured 2026-10-11 with the installed wheels: pandas.read_hdf(, 'time_period') and pe.uk.managed_microsimulation().dataset.years; a 2023 calculation returned defaults (income_tax 0.0) without raising." + }, + "executor": [ + "local", + "modal" + ], + "identity": { + "dataset": { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "size_bytes": 128369110 + }, + "materialization": "policyengine_data_dir", + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", + "packages": { + "policyengine": { + "record_sha256": "f5990ac205b959621c56585c16ed963a5d1b07b1403adaa7242e0131fbf61f40", + "version": "6.2.6", + "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" + }, + "policyengine-core": { + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine-uk": { + "record_sha256": "c39b8d44d042125cb380ff8d9f8bf75516aba129b877c416e536e63c49ce0b72", + "version": "2.125.1", + "wheel_sha256": "a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a" + } + }, + "python": "3.12", + "release_bundle": { + "bundle_id": "uk-6.2.6", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "policyengine_version": "6.2.6" + }, + "requirements_sha256": "5c333faef091e6780b2c178ac7584249ac7db0159841921727b913505577bcdb" + }, + "identity_sha256": "3f62a208a9bf522cccb391fe65c1411124abaf4488ab760c522b0568f5b417aa", + "layout": "replay_namespace", + "note": "The newest certified pair on 11 October 2026; data built with policyengine-uk 2.122.2 and certified for 2.125.1.", + "requirements": { + "path": "docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt", + "sha256": "5c333faef091e6780b2c178ac7584249ac7db0159841921727b913505577bcdb" + }, + "result_root": "results/uk/replays/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "status": "certified", + "variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" + } + ] +} diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md new file mode 100644 index 00000000..13035773 --- /dev/null +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -0,0 +1,246 @@ +# Qualification request and replay receipt (v1) + +Status: proposed by W6 (policyengine-scorecard) to W4 (policyengine.py +qualification) on 11 October 2026 and accepted in principle by W4 the same +day, with the amendments folded in below (RFC 8785 canonicalisation, the +receipt digest and commit in `STATUS.json`, extra `requested_by` fields, and +the post-release flow). JSON Schemas: +[`schemas/qualification_request.v1.json`](schemas/qualification_request.v1.json), +[`schemas/replay_receipt.v1.json`](schemas/replay_receipt.v1.json) and +[`schemas/replay_status.v1.json`](schemas/replay_status.v1.json). +Implementation: `pipeline/uk_replay_interface.py`. Golden fixtures with +pinned digests: `tests/fixtures/uk_replay/` (`DIGESTS.json`, and the exact +JCS bytes in `*.jcs`). + +## Canonical bytes + +Both sides compute digests over **RFC 8785 (JCS)** bytes, using the +`rfc8785` package (0.1.4 here). `json.dumps(sort_keys=True)` is not +equivalent: it writes `1e-07` where JCS writes `1e-7`, `1.5e+16` where JCS +writes `15000000000000000`, and escapes non-ASCII characters that JCS +writes as UTF-8. Integers must lie within ±(2^53 − 1); the package refuses +larger ones. The golden request carries these probes in +`requested_by.jcs_probe` and a non-ASCII `requested_by.note`. + +Two flows use the same receiver: + +* **Pre-promotion qualification.** W4's certify workflow, holding a candidate + model/data pair it has not yet promoted, sends an immutable request. W6 + validates the candidate's identity and compatibility, reruns the supported + event corpus, diffs it against the latest certified variant, and returns an + artifact-bound receipt that qualification consumes. +* **Post-promotion consumer dispatch.** After a policyengine.py release, + W4's NotifyConsumers sends the same event with `kind: "released_bundle"`, + the policyengine PyPI wheel SHA-256 and the packaged manifest SHA-256. The + same receiver handles it, so scorecard ends up with a certified variant + and a sealed namespace for every release; there is no separate handler. + +## Transport + +* W4 sends `repository_dispatch` to `PolicyEngine/policyengine-scorecard` + with `event_type: "uk-replay-qualification-request"` and + `client_payload: {"request": }`. W4 alone edits the policyengine.py + dispatcher and its GitHub App token scope; scorecard needs + `contents: read` on policyengine.py (public) and nothing else from W4. +* GitHub keeps at most one pending run per concurrency group and cancels + older pending runs, so the receiver's first step writes the request to the + persistent queue on the `replay-state` branch before anything else. A + scheduled catch-up (every 3 hours) drains the queue and discovers + policyengine.py releases that were never dispatched. +* State for request `` lives at + `replay-state:requests//`: + `REQUEST.json` (the bytes received), `STATUS.json` + (`queued` → `running` → `complete` | `rejected` | `failed`, with a + history of timestamps and Actions run URLs), and on completion + `REPLAY_RECEIPT.json`. The receipt is committed first; a terminal + `STATUS.json` is written in a later commit and names `receipt_sha256` and + `receipt_commit`, so one contents read binds the status to the receipt + bytes. +* W4 polls `GET /repos/PolicyEngine/policyengine-scorecard/contents/requests//STATUS.json?ref=replay-state` + (the repository is public). A receipt is addressed by the request's own + digest, so W4 needs no callback and no scorecard token. +* Duplicate requests (same `request_id`) are idempotent: an existing + terminal status is returned, never rerun. A failed request can be retried + only by a new request (a different `attempt`). + +## Request + +```json +{ + "schema": "policyengine.uk-replay-qualification-request/v1", + "kind": "candidate" , + "attempt": 1, + "candidate": { + "policyengine": { + "version": "6.2.7", + "source": {"git": {"repository": "PolicyEngine/policyengine.py", "commit": "<40-hex>"}}, + "packaged_bundle_manifest_sha256": "<64-hex>" + }, + "policyengine_uk": {"version": "2.126.8", "wheel_sha256": "<64-hex>"}, + "policyengine_core": {"version": "3.33.0", "wheel_sha256": "<64-hex>"}, + "data": { + "build_id": "policyengine-uk-data-1.58.0", + "dataset": "enhanced_frs_2024_25", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "path": "enhanced_frs_2024_25.h5", + "sha256": "<64-hex>", + "size_bytes": 128369110, + "release_manifest_sha256": "<64-hex>" + }, + "expected_release_bundle": { "...": "the exact pe.uk.uk_latest.release_bundle under packaged certification" } + }, + "compare_to": "latest_certified", + "requested_by": { + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 1234567890, + "commit": "<40-hex>" + } +} +``` + +* `request_id` is not a field: it is the SHA-256 of the request's JCS + bytes. Both sides test against the golden fixture. +* `requested_by` names the requesting repository and commit; + `workflow_run_id` is optional (a scorecard catch-up run started outside + Actions has none), and additional provenance fields such as `run_attempt` + are accepted. Every other object is closed. +* The wrapper is identified by `version` plus either a PyPI wheel + (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose + regenerated manifest is not on PyPI yet, a public git commit. Scorecard + installs exactly that, then checks the installed packaged manifest's SHA-256. +* Model and core are PyPI wheels pinned by SHA-256. +* `kind` is `candidate` or `released_bundle`. +* `compare_to` is `latest_certified` (default) or a `variant_id`. + +## What scorecard validates before running + +The candidate becomes a `candidate` variant record (RELEASE_REPLAY.md). It is +**rejected**, with a code, when: + +| Code | Condition | +|---|---| +| `schema_invalid` | The request fails the v1 schema. | +| `artifact_hash_mismatch` | A downloaded wheel, the installed packaged manifest or the dataset bytes differ from the request. | +| `release_bundle_mismatch` | `pe.uk.uk_latest.release_bundle` under packaged certification differs from `expected_release_bundle`. | +| `incompatible_pair` | The packaged certification does not name the candidate model version, or the dataset is not the bundle's certified artifact. | +| `data_access_failed` | The dataset could not be fetched with the receiver's credential. | +| `no_supported_years` | No event year falls in the dataset's supported years. | + +A rejected request has no numerical results; its receipt says why. + +## Receipt + +```json +{ + "schema": "policyengine.uk-replay-receipt/v1", + "request_id": "<64-hex>", + "status": "complete", + "rejection": null, + "variant_id": "pe6.2.7_uk2.126.8_policyengine-uk-data-1.58.0", + "variant_identity_sha256": "<64-hex>", + "execution": { + "packages": {"policyengine": {"version": "6.2.7", "record_sha256": "…"}, "...": "..."}, + "packaged_bundle_manifest_sha256": "…", + "release_bundle": {"...": "..."}, + "dataset_sha256": "…", + "requirements_sha256": "…", + "python": "3.12.13", + "platform": "Linux-…", + "runner": "github-actions ubuntu-latest" + }, + "corpus": { + "corpus_version": "uk-fiscal-event-replay/1", + "events": { + "autumn_budget_2024": { + "registry_sha256": "…", + "supported_years": [2024, 2025, 2026, 2027, 2028, 2029], + "unsupported_years": [], + "run_manifest_sha256": "…", + "staging_manifest_sha256": "…", + "comparison_provenance_sha256": "…", + "artifacts": 30, + "full_supported_grid_complete": true, + "inert_measure_years": [] + } + } + }, + "integrity": { + "status": "complete", + "failures": [] + }, + "diff": { + "base_variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "diff_sha256": "…", + "flag_counts": {"numerical": 0, "structural": 0, "coverage": 0, "mapping": 0, "support": 0, "inertness": 0, "construction": 0}, + "flagged_rows": [ {"event": "…", "measure_key": "…", "year": 2026, "level": "head", "key": "income_tax", "flags": ["numerical"], "base_gbp": 0.0, "head_gbp": 0.0} ] + }, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "commit": "<40-hex>", + "path": "requests//results/" + }, + "receipt_sha256": "<64-hex of the canonical receipt without this field>" +} +``` + +### Division of judgement + +How flags arise, so a gate built on them sees the intended events: + +* A year unsupported on one side raises `support` on that year's rows, not + `coverage`. A lost counterpart is `coverage` only where both sides support + the year. +* `construction` means the executed reform dictionaries differ or a premise + check changed its result. A change in the current-law value of a parameter + the construction touches is recorded as context in the row's details, not + as a flag, because it is what a model release is expected to change. +* For a candidate, the base is the latest certified variant, normally on the + same data build; the first certified-variant run diffs against the legacy + populace variant, where the data family itself changes. + +* `integrity.status` is `complete`, `incomplete` or `failed`. `failed` means + a crash, a non-finite aggregate, an identity mismatch after the run started, + or an accounting identity broken in staging. These are structural failures + in the PLAN's sense; W4 blocks promotion on them. +* `diff.flag_counts` and `flagged_rows` describe movements against the base + variant. They carry no verdict. Whether an unexplained material move blocks + promotion, and who reviews it, is W4's qualification policy under d1344. +* Unsupported years are listed per event and are never counted as failures. + +## Agreed points + +1. The candidate wrapper is a git commit on a policyengine.py + `candidate/uk/-` branch whose packaged manifest, pins, + `uv.lock` and TRO certify the pair; that commit becomes the promotion PR + head, so the replayed bytes are the promoted bytes. +2. Receipt delivery is by polling `replay-state`; no callback. +3. W4's qualification receipt records `request_id`, `receipt_sha256` and + `receipt_commit`; nothing of this receipt nests inside W4's execution + receipt. +4. W4 computes `expected_release_bundle` in a subprocess with the three + Hugging Face token variables and `NO_PROXY`/`no_proxy` removed, and the + HTTP(S) proxy variables pointed at a closed local port, exactly as + `uk_replay_variants.isolate_certification_environment` does. (Amended 11 + October and confirmed by W4 the same day, implemented in + policyengine.py#574: removing the tokens alone forces the packaged + certification only for a private data repository; policyengine.py fetches + the release manifest with `requests`, which ignores `HF_HUB_OFFLINE`, so a + public one would still answer.) +5. W4's proposed gate on these fields (pending d1344): `rejected`, `failed`, + integrity `failed` or `incomplete`, and structural, coverage, mapping, + inertness or construction flags fail the replay gate; numerical and + support flags go to review; `no_supported_years` fails; unsupported years + alone never fail. + +## Open points + +1. Expected turnaround: not yet measured for a full sweep. One probe on + 6.2.6/2.125.1/1.58.0 loaded the baseline and computed five annual + aggregates in 10.6s at 2.3GB peak, so a five-event sweep should take + minutes rather than hours on a 16GB runner; the first full run will + replace this estimate. The queue serialises runs, so a backlog adds wait. +2. The receipt schema versioning rule: additive optional fields keep `v1`; + renamed or removed fields bump to `v2`. diff --git a/docs/uk_replay/RECIPE.md b/docs/uk_replay/RECIPE.md index 380aa252..a2741290 100644 --- a/docs/uk_replay/RECIPE.md +++ b/docs/uk_replay/RECIPE.md @@ -407,6 +407,28 @@ receipts; corrected construction explanations are supplied by this recipe and comparison annotations. A current-builder `--check` applies to a rebuilt event registry, rather than regenerating those preserved input receipts. +## Running on another certified variant + +`data/uk/replay_variants.json` lists the certified replay variants; the +default is the populace-2023 world above. `--variant ` selects another +one, validated exactly (see [RELEASE_REPLAY.md](RELEASE_REPLAY.md)): build an +environment from the variant's requirements file, then + +```bash +UV_CACHE_DIR=.venv-uv-cache uv venv --python 3.12 .venv-candidate +UV_CACHE_DIR=.venv-uv-cache uv pip install --python .venv-candidate/bin/python -r docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt +PYTHONPATH=. .venv-candidate/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0 --guards +PYTHONPATH=. .venv-candidate/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0 --workers 2 +PYTHONPATH=. .venv-candidate/bin/python pipeline/stage_uk_event.py --event autumn_budget_2024 --artifact-dir results/uk/replays/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0/autumn_budget_2024 +``` + +The dataset must already be local: the runner finds the certified bytes by +SHA-256 in the Hugging Face cache, or takes `--dataset-file `, and +copies them into `.venv-replay-checks/runtime//data/`. Results +default to `results/uk/replays///`. Years outside the +variant's data years are not run; they are recorded in the manifest and +staged as `unsupported_on_variant`. + ## Commands and compute budget Use the existing `.venv-replay` and `.venv-replay-checks/modal-control` for diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md new file mode 100644 index 00000000..4c8c1f8b --- /dev/null +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -0,0 +1,339 @@ +# Replaying the event corpus on every certified release + +Status: design contract for workstream W6 of the Autumn Budget 2026 CI plan, +11 October 2026. It extends the track 1 replay ([RECIPE.md](RECIPE.md), +[YEARS.md](YEARS.md)) from one hard-pinned world to any number of +**certified replay variants**, and turns each variant's run into a receipt +that release qualification (W4, policyengine.py) can consume before a +model/data pair is promoted. + +Nothing here changes a registered construction. A variant reruns the same +frozen registries; where a construction's premise no longer holds on a +variant, that is reported as a construction revision for review, never +silently repaired. Methodology changes (counterfactual intent, annualisation +semantics, acceptance policy) wait on cos d1344. + +## Why the existing runner cannot rerun + +Read in code at `61c13e38`: + +* `pipeline/uk_engine_registry.py` refuses any engine except `2.89.2`. +* `compute_uk_ab2025.preflight()` and `compute_uk_obr_costings.assert_engine_matches_bundle()` + read `data/uk/certified_bundle.json`, whose only compatible model is + `policyengine-uk ==2.89.2`, and require the installed engine to equal it. +* `modal_uk_event.py` hardcodes `ENGINE_PINS` (`policyengine 5.0.2`, + core `3.27.1`, UK `2.89.2`), the populace Hugging Face cache layout and + the audited HF commit. +* `compute_uk_event.validate_years` allows CY2023–2030 for every run. +* `compare_uk_event.review_diagnosis` attaches ten 2.89.2 findings to every + row with `"engine_version": "2.89.2"` whatever engine produced the row; + `construction_note` states 2.89.2 facts (the £3.15 Class 2 rate, "the pin + still has the 3% surcharge scale") unconditionally. +* `diagnose_uk_event_models.investigate` refuses any engine but 2.89.2. + +The newest certified pair is policyengine.py **6.2.6** (PR #555 and its +version bump `481940ec`): `policyengine-uk 2.125.1`, `policyengine-core +3.33.0`, data `policyengine-uk-data-1.58.0`, default dataset +`enhanced_frs_2024_25` (SHA-256 `5ba399ea…60b6`, 128,369,110 bytes, +`hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0`). + +Measured on that pair on 11 October with the installed wheels (one managed +simulation, offline, no token): + +* The HDF5's `time_period` table holds `["2024"]`; the managed simulation's + `dataset.years` is `[2024, …, 2030]` (`extend_single_year_dataset`, + `end_year=2030`, in `policyengine_uk/data/economic_assumptions.py`). +* **Calculating 2023 does not raise.** `income_tax` for 2023 sums to £0.0bn, + against £275.5bn in 2024: the simulation has no 2023 inputs and returns + defaults. Year support therefore has to be enforced from the data; a + successful calculation proves nothing. +* `pe.uk.managed_microsimulation()` materialises the default dataset at + `./data/` relative to the working directory (policyengine.py + `provenance/dataset_materialization.py`, `DEFAULT_DATA_DIR = Path("./data")`), + reusing a file whose SHA-256 matches and otherwise downloading over HTTPS. + The 5.0.2 HF-cache mirror path does not apply. +* With no Hugging Face token the release-manifest fetch fails and the + wrapper uses its packaged certification (`compatibility_basis + "legacy_compatible_model_package"`, `certified_for_model_version + "2.125.1"`). With a token, `certify_data_release_compatibility` derives a + certification from the remote release manifest instead, which changes + `release_bundle` fields. The 5.0.2 audit recorded the same split + (`OFFLINE_BUNDLE_AUDIT.json`, `release_bundle_field_changes_if_remote_manifest_is_available`). +* Peak resident memory for the baseline load plus five annual aggregates was + 2.3GB; the whole probe took 10.6s. + +## Certified replay variants + +`data/uk/replay_variants.json` lists every variant the corpus may run on. +`data/uk/certified_bundle.json` is unchanged and remains the legacy +variant's identity: AB2025, the OBR-costing runner and every existing +receipt keep reading it byte-for-byte. + +```json +{ + "schema_version": 1, + "default_variant": "", + "variants": [ , ... ] +} +``` + +A variant record: + +| Field | Meaning | +|---|---| +| `variant_id` | `pe_uk_`, recomputed from `identity`; when another identity already holds that slug (same versions, other wheels or numerical environment) the id adds `--`. | +| `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. `data_years`, `layout`, `status` and `executor` are outside it on purpose: they describe or file a run and do not change a computed byte. `data_years` is checked instead against the simulation's own `dataset.years` in every run, and staging recomputes the supported years from the record. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256`: the wheel's digest as published on PyPI, recorded at certification; the receiver verifies each downloaded wheel against it, and the runner, which never sees a wheel, checks only its format. | +| `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | +| `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | +| `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`; `release_bundle.default_dataset_uri` must name the same file. | +| `identity.materialization` | `hf_cache_mirror` (policyengine 5.x: the existing HF-cache plus `data/uk/policyengine-local` mirror) or `policyengine_data_dir` (6.x: a verified copy at `/data/`). | +| `identity.python` | Interpreter minor version, e.g. `3.12`. | +| `identity.requirements_sha256` | The SHA-256 of the requirements freeze below: the numerical environment (numpy, pandas, h5py and the rest) is part of the identity, because it can move numbers. | +| `requirements` | `{path, sha256}`: the exact freeze installed for this variant. Legacy: `docs/uk_replay/requirements.txt`. Others: `docs/uk_replay/requirements/.txt`. | +| `data_years` | Calendar years the population supports: from the dataset's first `time_period` through the engine's extension end, as observed in `simulation.dataset.years`. | +| `data_years_evidence` | The observed `time_period`, `dataset.years` and the engine file/line that sets the extension end. | +| `certification` | Where the pairing is certified: `source` (e.g. `policyengine.py 6.2.6 packaged manifest`), `compatibility_basis`, `certified_for_model_version`, `data_build_id`. | +| `status` | `certified` (a released policyengine.py bundle), `candidate` (a W4 qualification request), or `retired`. Only a certified variant runs, and the default must be certified; a qualification run, which replays its own candidate record, passes `--allow-uncertified`. | +| `qualification_request_sha256` | For candidates, the request the variant was built from. | +| `executor` | `local` (GitHub runner or workstation) and/or `modal`. The populace 1.3GB dataset needs Modal's 32GiB; the 128MB enhanced FRS runs on a 16GB runner. | + +### What "validated exactly" means + +Before any simulation the runner checks all of the following and refuses on +the first failure. The numbering groups them by subject; the code runs record +consistency, the installed environment, certification isolation, dataset +materialisation, bytecode isolation, the engine import and its origins, then +the release-bundle comparison (`compute_uk_event.variant_preflight`; the +legacy variant's `preflight` runs the same checks around its original +managed-release preflight). + +1. **Record consistency (engine-free).** `variant_id` and `identity_sha256` + recompute; `release_bundle.model_version == packages["policyengine-uk"].version`; + `release_bundle.policyengine_version == packages["policyengine"].version`; + `certification.certified_for_model_version == packages["policyengine-uk"].version`; + `release_bundle.certified_data_artifact_sha256 == dataset.sha256`; + `release_bundle.certified_data_build_id == certification.data_build_id`; + `release_bundle.default_dataset_uri` names `dataset.repo_id`, `path` and + `revision`; the requirements file hashes to `requirements.sha256` and pins + exactly the three package versions. For the legacy variant, additionally + `certified_bundle.json`'s `revision`, `sha256`, `size_bytes`, `repo_id` + and `compatible_model_packages` agree with the record. +2. **Installed environment.** Installed versions of the three packages equal + the record; every installed file matches its `RECORD` hash and the + normalised `RECORD` hashes to `record_sha256`; nothing sits in the three + packages' directories that the package's own `RECORD` does not list: + policyengine-core loads every file in a parameter directory and every + module in a variable directory, recursing into subdirectories + (`__pycache__` included) and following symlinked ones, so an unlisted + file, any symlink, and anything but `*.pyc` inside a `__pycache__` are + refused, and another distribution's `RECORD` does not count; every + `*.pth` startup file beside the packages belongs to a distribution pinned + in the requirements freeze or is the virtual environment's own + `_virtualenv.pth`; every distribution pinned in the requirements freeze + is installed at its pinned version (extra distributions, such as the + receipt tools, are allowed); the installed packaged manifest hashes to + `packaged_bundle_manifest_sha256`; the interpreter minor version matches. + The legacy variant is checked the same way. + + What this does not bind: the files of pinned distributions other than the + three packages are checked by version, not by hash, and an extra + distribution's importable modules are not inspected. The check catches a + drifted or mixed environment; it is not a defence against someone who + can edit the environment at will. +3. **What is imported.** Before the engine is imported, bytecode caching is + turned off and `sys.pycache_prefix` points at an empty directory, so + Python compiles every engine module from the verified sources and never + reads a `__pycache__` beside them; spawned year workers inherit both + settings. After import, each package's `__file__` must be its verified + distribution's own `__init__.py`, not a same-named tree earlier on + `sys.path`; each year worker checks this too. +4. **Certification path.** Before importing the engine the runner removes + `HUGGING_FACE_TOKEN`, `HF_TOKEN` and `HUGGINGFACE_HUB_TOKEN` and the + `NO_PROXY` exceptions from the environment, forces Hugging Face offline + mode, and points the HTTP(S) proxy variables at a closed local port. + policyengine.py 6.x fetches the data release manifest with `requests` + (`provenance/manifest.py`, `get_data_release_manifest`), which ignores + `HF_HUB_OFFLINE` but reads those proxy variables, so the fetch fails at + once for a private or a public data repository, and + `certify_data_release_compatibility` falls back to the wrapper's packaged + certification. Then + `dict(pe.uk.uk_latest.release_bundle) == identity.release_bundle`, exactly. + The legacy variant runs under the same isolation and the same exact + comparison, in addition to its original check against + `data/uk/certified_bundle.json`. +5. **Dataset bytes.** The materialised file hashes to `dataset.sha256` and + has `size_bytes`, before construction and after the last aggregate + (existing `run_managed_simulation` checks), and the simulation's reported + runtime dataset source is that file. +6. **Year support.** Requested years must lie in `data_years`, and inside + every simulation `year in simulation.dataset.years` or the run stops; on a + non-legacy variant a dataset that does not state its years stops the run + too (the legacy check skips it, as it always has). Registry years outside `data_years` are not + requested by default; they are reported as unsupported (below). + +A model/data combination is **incompatible** when no certified variant +records it. There is no flag that relaxes a check: running a newer engine +means adding a variant whose certification names it. + +## Unsupported years + +Each event registry's `calendar_years` is the construction's window; a +variant runs `calendar_years ∩ data_years`. The rest are **unsupported on the +variant**, with a reason (`before_data_year` or `after_extension_end`). +`RUN_MANIFEST.json` records `supported_years`, `unsupported_years` and +`full_supported_grid_complete`; `full_event_complete` keeps its meaning +(every registry year). Staging does not take those fields on trust: it loads +the manifest's `variant_id` from the variants document, requires the +manifest's identity digest, `data_years`, `supported_years` and +`unsupported_years` to be the ones that record gives, and requires every +artifact's variant fields to match (a legacy manifest and its artifacts carry +none). `--resume` reads receipts only for supported years and refuses any +receipt that names an unsupported one. Staging marks a source row whose year is unsupported +as `status: "unsupported_on_variant"` with no numerical counterpart, kept in +the row universe and counted separately from `not_computed`. On +`pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0`, CY2023 is unsupported, so +Spring Budget 2023, Autumn Statement 2023 and Spring Budget 2024 lose their +FY2023–24 counterparts; that loss is shown, never hidden. + +## Result namespaces + +* Legacy results stay where they are, in `results/uk/events//`. +* Every other variant writes to `results/uk/replays//`: + `/` (the same files as a legacy event directory), `SUMMARY.md`, + `SUPPORT.json`, `CONSTRUCTION_PREMISES.json`, `MODEL_DIAGNOSTICS.json`, + `REQUEST.json` (the qualification request the run answers), and finally + `NAMESPACE_MANIFEST.json` and `REPLAY_RECEIPT.json`. +* `data/uk/replay_variants.json` is the index: each record's `result_root` + names its namespace (legacy: `results/uk/events`) and its `status`. A + candidate's record and results live on the `replay-state` branch under + `requests//results/` until it is certified. +* A namespace with `REPLAY_RECEIPT.json` is **sealed**: the runner refuses + to write there (compute, staging, comparison and the summary all check). + A rerun of a sealed variant writes to an explicit separate output + directory, and its bytes against the sealed run are a determinism check. + The receiver (`pipeline/uk_replay_qualify.py`, which arrives with the + receiver change, PR #163) writes the seal itself: + `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under the + namespace (and of its release diff) and the receipt binds that manifest's + digest. The same change adds the CI job that fails when a sealed byte + changes and replays one real measure-year; until it lands, neither exists. +* Release differences live in `results/uk/replays/diffs/__/`. + +## Version-scoped diagnoses + +Every diagnosis and construction note carries the engine version it was +established on. A comparison row attaches a diagnosis as current only when +the artifact's `engine_version` is in that set. On any other engine the row +carries `prior_diagnoses` with `established_on` and `status: +"not_reverified"` until `diagnose_uk_event_models.py` has been rerun on that +engine, whose per-case predicate records `persists`, `resolved` or +`inconclusive`. The legacy event outputs remain byte-identical. + +## Construction premises + +`pipeline/audit_uk_event_premises.py` records, for each executable measure, +year and parameter path the construction touches, on a given engine: +current-law value at the annual lookup (1 January of the year), the raw +value at 30 April, the executed baseline-world and reform-world values, and +these checks: + +| Check | Fails when | +|---|---| +| `forward_delta_changes_current_law` | A forward delta equals current law in every in-force supported year (inert by construction). | +| `reversal_differs_from_current_law` | A reversal modifier equals current law in every in-force supported year. | +| `derived_counterfactual_reproduces` | Re-deriving `counterfactual_derivation` with this engine's parameters differs from the frozen registry values. | +| `annual_value_matches_30_april_sample` | The processed annual value differs from the raw 30 April value (the engine blends or annualises differently from the 2.89.2 note). | + +A failed check is a **construction revision candidate**, reported in the +release diff's construction flags and summarised in +`docs/uk_replay/CONSTRUCTION_REVISIONS.md` with a proposed revision. Running +a revised construction is a methodology change and waits on d1344. + +Observed on 2.125.1 on 11 October (parameter tree only): + +* `gov.contrib.labour.private_school_vat` is still 0 in every year, so the + AB2024 VAT forward delta still changes current law. +* The SDLT additional-dwelling scale is still 3/5/8/13/15%, so the AB2024 + forward 2pp delta still applies. +* CGT `basic_rate` at the 2024 annual lookup is 0.1346. In 2.125.1, + `parameters/gov/hmrc/cgt/basic_rate.yaml` dates the 18% rate from + 2024-10-30 and sets `metadata.fiscal_year_blend: true`, and + `utils/parameters.convert_to_fiscal_year_parameters` day-weights such + parameters over 6 April–5 April (`fiscal_year_average`) instead of + sampling 30 April: 0.10 × 207/365 + 0.18 × 158/365 = 0.1346. The AB2024 + note assumes 2.89.2's raw 2025-04-06 date and 30 April sampling. The + reversal, first selected in January 2025, leaves the blended 2024 increase + in both worlds, so CY2024 shows no effect although current law has one. +* The UC childcare cap path (2024 £1,014.63 for one child) and the benefit + CPI path feed the SB2023 and SS2025 frozen counterfactuals, which were + derived from 2.89.2's values. +* The Class 2 flat rate for 2023 is £3.45, so the "pin uses £3.15" note is a + 2.89.2 fact only. + +## Release differences + +`pipeline/diff_uk_event_runs.py --base --head ` +compares two complete namespaces, engine-free, after re-verifying each side's +staging and comparison receipts. Keys: + +* source row: `(event, source_row_id)`; +* measure-year total: `(event, measure_key, year)`; +* PE head: `(event, measure_key, year, variable)` from artifact `head_effects`. + +Flags are descriptive; none is a pass or a failure. A flag names a move to +investigate, and the existing signed PE/OBR bins are reported unchanged as +bin transitions. + +| Flag | Raised when | +|---|---| +| `numerical` | Both sides numeric and `abs(head − base) > max(£1,000,000, 0.01 × max(abs(head), abs(base)))`. Strictly greater. | +| `structural` | Sign reversal or a zero crossing (zero means `abs(x) ≤ £0.50`, accounting roundoff); artifact schema or orientation changed; non-finite value. Regardless of size. | +| `coverage` | A counterpart, artifact pair or source row present on one side only, in a year both sides support. | +| `mapping` | A row's `head_variables`, an artifact's `head_channels` or `head_definitions`, or the registry SHA-256 differs. | +| `support` | A year supported on one side only. | +| `inertness` | `inert_construction` on one side only, or identical worlds in an in-force year on one side only. | +| `construction` | Executed `baseline_world`/`reform_world` reform dicts differ, or a premise check's result differs. A changed current-law value of a touched parameter is recorded as context on the row, not flagged: it is what a model release is expected to change. | + +Identity changes (wrapper, engine, core, data build, dataset digest) are +recorded once per diff; they explain moves and are not flags on rows. + +Outputs: `DIFF.json` (canonical, sorted), `DIFF.md`, and one issue packet per +flagged `(event, measure, year)` under `packets/`. A packet holds the source +cells (table, column, row number, source-row id, OBR value), both execution +identities, registry and artifact digests, the executed reform dicts, the +affected heads with base/head values, focused rerun commands for each +variant, the statutory sources the registry already cites, and empty +`minimal_case` and `statutory_evidence` sections for the investigating lane. +Packets are evidence for the coordinating session, which files upstream +issues; this pipeline files none. + +## Qualification requests and receipts + +See [QUALIFICATION_INTERFACE.md](QUALIFICATION_INTERFACE.md) for the request +W4 sends before promotion, the receipt returned, and the persistent queue. + +## Invariants + +Each has deterministic boundary cases and Hypothesis properties in +`tests/`: + +* **Exact variant validation.** For any record and environment differing from + it in one field, validation refuses; for the matching pair it accepts. +* **Support honesty.** No artifact exists for a year outside the variant's + `data_years`; every registry source row appears exactly once in staging, + with unsupported years as `unsupported_on_variant`. +* **Replay determinism.** For the same complete execution fingerprint + (variant identity, registry bytes, measure, year), numerical artifact bytes + are identical; timings stay in `RUN_LOG.json`. +* **Legacy preservation.** Legacy staging, comparison and summary outputs + regenerate byte-identically from the committed artifacts + (`tests/test_uk_replay_legacy.py` reruns all five events and compares + bytes), and no committed legacy artifact, manifest or receipt carries a + variant field. +* **Diff flags.** `diff(a, a)` has no row flags. Flag sets are symmetric under + swapping sides (with signs reversed). No numerical flag at or below the + threshold; one strictly above it. Row order and file order do not change + output bytes. +* **Sealed namespaces.** Every byte under a sealed namespace matches its + receipt. diff --git a/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt b/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt new file mode 100644 index 00000000..8cf183d0 --- /dev/null +++ b/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt @@ -0,0 +1,84 @@ +annotated-types==0.8.0 +anyio==4.15.1 +asttokens==3.0.2 +attrs==26.1.0 +blosc2==4.14.1 +certifi==2026.7.22 +charset-normalizer==3.5.2 +click==8.5.0 +decorator==5.3.1 +dpath==2.2.0 +executing==2.3.0 +filelock==4.1.0 +fsspec==2026.9.0 +h11==0.16.0 +h2==4.4.1 +h5py==3.16.0 +hf-xet==1.7.0 +hpack==4.2.0 +httpcore==1.0.9 +httpcore2==2.13.1 +httpx==0.28.1 +httpx2==2.13.1 +huggingface-hub==2.2.0 +hyperframe==6.1.0 +hypothesis==6.169.0 +idna==3.20 +iniconfig==2.3.1 +ipython==8.39.0 +jedi==0.20.0 +jinja2==3.1.6 +jsonpickle==4.1.3 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +markdown-it-py==4.2.0 +markupsafe==3.0.4 +matplotlib-inline==0.2.2 +mdurl==0.1.2 +microdf-python==1.3.10 +msgpack==1.2.3 +ndindex==1.10.1 +networkx==3.7 +numexpr==2.14.2 +numpy==2.5.4 +packaging==26.3 +pandas==3.0.6 +parso==0.8.7 +pexpect==4.9.0 +plotly==5.24.1 +pluggy==1.6.0 +policyengine==6.2.6 +policyengine-core==3.33.0 +policyengine-uk==2.125.1 +prompt-toolkit==3.0.53 +psutil==6.1.1 +ptyprocess==0.7.0 +pure-eval==0.2.4 +py-cpuinfo==9.0.0 +pydantic==2.14.0 +pydantic-core==2.50.0 +pygments==2.21.0 +pypdf==6.20.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +pyvis==0.3.2 +pyyaml==6.0.3 +referencing==0.37.0 +requests==2.34.2 +rich==15.0.0 +rpds-py==2026.9.1 +six==1.17.0 +sortedcontainers==2.4.0 +stack-data==0.6.3 +standard-imghdr==3.13.0 +tables==3.11.1 +tenacity==9.2.1 +threadpoolctl==3.7.0 +tqdm==4.70.1 +traitlets==5.16.1 +truststore==0.10.4 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.8.0 +wcwidth==0.9.2 +wheel==0.48.0 diff --git a/docs/uk_replay/schemas/qualification_request.v1.json b/docs/uk_replay/schemas/qualification_request.v1.json new file mode 100644 index 00000000..5865ca3e --- /dev/null +++ b/docs/uk_replay/schemas/qualification_request.v1.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/qualification_request.v1.json", + "title": "UK fiscal-event replay qualification request, v1", + "description": "Sent by policyengine.py release qualification (W4) to policyengine-scorecard (W6). request_id is the SHA-256 of the RFC 8785 (JCS) bytes of the whole request; it is never a field. Additive optional fields keep v1; renamed or removed fields bump the version.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "kind", "attempt", "candidate", "compare_to", "requested_by"], + "properties": { + "schema": {"const": "policyengine.uk-replay-qualification-request/v1"}, + "kind": { + "description": "candidate: a pair not yet promoted; released_bundle: a published policyengine.py release.", + "enum": ["candidate", "released_bundle"] + }, + "attempt": { + "description": "A retry of a failed request is a new request with a higher attempt.", + "type": "integer", + "minimum": 1 + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": ["policyengine", "policyengine_uk", "policyengine_core", "data", "expected_release_bundle"], + "properties": { + "policyengine": { + "type": "object", + "additionalProperties": false, + "required": ["version", "source", "packaged_bundle_manifest_sha256"], + "properties": { + "version": {"$ref": "#/$defs/version"}, + "source": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["git"], + "properties": { + "git": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "commit"], + "properties": { + "repository": {"const": "PolicyEngine/policyengine.py"}, + "commit": {"$ref": "#/$defs/sha1"} + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["pypi"], + "properties": { + "pypi": { + "type": "object", + "additionalProperties": false, + "required": ["wheel_sha256"], + "properties": {"wheel_sha256": {"$ref": "#/$defs/sha256"}} + } + } + } + ] + }, + "packaged_bundle_manifest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "policyengine_uk": {"$ref": "#/$defs/pypi_package"}, + "policyengine_core": {"$ref": "#/$defs/pypi_package"}, + "data": { + "type": "object", + "additionalProperties": false, + "required": ["build_id", "dataset", "repo_id", "repo_type", "revision", "path", "sha256", "size_bytes", "release_manifest_sha256"], + "properties": { + "build_id": {"type": "string", "minLength": 1}, + "dataset": {"type": "string", "minLength": 1}, + "repo_id": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "repo_type": {"enum": ["model", "dataset"]}, + "revision": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1, "pattern": "^(?!.*(^|/)\\.\\.(/|$))[^/][^\\\\]*$"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "size_bytes": {"type": "integer", "minimum": 1}, + "release_manifest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "expected_release_bundle": { + "description": "The exact dict(pe.uk.uk_latest.release_bundle) under packaged certification: computed in a subprocess with HF_TOKEN, HUGGING_FACE_TOKEN, HUGGINGFACE_HUB_TOKEN, NO_PROXY and no_proxy removed and HTTP_PROXY, HTTPS_PROXY, ALL_PROXY (and their lowercase forms) set to http://127.0.0.1:9, so the wrapper's release-manifest request cannot reach the network and the packaged certification is used for a private or public data repository (pipeline/uk_replay_variants.py, isolate_certification_environment).", + "type": "object", + "required": ["policyengine_version", "model_version", "certified_data_build_id", "certified_data_artifact_sha256", "default_dataset_uri"], + "additionalProperties": {"type": ["string", "null"]}, + "properties": { + "policyengine_version": {"$ref": "#/$defs/version"}, + "model_version": {"$ref": "#/$defs/version"}, + "certified_data_build_id": {"type": "string", "minLength": 1}, + "certified_data_artifact_sha256": {"$ref": "#/$defs/sha256"}, + "default_dataset_uri": {"type": "string", "pattern": "^hf://"} + } + } + } + }, + "compare_to": { + "description": "latest_certified, or an explicit scorecard variant_id.", + "type": "string", + "minLength": 1 + }, + "requested_by": { + "type": "object", + "additionalProperties": true, + "required": ["repository", "commit"], + "properties": { + "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "workflow_run_id": {"type": "integer", "minimum": 1}, + "run_attempt": {"type": "integer", "minimum": 1}, + "commit": {"$ref": "#/$defs/sha1"} + } + } + }, + "allOf": [ + { + "if": {"properties": {"kind": {"const": "released_bundle"}}}, + "then": { + "properties": { + "candidate": { + "properties": { + "policyengine": {"properties": {"source": {"required": ["pypi"]}}} + } + } + } + } + } + ], + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "sha1": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "version": {"type": "string", "pattern": "^[0-9]+(\\.[0-9]+)*([a-z0-9.+-]*)$"}, + "pypi_package": { + "type": "object", + "additionalProperties": false, + "required": ["version", "wheel_sha256"], + "properties": { + "version": {"$ref": "#/$defs/version"}, + "wheel_sha256": {"$ref": "#/$defs/sha256"} + } + } + } +} diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json new file mode 100644 index 00000000..a9d52614 --- /dev/null +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -0,0 +1,638 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/replay_receipt.v1.json", + "title": "UK fiscal-event replay receipt, v1", + "description": "Returned by policyengine-scorecard for one qualification request. receipt_sha256 is the SHA-256 of the RFC 8785 (JCS) bytes of the receipt without that field. Flags describe movements against the base variant; they carry no verdict.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "request_id", + "status", + "rejection", + "variant_id", + "variant_identity_sha256", + "execution", + "corpus", + "integrity", + "diff", + "results", + "receipt_sha256" + ], + "properties": { + "schema": { + "const": "policyengine.uk-replay-receipt/v1" + }, + "request_id": { + "$ref": "#/$defs/sha256" + }, + "status": { + "enum": [ + "complete", + "rejected", + "failed" + ] + }, + "rejection": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "detail" + ], + "properties": { + "code": { + "enum": [ + "schema_invalid", + "artifact_hash_mismatch", + "release_bundle_mismatch", + "incompatible_pair", + "data_access_failed", + "no_supported_years" + ] + }, + "detail": { + "type": "string", + "minLength": 1 + } + } + } + ] + }, + "variant_id": { + "type": [ + "string", + "null" + ], + "minLength": 1 + }, + "variant_identity_sha256": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/sha256" + } + ] + }, + "execution": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "packages", + "packaged_bundle_manifest_sha256", + "release_bundle", + "dataset_sha256", + "requirements_sha256", + "python", + "platform", + "runner" + ], + "properties": { + "packages": { + "type": "object", + "additionalProperties": false, + "required": [ + "policyengine", + "policyengine-uk", + "policyengine-core" + ], + "properties": { + "policyengine": { + "$ref": "#/$defs/installed_package" + }, + "policyengine-uk": { + "$ref": "#/$defs/installed_package" + }, + "policyengine-core": { + "$ref": "#/$defs/installed_package" + } + } + }, + "packaged_bundle_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "release_bundle": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "null" + ] + } + }, + "dataset_sha256": { + "$ref": "#/$defs/sha256" + }, + "requirements_sha256": { + "$ref": "#/$defs/sha256" + }, + "python": { + "type": "string", + "pattern": "^3\\.[0-9]+\\.[0-9]+$" + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "runner": { + "type": "string", + "minLength": 1 + } + } + } + ] + }, + "corpus": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "corpus_version", + "events" + ], + "properties": { + "corpus_version": { + "type": "string", + "minLength": 1 + }, + "events": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/$defs/event_result" + } + } + } + } + ] + }, + "integrity": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "failures" + ], + "properties": { + "status": { + "description": "complete: every supported measure-year ran and every receipt verified. incomplete: some supported pairs are missing. failed: a crash, a non-finite value, an identity mismatch after the run started, or a broken accounting identity.", + "enum": [ + "complete", + "incomplete", + "failed", + "not_run" + ] + }, + "failures": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "detail" + ], + "properties": { + "code": { + "type": "string", + "minLength": 1 + }, + "detail": { + "type": "string", + "minLength": 1 + }, + "event": { + "type": "string" + }, + "measure_key": { + "type": "string" + }, + "year": { + "type": "integer" + } + } + } + } + } + }, + "diff": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "base_variant_id", + "diff_sha256", + "flag_counts", + "flagged_rows" + ], + "properties": { + "base_variant_id": { + "type": "string", + "minLength": 1 + }, + "diff_sha256": { + "$ref": "#/$defs/sha256" + }, + "flag_counts": { + "type": "object", + "additionalProperties": false, + "required": [ + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction" + ], + "properties": { + "numerical": { + "type": "integer", + "minimum": 0 + }, + "structural": { + "type": "integer", + "minimum": 0 + }, + "coverage": { + "type": "integer", + "minimum": 0 + }, + "mapping": { + "type": "integer", + "minimum": 0 + }, + "support": { + "type": "integer", + "minimum": 0 + }, + "inertness": { + "type": "integer", + "minimum": 0 + }, + "construction": { + "type": "integer", + "minimum": 0 + } + } + }, + "flagged_rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "measure_key", + "year", + "level", + "key", + "flags", + "base_gbp", + "head_gbp" + ], + "properties": { + "event": { + "type": "string", + "minLength": 1 + }, + "measure_key": { + "type": "string", + "minLength": 1 + }, + "year": { + "type": "integer" + }, + "level": { + "enum": [ + "source_row", + "measure_year", + "head" + ] + }, + "key": { + "type": "string", + "minLength": 1 + }, + "flags": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction" + ] + } + }, + "base_gbp": { + "type": [ + "number", + "null" + ] + }, + "head_gbp": { + "type": [ + "number", + "null" + ] + } + } + } + } + } + } + ] + }, + "results": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "ref", + "path" + ], + "properties": { + "repository": { + "const": "PolicyEngine/policyengine-scorecard" + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "commit": { + "$ref": "#/$defs/sha1" + }, + "path": { + "type": "string", + "minLength": 1 + }, + "manifest_sha256": { + "description": "SHA-256 of NAMESPACE_MANIFEST.json, which lists the SHA-256 of every file in the sealed result namespace except itself and the receipt.", + "$ref": "#/$defs/sha256" + } + } + }, + "receipt_sha256": { + "$ref": "#/$defs/sha256" + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "object" + }, + "corpus": { + "type": "null" + }, + "diff": { + "type": "null" + }, + "variant_id": { + "type": "null" + }, + "variant_identity_sha256": { + "type": "null" + }, + "execution": { + "type": "null" + }, + "integrity": { + "properties": { + "status": { + "const": "not_run" + } + } + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "null" + }, + "variant_id": { + "type": "string" + }, + "execution": { + "type": "object" + }, + "variant_identity_sha256": { + "$ref": "#/$defs/sha256" + }, + "results": { + "required": [ + "manifest_sha256" + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "integrity": { + "properties": { + "status": { + "enum": [ + "complete", + "incomplete" + ] + } + } + }, + "corpus": { + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "failed" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "null" + }, + "integrity": { + "properties": { + "status": { + "const": "failed" + } + } + } + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "installed_package": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "record_sha256" + ], + "properties": { + "version": { + "type": "string", + "minLength": 1 + }, + "record_sha256": { + "$ref": "#/$defs/sha256" + }, + "wheel_sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "event_result": { + "type": "object", + "additionalProperties": false, + "required": [ + "registry_sha256", + "supported_years", + "unsupported_years", + "run_manifest_sha256", + "staging_manifest_sha256", + "comparison_provenance_sha256", + "artifacts", + "full_supported_grid_complete", + "inert_measure_years" + ], + "properties": { + "registry_sha256": { + "$ref": "#/$defs/sha256" + }, + "supported_years": { + "type": "array", + "items": { + "type": "integer" + }, + "uniqueItems": true + }, + "unsupported_years": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "year", + "reason" + ], + "properties": { + "year": { + "type": "integer" + }, + "reason": { + "enum": [ + "before_data_year", + "after_extension_end" + ] + } + } + } + }, + "run_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "staging_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "comparison_provenance_sha256": { + "$ref": "#/$defs/sha256" + }, + "artifacts": { + "type": "integer", + "minimum": 0 + }, + "full_supported_grid_complete": { + "type": "boolean" + }, + "inert_measure_years": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "measure_key", + "year" + ], + "properties": { + "measure_key": { + "type": "string" + }, + "year": { + "type": "integer" + } + } + } + } + } + } + } +} diff --git a/docs/uk_replay/schemas/replay_status.v1.json b/docs/uk_replay/schemas/replay_status.v1.json new file mode 100644 index 00000000..3694ca4e --- /dev/null +++ b/docs/uk_replay/schemas/replay_status.v1.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/replay_status.v1.json", + "title": "UK fiscal-event replay request status, v1", + "description": "replay-state:requests//STATUS.json. A terminal status names the receipt's digest and the replay-state commit that added REPLAY_RECEIPT.json, so one contents read binds the status to the receipt bytes.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "request_id", "status", "history"], + "properties": { + "schema": {"const": "policyengine.uk-replay-status/v1"}, + "request_id": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "status": {"enum": ["queued", "running", "complete", "rejected", "failed"]}, + "history": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["status", "at", "run_url"], + "properties": { + "status": {"enum": ["queued", "running", "complete", "rejected", "failed"]}, + "at": {"type": "string", "format": "date-time"}, + "run_url": {"type": "string", "pattern": "^https://github\\.com/PolicyEngine/policyengine-scorecard/actions/runs/[0-9]+"} + } + } + }, + "receipt_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "receipt_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"enum": ["complete", "rejected", "failed"]}}}, + "then": {"required": ["receipt_sha256", "receipt_commit"]} + } + ] +} diff --git a/pipeline/compare_uk_event.py b/pipeline/compare_uk_event.py index c5bf7108..e7461927 100644 --- a/pipeline/compare_uk_event.py +++ b/pipeline/compare_uk_event.py @@ -25,6 +25,7 @@ sys.path.insert(0, str(ROOT)) from pipeline.build_uk_event_registry import _sum as source_gbp_sum +from pipeline import uk_replay_variants as variants from pipeline.compare_uk_obr_costings import atomic_write_bytes, ratio_and_bin AXES_PATH = ROOT / "data" / "uk" / "obr_divergence_axes.json" @@ -1053,6 +1054,7 @@ def write_comparison( ) if registry.get("event_slug") != event: raise EventComparisonError("registry event differs from --event") + variants.assert_not_sealed(output_dir) output_dir.mkdir(parents=True, exist_ok=True) staging_manifest = _json(staging_manifest_path) replay_grid = { @@ -1387,6 +1389,7 @@ def write_summary(events_root: Path, *, artifact_root: Path = ROOT) -> None: (artifact_root / "data" / "uk" / "events").glob("*_measures.json") ) } + variants.assert_not_sealed(events_root) events_root.mkdir(parents=True, exist_ok=True) atomic_write_bytes( events_root / "SUMMARY.md", diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index f5d02e39..b7ecf664 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -25,6 +25,7 @@ from pipeline import build_uk_event_registry as registry_builder from pipeline import compute_uk_ab2025 as worlds from pipeline import compute_uk_obr_costings as fiscal +from pipeline import uk_replay_variants as variants ROOT = worlds.ROOT EVENT_PATTERN = re.compile(r"^[a-z0-9]+(?:_[a-z0-9]+)*$") @@ -189,6 +190,7 @@ def build_artifact( certified: dict, registry_sha256: str, event: str, + variant_fields: dict | None = None, ) -> dict: """Numerical facts only: identical inputs produce identical artifact bytes. @@ -275,13 +277,23 @@ def build_artifact( } if reversal: artifact["literal_reversal_minus_certified_gbp"] = literal + # Legacy artifacts carry no variant fields, so their bytes are unchanged. + artifact.update(variant_fields or {}) return artifact -def preflight() -> dict: - """The committed pin and the managed release must agree before any run.""" +def preflight(variant: dict | None = None) -> dict: + """The committed pin and the managed release must agree before any run. + + Given the legacy variant record, the run is also isolated like any other + variant's (packaged certification, engine compiled from its verified + sources and imported from its own distributions) and the release bundle + must equal the record's exactly.""" fiscal.configure_offline() worlds.configure_offline() + if variant is not None: + variants.isolate_certification_environment() + variants.isolate_bytecode(variants.RUNTIME_ROOT / variant["variant_id"]) committed = worlds.preflight() managed = fiscal.preflight_certified_dataset() if ( @@ -289,9 +301,71 @@ def preflight() -> dict: or fiscal.data_bundle_id(managed["release_bundle"]) != committed["revision"] ): raise RuntimeError("managed release differs from data/uk/certified_bundle.json") + if variant is not None: + import_engine() + variants.verify_release_bundle(variant, managed["release_bundle"]) + return {**managed, "check_import_origins": True} return managed +def import_engine() -> None: + """Import all three engine packages and check where each came from.""" + import policyengine # noqa: F401 + import policyengine_core # noqa: F401 + import policyengine_uk # noqa: F401 + + variants.verify_import_origins() + + +def variant_identity_fields(variant: dict) -> dict: + """Artifact and receipt identity of a non-legacy variant; legacy: none.""" + if variants.is_legacy(variant): + return {} + return { + "variant_id": variant["variant_id"], + "variant_identity_sha256": variant["identity_sha256"], + } + + +def check_variant_environment(variant: dict) -> dict: + """Engine-free: the record is certified and the installed wheels are its own.""" + variants.validate_record(variant) + return variants.verify_installed(variant) + + +def variant_preflight(variant: dict, *, dataset_source: Path | None = None) -> dict: + """Validate a non-legacy variant exactly before any simulation. + + Order matters: the certification environment is isolated before the + wrapper is imported, and the working directory moves to the runtime + directory whose ./data holds the verified dataset, where policyengine.py + 6.x materialises its default dataset. Spawned year workers inherit both. + """ + installed = check_variant_environment(variant) + variants.isolate_certification_environment() + fiscal.configure_offline() + worlds.configure_offline() + runtime_root = (variants.RUNTIME_ROOT / variant["variant_id"]).resolve() + dataset = variants.materialize_dataset(variant, runtime_root, source=dataset_source) + os.chdir(runtime_root) + variants.isolate_bytecode(runtime_root) + import policyengine as pe + + import_engine() + release_bundle = dict(pe.uk.uk_latest.release_bundle) + variants.verify_release_bundle(variant, release_bundle) + return { + "release_bundle": release_bundle, + "sha256": variant["identity"]["dataset"]["sha256"], + "runtime_dataset_source": str(dataset), + "installed": installed, + # A year outside the population's data years must fail, not + # calculate defaults (compute_uk_obr_costings.run_managed_simulation). + "require_dataset_years": True, + "check_import_origins": True, + } + + def run_sim(year: int, variables: list[str], reform: dict | None, pre: dict) -> dict: reform = annual_reform(reform) scenario = ( @@ -307,6 +381,7 @@ def run_sim(year: int, variables: list[str], reform: dict | None, pre: dict) -> runtime_dataset_source=Path(pre["runtime_dataset_source"]), expected_dataset_sha256=pre["sha256"], include_engine_provenance=True, + require_dataset_years=pre.get("require_dataset_years", False), **scenario_kwargs, ) fiscal.assert_managed_bundle(result["policyengine_bundle"], pre["release_bundle"]) @@ -323,6 +398,9 @@ def run_year_job(job: dict) -> dict: flush=True, ) baseline = run_sim(year, job["variables"], None, pre) + # This spawned worker imported the engine itself: check its origin too. + if pre.get("check_import_origins"): + variants.verify_import_origins() timings = [ { "world": "certified_baseline", @@ -365,6 +443,7 @@ def run_year_job(job: dict) -> dict: certified=baseline, registry_sha256=job["registry_sha256"], event=job["event"], + variant_fields=job.get("variant_fields"), ) path = job["output_dir"] / f"{key}_{year}.json" fiscal.atomic_write_bytes(path, canonical_bytes(artifact)) @@ -397,6 +476,7 @@ def progress_receipt(job: dict, artifacts: dict[str, str]) -> dict: "data_bundle": fiscal.data_bundle_id(job["pre"]["release_bundle"]), "engine_version": fiscal.package_version("policyengine-uk"), **runtime_versions(), + **(job.get("variant_fields") or {}), "artifacts": artifacts, } @@ -408,8 +488,10 @@ def retained_artifact_digests( years: list[int], registry_sha256: str, pre: dict, + variant_fields: dict | None = None, ) -> dict[str, str]: """Read only SHA-bound completed outputs; unreceipted files are ignored.""" + variant_fields = variant_fields or {} receipts = [] manifest_path = output_dir / "RUN_MANIFEST.json" if manifest_path.exists(): @@ -418,6 +500,7 @@ def retained_artifact_digests( manifest.get("event") == event and manifest.get("registry_sha256") == registry_sha256 and manifest.get("certified_dataset_sha256") == pre["sha256"] + and all(manifest.get(k) == v for k, v in variant_fields.items()) ): receipts.append(manifest) for year in years: @@ -431,6 +514,7 @@ def retained_artifact_digests( "year": year, "registry_sha256": registry_sha256, "pre": pre, + "variant_fields": variant_fields, }, {}, ) @@ -474,6 +558,38 @@ def validate_years(requested: list[int], registry: dict) -> list[int]: return years +def validate_variant_years( + requested: list[int], registry: dict, variant: dict +) -> list[int]: + """Registry years the variant's population supports. + + A year before the dataset's first time period computes without error but + from absent inputs (a 2023 income tax of zero on a 2024 dataset), so an + unsupported year is refused, never run; by default it is reported. + """ + allowed = registry.get("calendar_years") + if not allowed: + raise ValueError("registry must record the supported calendar_years") + supported, unsupported = variants.supported_years(variant, allowed) + if not supported: + raise ValueError( + f"{variant['variant_id']} supports none of this event's years {allowed}" + ) + years = sorted(set(requested or supported)) + if set(years) - set(allowed): + raise ValueError( + f"years {years} outside this event's documented support {allowed}" + ) + refused = sorted(set(years) - set(supported)) + if refused: + raise ValueError( + f"years {refused} are outside {variant['variant_id']}'s data years " + f"{variant['data_years'][0]}-{variant['data_years'][-1]}: " + f"{[row for row in unsupported if row['year'] in refused]}" + ) + return years + + def verify_registry_identity(path: Path, expected_sha256: str) -> None: if fiscal.sha256_file(path) != expected_sha256: raise ValueError( @@ -510,6 +626,26 @@ def main(argv: list[str] | None = None) -> int: worlds.configure_offline() ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--event", required=True) + ap.add_argument( + "--variant", + help="certified replay variant (data/uk/replay_variants.json); default: the file's default", + ) + ap.add_argument( + "--variants-file", + type=Path, + default=variants.VARIANTS_PATH, + help="variant records to select from (a qualification run adds its candidate)", + ) + ap.add_argument( + "--dataset-file", + type=Path, + help="a local file holding the variant's certified dataset bytes (verified by SHA-256)", + ) + ap.add_argument( + "--allow-uncertified", + action="store_true", + help="replay a candidate variant (qualification runs only)", + ) ap.add_argument("--registry", type=Path) ap.add_argument("--output-dir", type=Path) ap.add_argument("--measures", nargs="*") @@ -525,9 +661,26 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) + variant = variants.get_variant( + args.variant, + path=args.variants_file.resolve(), + allow_uncertified=args.allow_uncertified, + ) + legacy = variants.is_legacy(variant) + if legacy and args.dataset_file is not None: + raise ValueError( + "--dataset-file applies to a variant that materialises its dataset; " + "the legacy variant reads its pinned Hugging Face cache" + ) + variant_fields = variant_identity_fields(variant) default_registry, default_output = event_paths(args.event) - registry_path = args.registry or default_registry + if not legacy: + default_output = variants.event_output_dir(variant, args.event) + # Absolute before a variant preflight moves into its runtime directory. + registry_path = (args.registry or default_registry).resolve() output_dir = (args.output_dir or default_output).resolve() + if args.dataset_file is not None: + args.dataset_file = args.dataset_file.resolve() registry_payload = registry_path.read_bytes() registry = json.loads(registry_payload) validate_event_identity(registry, args.event) @@ -544,7 +697,11 @@ def main(argv: list[str] | None = None) -> int: "or an explicit --output-dir" ) registry_sha256 = fiscal.hashlib.sha256(registry_payload).hexdigest() - years = validate_years(args.years or [], registry) + years = ( + validate_years(args.years or [], registry) + if legacy + else validate_variant_years(args.years or [], registry, variant) + ) index = {measure["measure_key"]: measure for measure in registry["measures"]} planned = worlds.computable(index, years=years) runnable = {key: world for key, world in planned.items() if "alias_of" not in world} @@ -559,6 +716,11 @@ def main(argv: list[str] | None = None) -> int: if coverage: raise ValueError("\n".join(coverage)) if args.dry_run or args.guards: + if args.guards: + # --guards reads the installed engine: check it is the certified + # one and compile it from its verified sources. + check_variant_environment(variant) + variants.isolate_bytecode(variants.RUNTIME_ROOT / variant["variant_id"]) try: resolve = worlds.engine_resolver() except ImportError: @@ -569,32 +731,46 @@ def main(argv: list[str] | None = None) -> int: gaps = worlds.reversal_delta_mismatches(index, resolve, years=years) if gaps: raise ValueError("\n".join(gaps)) + if args.guards: + import_engine() guard = "every reversal delta restates certified current law" - print( - json.dumps( - { - "event": args.event, - "years": years, - "selected": selected, - "not_computable": worlds.not_computable(index, years=years), - "guard": guard, - "annual_lookup_reforms": { - key: { - field: annual_reform(runnable[key][field]) - for field in ("baseline_reform", "reform_reform") - } - for key in selected - }, - }, - indent=1, - sort_keys=True, + plan = { + "event": args.event, + "years": years, + "selected": selected, + "not_computable": worlds.not_computable(index, years=years), + "guard": guard, + "annual_lookup_reforms": { + key: { + field: annual_reform(runnable[key][field]) + for field in ("baseline_reform", "reform_reform") + } + for key in selected + }, + } + if not legacy: + supported, unsupported = variants.supported_years( + variant, registry["calendar_years"] ) - ) + plan.update( + variant_fields, + supported_years=supported, + unsupported_years=unsupported, + output_dir=fiscal.relative_to_root(output_dir), + ) + print(json.dumps(plan, indent=1, sort_keys=True)) return 0 started = time.perf_counter() + variants.assert_not_sealed(output_dir) verify_registry_identity(registry_path, registry_sha256) print(f"[pid {os.getpid()}] starting offline bundle preflight", flush=True) - pre = preflight() + if legacy: + check_variant_environment(variant) + pre = ( + preflight(variant) + if legacy + else variant_preflight(variant, dataset_source=args.dataset_file) + ) print( f"[pid {os.getpid()}] offline bundle preflight complete; starting processed-world reversal guard", flush=True, @@ -608,17 +784,26 @@ def main(argv: list[str] | None = None) -> int: print(f"[pid {os.getpid()}] processed-world reversal guard complete", flush=True) verify_registry_identity(registry_path, registry_sha256) output_dir.mkdir(parents=True, exist_ok=True) + supported_calendar_years = ( + registry["calendar_years"] + if legacy + else variants.supported_years(variant, registry["calendar_years"])[0] + ) previous_digests = retained_artifact_digests( output_dir, event=args.event, - years=registry["calendar_years"], + years=supported_calendar_years, registry_sha256=registry_sha256, pre=pre, + variant_fields=variant_fields, ) requested_pairs = {(key, year) for key in selected for year in years} full_pairs = { (key, year) for key in runnable for year in registry["calendar_years"] } + supported_pairs = { + (key, year) for key, year in full_pairs if year in supported_calendar_years + } expected_paths = { str((output_dir / f"{key}_{year}.json").relative_to(ROOT)): (key, year) for key, year in full_pairs @@ -630,6 +815,13 @@ def main(argv: list[str] | None = None) -> int: "retained receipt names a measure/year outside the event grid" ) key, year = expected_paths[relative] + if (key, year) not in supported_pairs: + # No artifact exists for a year the population cannot supply, + # whatever an older receipt says (RELEASE_REPLAY.md). + raise ValueError( + f"retained receipt names {key} {year}, outside the variant's " + "supported years" + ) if not args.resume and (key, year) in requested_pairs: continue path = ROOT / relative @@ -648,7 +840,7 @@ def main(argv: list[str] | None = None) -> int: or previous.get("year") != year or any( previous.get(field) != value - for field, value in runtime_versions().items() + for field, value in {**runtime_versions(), **variant_fields}.items() ) ): raise ValueError("retained artifact identity differs from its receipt") @@ -679,6 +871,7 @@ def main(argv: list[str] | None = None) -> int: "registry_sha256": registry_sha256, "event": args.event, "output_dir": output_dir, + "variant_fields": variant_fields, "retained_artifacts": { str( (output_dir / f"{key}_{year}.json").relative_to(ROOT) @@ -723,6 +916,16 @@ def main(argv: list[str] | None = None) -> int: **runtime_versions(), "not_computable": worlds.not_computable(index, years=years), } + if not legacy: + manifest.update( + variants.manifest_fields( + variant, + registry["calendar_years"], + full_pairs, + supported_pairs, + artifact_pairs, + ) + ) fiscal.atomic_write_bytes( output_dir / "RUN_MANIFEST.json", canonical_bytes(manifest) ) diff --git a/pipeline/compute_uk_obr_costings.py b/pipeline/compute_uk_obr_costings.py index c95e0999..04f2a858 100644 --- a/pipeline/compute_uk_obr_costings.py +++ b/pipeline/compute_uk_obr_costings.py @@ -1025,8 +1025,12 @@ def run_managed_simulation( expected_dataset_sha256: str, include_engine_provenance: bool = False, scenario: Any = None, + require_dataset_years: bool = False, ) -> dict[str, Any]: - """Run, aggregate, delete, and collect exactly one managed sim.""" + """Run, aggregate, delete, and collect exactly one managed sim. + + With ``require_dataset_years`` a simulation whose dataset does not state + its years fails instead of skipping the year check.""" if scenario is not None and reform is not None: raise ValueError("pass either reform or scenario, not both") @@ -1070,6 +1074,22 @@ def run_managed_simulation( ] if missing: raise RuntimeError(f"managed sim lacks variables: {missing}") + # A year outside the dataset's projected years has no inputs: it + # calculates defaults without raising (a 2023 income tax of zero + # on a 2024 dataset). Both pinned engines expose dataset.years. + dataset_years = getattr(getattr(sim, "dataset", None), "years", None) + if dataset_years is None and require_dataset_years: + raise RuntimeError( + "the simulation dataset does not state its years; " + f"cannot confirm {year} has inputs" + ) + if dataset_years is not None and int(year) not in { + int(value) for value in dataset_years + }: + raise RuntimeError( + f"{year} is outside the simulation dataset's years " + f"{sorted(int(value) for value in dataset_years)}" + ) aggregates: dict[str, float] = {} for name in variables: calculated = sim.calculate(name, str(year)) diff --git a/pipeline/modal_uk_event.py b/pipeline/modal_uk_event.py index d5f43612..998c640f 100644 --- a/pipeline/modal_uk_event.py +++ b/pipeline/modal_uk_event.py @@ -71,7 +71,9 @@ "pipeline/compute_uk_obr_costings.py", "pipeline/build_uk_event_registry.py", "pipeline/uk_engine_registry.py", + "pipeline/uk_replay_variants.py", "data/uk/certified_bundle.json", + "data/uk/replay_variants.json", "docs/uk_replay/requirements.txt", "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json", "sources/uk_replay/source.json", diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index 02e247f3..d1f5155b 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -17,6 +17,7 @@ from pipeline import compute_uk_event as compute from pipeline import compute_uk_obr_costings as fiscal from pipeline import stage_uk_ab2025 as ab_stage +from pipeline import uk_replay_variants as variants AXES = [ "behavioural_adjustment", @@ -99,12 +100,83 @@ def validate_artifact( raise ValueError("per-measure head effects do not sum to the measure total") +LEGACY_POPULATION_NOTE = ( + "OBR uses the forecast available at announcement; PE uses a single " + "certified 2023 population with later calibration and uprating targets." +) + + +def population_note(record: dict | None) -> str: + """The first row annotation: which population PE used.""" + if record is None: + return LEGACY_POPULATION_NOTE + dataset = record["identity"]["dataset"] + return ( + "OBR uses the forecast available at announcement; PE uses a single " + f"certified population ({dataset['path']}, {dataset['repo_id']} " + f"{dataset['revision']}) with later calibration and uprating targets." + ) + + +def manifest_variant( + manifest: dict | None, registry: dict, *, variants_path: Path +) -> dict | None: + """The variant record a compute manifest claims, checked against the + variants document rather than trusted: its identity digest must match, + and its unsupported years must be the ones the record's data years give. + A manifest naming no variant is the legacy layout and lists none.""" + if manifest is None: + return None + variant_id = manifest.get("variant_id") + if variant_id is None: + if manifest.get("unsupported_years") or "variant_identity_sha256" in manifest: + raise ValueError("a legacy compute manifest lists variant fields") + return None + record = variants.get_variant( + variant_id, path=variants_path, allow_uncertified=True + ) + if variants.is_legacy(record): + raise ValueError("the legacy variant's manifest names no variant_id") + if manifest.get("variant_identity_sha256") != record["identity_sha256"]: + raise ValueError("compute manifest variant identity differs from its record") + identity = record["identity"] + if manifest.get("certified_dataset_sha256") != identity["dataset"][ + "sha256" + ] or manifest.get("data_bundle") != fiscal.data_bundle_id( + identity["release_bundle"] + ): + raise ValueError("compute manifest dataset differs from the variant's record") + supported, unsupported = variants.supported_years( + record, registry["calendar_years"] + ) + if ( + manifest.get("unsupported_years") != unsupported + or manifest.get("supported_years") != supported + or manifest.get("data_years") != record["data_years"] + ): + raise ValueError( + "compute manifest supported years differ from the variant's data years" + ) + return record + + def stage_event( - registry: dict, manifest: dict | None, *, artifact_dir: Path, registry_sha256: str + registry: dict, + manifest: dict | None, + *, + artifact_dir: Path, + registry_sha256: str, + variants_path: Path = variants.VARIANTS_PATH, ) -> tuple[list[dict], dict]: """Engine-free staging; missing measures and unsupported FYs stay visible.""" artifacts = {} paths = {} + # A non-legacy variant's manifest names its identity and the registry + # years its population cannot supply (RELEASE_REPLAY.md); both are + # checked against the variants document. + record = manifest_variant(manifest, registry, variants_path=variants_path) + variant_id = record["variant_id"] if record is not None else None + unsupported = {row["year"] for row in (manifest or {}).get("unsupported_years", [])} if manifest is not None: event = registry.get("event_slug", registry.get("event")) if event is not None and manifest.get("event", event) != event: @@ -142,6 +214,16 @@ def stage_event( registry_sha256, manifest["certified_dataset_sha256"], ) + if artifact.get("variant_id") != variant_id or artifact.get( + "variant_identity_sha256" + ) != (manifest.get("variant_identity_sha256") if variant_id else None): + raise ValueError( + "artifact variant identity differs from the compute manifest" + ) + if year in unsupported: + raise ValueError( + f"{key} {year}: artifact in a year the variant's population does not support" + ) planned = compute.worlds.computable(index, years=registry["calendar_years"]) full_pairs = { (key, year) @@ -150,6 +232,7 @@ def stage_event( for year in registry["calendar_years"] } computed_pairs = set(artifacts) + supported_pairs = {pair for pair in full_pairs if pair[1] not in unsupported} if computed_pairs - full_pairs: raise ValueError( "committed artifact is outside the event's executable measure/year grid" @@ -217,6 +300,17 @@ def declared_pairs(field): raise ValueError( "compute manifest full-event completeness is inaccurate" ) + if variant_id is not None: + if declared_pairs("full_supported_grid") != supported_pairs: + raise ValueError( + "compute manifest supported grid differs from the variant's years" + ) + if manifest["full_supported_grid_complete"] != ( + supported_pairs <= computed_pairs + ): + raise ValueError( + "compute manifest supported-grid completeness is inaccurate" + ) elif "measures" in manifest and "years" in manifest: requested = { (key, year) @@ -281,7 +375,7 @@ def declared_pairs(field): "source_table": source.get("source_table"), "source_column": source.get("source_column"), "annotations": [ - "OBR uses the forecast available at announcement; PE uses a single certified 2023 population with later calibration and uprating targets.", + population_note(record), "PE calendar-year totals proxy fiscal-year totals; PE is static and its certified policy world differs from the announcement baseline.", ], } @@ -308,6 +402,18 @@ def declared_pairs(field): row["reason"] = ( "FY start outside this event's documented certified-bundle support" ) + elif year in unsupported: + data_years = manifest["data_years"] + reason = next( + r["reason"] + for r in manifest["unsupported_years"] + if r["year"] == year + ) + row["status"] = "unsupported_on_variant" + row["reason"] = ( + f"CY{year} is outside {variant_id}'s supported data years " + f"{data_years[0]}–{data_years[-1]} ({reason})" + ) elif artifact is None: row["reason"] = ( "no numerical artifact for this measure and calendar year" @@ -392,6 +498,20 @@ def declared_pairs(field): } if inert_pairs: tally["inert_measure_years"] = compute.grid_rows(inert_pairs) + if variant_id is not None: + tally.update( + { + "variant_id": variant_id, + "unsupported_years": sorted(unsupported), + "unsupported_measure_years": compute.grid_rows( + full_pairs - supported_pairs + ), + "supported_grid_size": len(supported_pairs), + "full_supported_grid_complete": supported_pairs <= computed_pairs + and not inert_pairs, + "unsupported_source_rows": counts.get("unsupported_on_variant", 0), + } + ) if tally["source_rows"] != tally["staged_rows"]: raise ValueError( "staging inventory does not preserve the source-row accounting identity" @@ -407,6 +527,12 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--registry", type=Path) ap.add_argument("--artifact-dir", type=Path) ap.add_argument("--output", type=Path) + ap.add_argument( + "--variants-file", + type=Path, + default=variants.VARIANTS_PATH, + help="variant records a manifest's variant_id is checked against", + ) args = ap.parse_args(argv) default_registry, default_artifacts = compute.event_paths(args.event) registry_path = args.registry or default_registry @@ -420,11 +546,18 @@ def main(argv: list[str] | None = None) -> int: registry = json.loads(payload) compute.validate_event_identity(registry, args.event) compute.registry_builder.validate_registry(registry) + output = args.output or artifact_dir / "STAGED.jsonl" + # A sealed namespace's staged rows are part of what its receipt binds. + variants.assert_not_sealed(artifact_dir) + variants.assert_not_sealed(output.parent) rows, tally = stage_event( - registry, manifest, artifact_dir=artifact_dir, registry_sha256=registry_sha256 + registry, + manifest, + artifact_dir=artifact_dir, + registry_sha256=registry_sha256, + variants_path=args.variants_file.resolve(), ) compute.verify_registry_identity(registry_path, registry_sha256) - output = args.output or artifact_dir / "STAGED.jsonl" output.parent.mkdir(parents=True, exist_ok=True) staged_payload = b"".join( (json.dumps(row, sort_keys=True, allow_nan=False) + "\n").encode() diff --git a/pipeline/uk_engine_registry.py b/pipeline/uk_engine_registry.py index 625d4290..11cdb224 100644 --- a/pipeline/uk_engine_registry.py +++ b/pipeline/uk_engine_registry.py @@ -4,11 +4,23 @@ import math -def engine_dumps(pin="2.89.2", system=None): +def variant_engine_pin(variant_id=None): + """The policyengine-uk version of a certified replay variant (default: the + variant registries are authored against).""" + try: + from pipeline import uk_replay_variants as variants + except ModuleNotFoundError: + import uk_replay_variants as variants + record = variants.get_variant(variant_id) + return record["identity"]["packages"]["policyengine-uk"]["version"] + + +def engine_dumps(pin=None, system=None, variant=None): """(parameter path -> 2026 value, variable name -> info, year probe).""" import policyengine_uk from policyengine_core.parameters import Parameter, ParameterNode, ParameterScale + pin = pin or variant_engine_pin(variant) installed = importlib.metadata.version("policyengine-uk") if installed != pin: raise SystemExit( diff --git a/pipeline/uk_replay_interface.py b/pipeline/uk_replay_interface.py new file mode 100644 index 00000000..3788f5a2 --- /dev/null +++ b/pipeline/uk_replay_interface.py @@ -0,0 +1,81 @@ +"""Canonical identities for qualification requests and replay receipts. + +Release qualification (policyengine.py) sends a request; this repository +returns a receipt. Both sides address them by digest, so both must produce +the same bytes: RFC 8785 JSON Canonicalization Scheme (JCS), via the +``rfc8785`` package. JCS fixes what ``json.dumps(sort_keys=True)`` leaves to +the implementation: number serialisation (``1e-7``, not ``1e-07``; +``15000000000000000``, not ``1.5e+16``) and literal UTF-8 for non-ASCII +strings. See docs/uk_replay/QUALIFICATION_INTERFACE.md. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_DIR = ROOT / "docs" / "uk_replay" / "schemas" +REQUEST_SCHEMA_ID = "policyengine.uk-replay-qualification-request/v1" +RECEIPT_SCHEMA_ID = "policyengine.uk-replay-receipt/v1" +STATUS_SCHEMA_ID = "policyengine.uk-replay-status/v1" +SCHEMA_FILES = { + REQUEST_SCHEMA_ID: "qualification_request.v1.json", + RECEIPT_SCHEMA_ID: "replay_receipt.v1.json", + STATUS_SCHEMA_ID: "replay_status.v1.json", +} + + +def jcs_bytes(value: Any) -> bytes: + """RFC 8785 canonical UTF-8 bytes; the package refuses non-finite numbers.""" + import rfc8785 + + return rfc8785.dumps(value) + + +def jcs_sha256(value: Any) -> str: + return hashlib.sha256(jcs_bytes(value)).hexdigest() + + +def request_id(request: dict) -> str: + """The request's own digest; it is never a field of the request.""" + if "request_id" in request: + raise ValueError("request_id is derived from the request, not carried in it") + return jcs_sha256(request) + + +def receipt_sha256(receipt: dict) -> str: + """Digest of the receipt without its own receipt_sha256 field.""" + return jcs_sha256({k: v for k, v in receipt.items() if k != "receipt_sha256"}) + + +def seal_receipt(receipt: dict) -> dict: + sealed = {k: v for k, v in receipt.items() if k != "receipt_sha256"} + sealed["receipt_sha256"] = receipt_sha256(sealed) + return sealed + + +def load_schema(schema_id: str) -> dict: + return json.loads((SCHEMA_DIR / SCHEMA_FILES[schema_id]).read_bytes()) + + +def validate(document: dict, schema_id: str) -> None: + """JSON Schema validation, then the digest identity a schema cannot express.""" + import jsonschema + + jsonschema.Draft202012Validator( + load_schema(schema_id), + format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER, + ).validate(document) + if schema_id == RECEIPT_SCHEMA_ID and ( + document["receipt_sha256"] != receipt_sha256(document) + ): + raise ValueError("receipt_sha256 does not match the receipt's canonical bytes") + + +def validate_request(request: dict) -> str: + """Validate a request and return its request_id.""" + validate(request, REQUEST_SCHEMA_ID) + return request_id(request) diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py new file mode 100644 index 00000000..a9d54cdd --- /dev/null +++ b/pipeline/uk_replay_variants.py @@ -0,0 +1,893 @@ +"""Certified replay variants: the exact worlds the event corpus may run on. + +A variant is one certified policyengine.py bundle's execution identity: the +wrapper, model and core distributions, the release bundle the wrapper reports +under its packaged certification, and the certified dataset's bytes. The +runner validates the selected variant exactly before any simulation and +refuses every other combination; there is no flag that relaxes a check. +Running a newer engine means adding a variant whose certification names it. +See docs/uk_replay/RELEASE_REPLAY.md. + +Engine-free: nothing here imports policyengine. Installed distributions are +read through importlib.metadata and their RECORD files. +""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib.metadata +import json +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VARIANTS_PATH = ROOT / "data" / "uk" / "replay_variants.json" +CERTIFIED_BUNDLE = ROOT / "data" / "uk" / "certified_bundle.json" +LEGACY_RESULT_ROOT = "results/uk/events" +REPLAY_RESULT_ROOT = "results/uk/replays" +RUNTIME_ROOT = ROOT / ".venv-replay-checks" / "runtime" +RECEIPT_NAME = "REPLAY_RECEIPT.json" +PACKAGES = ("policyengine", "policyengine-uk", "policyengine-core") +PACKAGED_MANIFEST = "policyengine/data/bundle/manifest.json" +TOKEN_VARIABLES = ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN") +OFFLINE_VARIABLES = { + "HF_HUB_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", +} +# policyengine.py 6.x fetches the data release manifest with requests +# (provenance/manifest.py, get_data_release_manifest), which ignores +# HF_HUB_OFFLINE. requests reads these proxy variables, so with them the +# fetch fails at once and certify_data_release_compatibility falls back to +# the wrapper's packaged certification, the one each record certifies. +NO_NETWORK = { + name: "http://127.0.0.1:9" + for name in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ) +} +PROXY_EXCEPTIONS = ("NO_PROXY", "no_proxy") +ENGINE_MODULES = { + "policyengine": "policyengine", + "policyengine-uk": "policyengine_uk", + "policyengine-core": "policyengine_core", +} +MATERIALIZATIONS = ("hf_cache_mirror", "policyengine_data_dir") +LAYOUTS = ("legacy_events", "replay_namespace") +STATUSES = ("certified", "candidate", "retired") +EXECUTORS = ("local", "modal") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +VERSION = re.compile(r"^[0-9]+(?:\.[0-9]+)*[a-z0-9.+-]*$") +PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)") +# Installer bookkeeping differs between pip and uv for the same wheel; only +# the distribution's own files identify what was installed. Paths outside +# site-packages ("../../../bin/...") are console-script wrappers the +# installer generates, with the environment's interpreter in their shebang. +INSTALLER_METADATA = ( + "INSTALLER", + "REQUESTED", + "RECORD", + "direct_url.json", + "uv_cache.json", +) + + +class VariantError(ValueError): + """A variant record, installed environment or dataset is not the certified one.""" + + +def canonical_identity_bytes(value: object) -> bytes: + """Sorted-key compact JSON. Identities hold strings, integers and nulls + only, for which this equals RFC 8785; a test checks that equality.""" + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + + +def identity_sha256(identity: dict) -> str: + return hashlib.sha256(canonical_identity_bytes(identity)).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def variant_slug(identity: dict) -> str: + packages = identity["packages"] + build = identity["release_bundle"]["certified_data_build_id"] + return ( + f"pe{packages['policyengine']['version']}" + f"_uk{packages['policyengine-uk']['version']}_{build}" + ) + + +def variant_id_for(identity: dict, taken: dict[str, str] | None = None) -> str: + """The version slug, suffixed with the identity digest when another + identity already holds that slug (same versions, other wheels or + numerical environment).""" + slug = variant_slug(identity) + digest = identity_sha256(identity) + holder = (taken or {}).get(slug) + return slug if holder in (None, digest) else f"{slug}--{digest[:10]}" + + +def normalized_freeze(text: str, versions: dict[str, str]) -> str: + """A freeze as exact pins. Engine packages installed from a verified local + wheel or a commit appear as ``name @ ``; they become ``name==version`` + (the wheel and installed-file digests bind them) with the source kept as a + comment. Any other non-pinned line refuses.""" + lines = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + name, at, source = stripped.partition(" @ ") + key = re.sub(r"[-_.]+", "-", name).lower() + if at and key in versions: + lines.append( + f"# {key} installed from {source.split('/')[-1] if source.startswith('file:') else source}" + ) + lines.append(f"{key}=={versions[key]}") + elif at: + raise VariantError(f"freeze line is not an exact pin: {stripped}") + else: + lines.append(stripped) + pins = requirements_pins("\n".join(lines)) + missing = sorted(set(versions) - set(pins)) + if missing: + raise VariantError(f"freeze lacks {missing}") + return "\n".join(lines) + "\n" + + +def hf_uri(dataset: dict) -> str: + return f"hf://{dataset['repo_id']}/{dataset['path']}@{dataset['revision']}" + + +def requirements_pins(text: str) -> dict[str, str]: + """Exact `name==version` lines only, normalised names; anything else refuses.""" + pins = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + match = PIN.fullmatch(line) + if not match: + raise VariantError(f"requirements must be exact pins, found {line!r}") + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() + if name in pins: + raise VariantError(f"duplicate requirement {name}") + pins[name] = match.group(2) + return pins + + +# --- the variant file ------------------------------------------------------- + + +def load_document(path: Path = VARIANTS_PATH) -> dict: + document = json.loads(path.read_bytes()) + if document.get("schema_version") != 1: + raise VariantError("replay_variants.json schema_version must be 1") + variants = document.get("variants") + if not isinstance(variants, list) or not variants: + raise VariantError("replay_variants.json lists no variants") + ids = [variant.get("variant_id") for variant in variants] + if len(ids) != len(set(ids)): + raise VariantError("replay_variants.json repeats a variant_id") + if document.get("default_variant") not in ids: + raise VariantError("default_variant is not a listed variant") + default = variants[ids.index(document["default_variant"])] + if default.get("status") != "certified": + raise VariantError("default_variant must be a certified variant") + return document + + +def get_variant( + variant_id: str | None = None, + *, + path: Path = VARIANTS_PATH, + root: Path = ROOT, + allow_uncertified: bool = False, +) -> dict: + """The validated record; None selects the file's default variant. + + Only a certified variant runs, unless a qualification run, which replays + its own candidate record, asks with ``allow_uncertified``.""" + document = load_document(path) + wanted = variant_id or document["default_variant"] + for record in document["variants"]: + if record["variant_id"] == wanted: + validate_record(record, root=root) + if record["status"] != "certified" and not allow_uncertified: + raise VariantError( + f"{wanted} is {record['status']}, not certified; only a " + "qualification run replays an uncertified variant" + ) + return record + raise VariantError(f"no replay variant {wanted!r}") + + +def is_legacy(record: dict) -> bool: + """The populace-2023 world whose results keep their original layout.""" + return record["layout"] == "legacy_events" + + +def validate_record(record: dict, *, root: Path = ROOT) -> None: + """Engine-free consistency: the record is one certified pairing.""" + problems = [] + + def need(condition: bool, message: str) -> None: + if not condition: + problems.append(message) + + try: + identity = record["identity"] + packages = identity["packages"] + bundle = identity["release_bundle"] + dataset = identity["dataset"] + certification = record["certification"] + except (KeyError, TypeError) as exc: + raise VariantError(f"variant record is missing {exc}") from exc + if not all( + isinstance(part, dict) + for part in (identity, packages, bundle, dataset, certification) + ): + raise VariantError( + "identity, its packages, release_bundle and dataset, and " + "certification must be objects" + ) + need( + set(packages) == set(PACKAGES), + "identity.packages must name exactly the three packages", + ) + for name in PACKAGES: + package = packages.get(name) or {} + need( + isinstance(package.get("version"), str) + and bool(VERSION.fullmatch(package.get("version", ""))), + f"{name}: version", + ) + need( + bool(SHA256.fullmatch(str(package.get("record_sha256")))), + f"{name}: record_sha256", + ) + if "wheel_sha256" in package: + need( + bool(SHA256.fullmatch(str(package["wheel_sha256"]))), + f"{name}: wheel_sha256", + ) + if problems: + raise VariantError("; ".join(problems)) + uk = packages["policyengine-uk"]["version"] + wrapper = packages["policyengine"]["version"] + need( + record.get("variant_id") + in ( + variant_slug(identity), + f"{variant_slug(identity)}--{identity_sha256(identity)[:10]}", + ), + "variant_id differs from its identity", + ) + need( + record.get("identity_sha256") == identity_sha256(identity), + "identity_sha256 differs", + ) + need( + bool(SHA256.fullmatch(str(identity.get("packaged_bundle_manifest_sha256")))), + "packaged_bundle_manifest_sha256", + ) + need( + isinstance(bundle, dict) + and all(value is None or isinstance(value, str) for value in bundle.values()), + "release_bundle values must be strings or null", + ) + need( + bundle.get("model_package") == "policyengine-uk", "release_bundle model_package" + ) + need( + bundle.get("model_version") == uk, + "release_bundle model_version differs from the installed model pin", + ) + need( + bundle.get("policyengine_version") == wrapper, + "release_bundle policyengine_version differs from the wrapper pin", + ) + need( + certification.get("certified_for_model_version") == uk, + "the certification does not name this model version", + ) + need( + certification.get("data_build_id") == bundle.get("certified_data_build_id"), + "the certification names another data build", + ) + need( + certification.get("compatibility_basis") == bundle.get("compatibility_basis"), + "certification compatibility_basis differs from the release bundle", + ) + for field in ("repo_id", "repo_type", "revision", "path", "sha256", "size_bytes"): + need(field in dataset, f"dataset.{field}") + if problems: + raise VariantError("; ".join(problems)) + need(dataset["repo_type"] in ("model", "dataset"), "dataset.repo_type") + need(bool(SHA256.fullmatch(str(dataset["sha256"]))), "dataset.sha256") + need( + isinstance(dataset["size_bytes"], int) and dataset["size_bytes"] > 0, + "dataset.size_bytes", + ) + need( + bundle.get("certified_data_artifact_sha256") == dataset["sha256"], + "the certified artifact digest differs from the dataset", + ) + need( + bundle.get("default_dataset_uri") == hf_uri(dataset), + "default_dataset_uri differs from the dataset", + ) + need( + identity.get("materialization") in MATERIALIZATIONS, "identity.materialization" + ) + need( + isinstance(identity.get("python"), str) + and bool(re.fullmatch(r"3\.[0-9]+", identity.get("python", ""))), + "identity.python", + ) + need(record.get("status") in STATUSES, "status") + need(record.get("layout") in LAYOUTS, "layout") + need( + isinstance(record.get("executor"), list) + and bool(record["executor"]) + and set(record["executor"]) <= set(EXECUTORS), + "executor", + ) + years = record.get("data_years") + need( + isinstance(years, list) + and bool(years) + and all(isinstance(year, int) and not isinstance(year, bool) for year in years) + and years == list(range(years[0], years[-1] + 1)), + "data_years must be consecutive calendar years", + ) + expected_root = ( + LEGACY_RESULT_ROOT + if record.get("layout") == "legacy_events" + else f"{REPLAY_RESULT_ROOT}/{record.get('variant_id')}" + ) + need( + record.get("result_root") == expected_root, + f"result_root must be {expected_root}", + ) + requirements = record.get("requirements") or {} + need( + identity.get("requirements_sha256") == requirements.get("sha256"), + "the identity binds another requirements file", + ) + requirements_path = root / str(requirements.get("path", "")) + if not requirements.get("path") or not requirements_path.is_file(): + problems.append("requirements file is missing") + else: + payload = requirements_path.read_bytes() + need( + hashlib.sha256(payload).hexdigest() == requirements.get("sha256"), + "requirements sha256 differs", + ) + try: + pins = requirements_pins(payload.decode()) + except VariantError as exc: + problems.append(str(exc)) + else: + for name in PACKAGES: + need( + pins.get(name) == packages[name]["version"], + f"requirements pin {name}", + ) + if record.get("layout") == "legacy_events": + legacy = json.loads( + (root / "data" / "uk" / "certified_bundle.json").read_bytes() + ) + need( + legacy.get("repo_id") == dataset["repo_id"] + and legacy.get("artifact") == dataset["path"] + and legacy.get("revision") == dataset["revision"] + and legacy.get("revision") == bundle.get("certified_data_build_id") + and legacy.get("sha256") == dataset["sha256"] + and legacy.get("size_bytes") == dataset["size_bytes"], + "the legacy variant differs from data/uk/certified_bundle.json", + ) + need( + any( + row.get("name") == "policyengine-uk" + and row.get("specifier") == f"=={uk}" + for row in legacy.get("compatible_model_packages", []) + ), + "data/uk/certified_bundle.json does not declare this model version", + ) + if problems: + raise VariantError(f"{record.get('variant_id')}: " + "; ".join(problems)) + + +# --- the installed environment --------------------------------------------- + + +def _record_rows(distribution) -> list[tuple[str, str, str]]: + text = distribution.read_text("RECORD") + if text is None: + raise VariantError(f"{distribution.metadata['Name']}: no RECORD file") + rows = [] + for line in text.splitlines(): + if not line.strip(): + continue + path, digest, size = (line.rsplit(",", 2) + ["", ""])[:3] + name = path.rsplit("/", 1)[-1] + if ( + ".dist-info/" in path + and name in INSTALLER_METADATA + or "__pycache__/" in path + or path.endswith(".pyc") + or path.startswith("../") + ): + continue + rows.append((path, digest, size)) + return sorted(rows) + + +def distribution_record_sha256(distribution, *, verify_files: bool = True) -> str: + """Digest of the distribution's own RECORD entries, after checking every + listed file on disk against its recorded hash. Installer bookkeeping and + bytecode are excluded, so pip and uv installs of one wheel agree.""" + rows = _record_rows(distribution) + if verify_files: + for path, digest, size in rows: + if not digest: + raise VariantError(f"{path}: RECORD lists no hash") + algorithm, _, expected = digest.partition("=") + if algorithm != "sha256": + raise VariantError(f"{path}: RECORD hash is not sha256") + payload = Path(distribution.locate_file(path)).read_bytes() + actual = ( + base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + .rstrip(b"=") + .decode() + ) + if actual != expected or (size and int(size) != len(payload)): + raise VariantError( + f"installed file differs from its wheel RECORD: {path}" + ) + body = "\n".join(",".join(row) for row in rows).encode() + return hashlib.sha256(body).hexdigest() + + +def installed_distributions() -> dict[str, str]: + """Version of each importable distribution. When a name appears twice on + sys.path the first one is what Python imports (and what + importlib.metadata.version reports), so the first wins here too: a + host's own packages behind the pinned environment, such as Modal's + client dependencies in its containers, do not count.""" + versions = {} + for distribution in importlib.metadata.distributions(): + name = re.sub(r"[-_.]+", "-", distribution.metadata["Name"]).lower() + versions.setdefault(name, distribution.version) + return versions + + +def installed_identity(*, verify_files: bool = True) -> dict: + packages = {} + for name in PACKAGES: + try: + distribution = importlib.metadata.distribution(name) + except importlib.metadata.PackageNotFoundError as exc: + raise VariantError(f"{name} is not installed") from exc + packages[name] = { + "version": distribution.version, + "record_sha256": distribution_record_sha256( + distribution, verify_files=verify_files + ), + } + manifest = Path( + importlib.metadata.distribution("policyengine").locate_file(PACKAGED_MANIFEST) + ) + return { + "packages": packages, + "packaged_bundle_manifest_sha256": sha256_file(manifest), + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "distributions": installed_distributions(), + "unlisted_files": unlisted_package_files() if verify_files else {}, + "startup_files": startup_files() if verify_files else [], + } + + +def unlisted_package_files(names=PACKAGES) -> dict[str, list[str]]: + """Files and links in each package's top-level directories that the + package's own RECORD does not list. + + A listed file is checked against its hash, but policyengine-core also + loads files nobody listed: every file in a parameter directory and its + subdirectories (parameters/parameter_node.py: sorted os.listdir, then + os.path.isdir) and every module in a variable directory and its + subdirectories (taxbenefitsystems/tax_benefit_system.py, + add_variables_from_directory: glob "*.py", then glob "*/"). Both recurse + into a ``__pycache__`` and follow a symlinked directory, so: + + * only ``*.pyc`` files inside a ``__pycache__`` are skipped (runs never + read them: isolate_bytecode); anything else there is reported; + * every symlink, to a file or a directory, is reported; + * a file counts as listed only in its own package's RECORD, by its + unresolved path: another distribution's RECORD is never hash-checked. + """ + found = {} + for name in names: + distribution = importlib.metadata.distribution(name) + rows = _record_rows(distribution) + listed = { + os.path.normpath(str(distribution.locate_file(row[0]))) for row in rows + } + tops = { + row[0].split("/", 1)[0] + for row in rows + if "/" in row[0] and ".dist-info/" not in row[0] + } + root = str(distribution.locate_file("")) + extra = [] + for top in sorted(tops): + base = str(distribution.locate_file(top)) + if os.path.islink(base): + extra.append(f"{top} (symlink)") + continue + for directory, subdirectories, files in os.walk(base): + subdirectories.sort() + cache = "__pycache__" in Path(directory).parts + for entry in sorted([*subdirectories, *files]): + path = os.path.normpath(os.path.join(directory, entry)) + relative = os.path.relpath(path, root) + if os.path.islink(path): + extra.append(f"{relative} (symlink)") + elif entry in subdirectories: + continue + elif cache and entry.endswith(".pyc"): + continue + elif path not in listed: + extra.append(relative) + if extra: + found[name] = extra + return found + + +VIRTUALENV_SHIM = ("_virtualenv.pth", "import _virtualenv") + + +def startup_files(names=PACKAGES) -> list[dict]: + """The ``*.pth`` files beside the engine packages. Python runs their + ``import`` lines at startup (site.py), before any check here, so each + must belong to a pinned distribution or be the virtual environment's + own shim; verify_installed decides.""" + owners = {} + for distribution in importlib.metadata.distributions(): + owner = re.sub(r"[-_.]+", "-", distribution.metadata["Name"]).lower() + for line in (distribution.read_text("RECORD") or "").splitlines(): + relative = line.rsplit(",", 2)[0] + if relative.endswith(".pth"): + owners[os.path.normpath(str(distribution.locate_file(relative)))] = ( + owner + ) + sites = sorted( + {str(importlib.metadata.distribution(name).locate_file("")) for name in names} + ) + out = [] + for site in sites: + for entry in sorted(os.listdir(site)): + if not entry.endswith(".pth"): + continue + path = os.path.normpath(os.path.join(site, entry)) + out.append( + { + "name": entry, + "owner": owners.get(path), + "text": Path(path).read_text(errors="replace").strip()[:200], + "symlink": os.path.islink(path), + } + ) + return out + + +def startup_file_problems(files: list[dict], pins: dict[str, str]) -> list[str]: + problems = [] + for file in files: + shim = (file["name"], file["text"]) == VIRTUALENV_SHIM + if file.get("symlink") or not (shim or file["owner"] in pins): + problems.append( + f"startup file {file['name']} belongs to no pinned distribution" + ) + return problems + + +def isolate_bytecode(runtime_root: Path, environ=os.environ) -> Path: + """Compile the engine from its verified sources, never from a cache. + + Bytecode is read only from sys.pycache_prefix when one is set, so an + empty prefix with writing turned off ignores every __pycache__ beside + the sources (including one whose header matches its source). Spawned + year workers inherit both settings. Call before importing the engine.""" + loaded = sorted(m for m in ENGINE_MODULES.values() if m in sys.modules) + if loaded: + raise VariantError(f"engine modules imported before isolation: {loaded}") + prefix = runtime_root.resolve() / "no-bytecode" + if prefix.is_symlink(): # before anything resolves or removes it + raise VariantError(f"{prefix} is a symlink; refusing it") + if prefix.exists(): + shutil.rmtree(prefix) + prefix.mkdir(parents=True) + sys.pycache_prefix = str(prefix) + sys.dont_write_bytecode = True + environ["PYTHONPYCACHEPREFIX"] = str(prefix) + environ["PYTHONDONTWRITEBYTECODE"] = "1" + return prefix + + +def verify_import_origins(names=PACKAGES) -> None: + """Each imported engine package is the verified distribution's own + files, not a same-named tree earlier on sys.path.""" + for name in names: + module_name = ENGINE_MODULES[name] + module = sys.modules.get(module_name) + if module is None: + raise VariantError(f"{module_name} is not imported yet") + expected = Path( + importlib.metadata.distribution(name).locate_file( + f"{module_name}/__init__.py" + ) + ).resolve() + if Path(module.__file__).resolve() != expected: + raise VariantError( + f"{module_name} imported from {module.__file__}, not its " + f"verified distribution ({expected})" + ) + + +def environment_mismatches(pins: dict[str, str], installed: dict) -> list[str]: + """Every pinned distribution installed at its pinned version. Extra + distributions (the receipt tools) are allowed; a missing or different + numerical dependency is not.""" + have = installed.get("distributions", {}) + return [ + f"{name}: pinned {version}, installed {have.get(name)}" + for name, version in sorted(pins.items()) + if have.get(name) != version + ] + + +def identity_mismatches(record: dict, installed: dict) -> list[str]: + identity = record["identity"] + out = [] + for name in PACKAGES: + want = identity["packages"][name] + have = installed["packages"].get(name, {}) + for field in ("version", "record_sha256"): + if want[field] != have.get(field): + out.append( + f"{name} {field}: certified {want[field]}, installed {have.get(field)}" + ) + if ( + identity["packaged_bundle_manifest_sha256"] + != installed["packaged_bundle_manifest_sha256"] + ): + out.append( + "packaged bundle manifest: certified " + f"{identity['packaged_bundle_manifest_sha256']}, installed " + f"{installed['packaged_bundle_manifest_sha256']}" + ) + if identity["python"] != installed["python"]: + out.append( + f"python: certified {identity['python']}, running {installed['python']}" + ) + return out + + +def verify_installed( + record: dict, *, installed: dict | None = None, root: Path = ROOT +) -> dict: + installed = installed if installed is not None else installed_identity() + mismatches = identity_mismatches(record, installed) + pins = requirements_pins((root / record["requirements"]["path"]).read_text()) + mismatches += environment_mismatches(pins, installed) + mismatches += [ + f"{name}: files outside its RECORD: {', '.join(paths[:5])}" + + (f" and {len(paths) - 5} more" if len(paths) > 5 else "") + for name, paths in sorted(installed.get("unlisted_files", {}).items()) + ] + mismatches += startup_file_problems(installed.get("startup_files", []), pins) + if mismatches: + raise VariantError( + f"{record['variant_id']}: installed environment is not the certified one: " + + "; ".join(mismatches) + ) + return installed + + +def isolate_certification_environment(environ=os.environ) -> list[str]: + """Use the wrapper's packaged certification on every host. + + With a Hugging Face token, policyengine.py derives the certification from + the remote release manifest instead, which changes release_bundle fields + (OFFLINE_BUNDLE_AUDIT.json records the 5.0.2 case; 6.2.6 behaves alike). + Without a token a private data repository answers 401; a public one + would still answer, so the proxy variables (NO_NETWORK) make the fetch + fail for both. Call before importing policyengine; returns the + variables removed.""" + removed = [ + name + for name in (*TOKEN_VARIABLES, *PROXY_EXCEPTIONS) + if environ.pop(name, None) is not None + ] + environ.update(OFFLINE_VARIABLES) + environ.update(NO_NETWORK) + return removed + + +def verify_release_bundle(record: dict, bundle: dict) -> None: + expected = record["identity"]["release_bundle"] + if dict(bundle) != expected: + differences = { + key: {"certified": expected.get(key), "runtime": bundle.get(key)} + for key in sorted(set(expected) | set(bundle)) + if expected.get(key) != bundle.get(key) + } + raise VariantError( + f"{record['variant_id']}: runtime release bundle differs: {differences}" + ) + + +# --- the dataset -------------------------------------------------------------- + + +def _cache_candidates(dataset: dict, hub_cache: Path) -> list[Path]: + owner, name = dataset["repo_id"].split("/", 1) + prefix = "models" if dataset["repo_type"] == "model" else "datasets" + snapshots = hub_cache / f"{prefix}--{owner}--{name}" / "snapshots" + return sorted(snapshots.glob(f"*/{dataset['path']}")) if snapshots.is_dir() else [] + + +def find_dataset_bytes(record: dict, *, source: Path | None = None) -> Path: + """A local file holding the certified bytes, found by digest, never by name.""" + dataset = record["identity"]["dataset"] + hub_cache = Path( + os.environ.get("HF_HUB_CACHE") + or Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) + / "hub" + ) + candidates = ( + [source] if source is not None else _cache_candidates(dataset, hub_cache) + ) + for candidate in candidates: + path = Path(candidate) + if ( + path.is_file() + and path.stat().st_size == dataset["size_bytes"] + and sha256_file(path) == dataset["sha256"] + ): + return path + raise VariantError( + f"{record['variant_id']}: no local file holds the certified dataset " + f"{dataset['path']} (sha256 {dataset['sha256']}); searched {[str(c) for c in candidates]}" + ) + + +def materialize_dataset( + record: dict, runtime_root: Path | None = None, *, source: Path | None = None +) -> Path: + """Place verified certified bytes where policyengine.py 6.x looks: the + runtime directory's ./data/ (provenance/dataset_materialization.py, + DEFAULT_DATA_DIR). Returns the absolute path.""" + if record["identity"]["materialization"] != "policyengine_data_dir": + raise VariantError( + f"{record['variant_id']} does not materialise into a data dir" + ) + dataset = record["identity"]["dataset"] + runtime_root = (runtime_root or RUNTIME_ROOT / record["variant_id"]).resolve() + destination = runtime_root / "data" / Path(dataset["path"]).name + for path in (runtime_root / "data", destination): + if path.is_symlink(): + raise VariantError(f"{path} is a symlink; refusing it") + if destination.resolve().parent != (runtime_root / "data").resolve(): + raise VariantError("dataset destination escapes the runtime directory") + if ( + destination.is_file() + and destination.stat().st_size == dataset["size_bytes"] + and sha256_file(destination) == dataset["sha256"] + ): + return destination + origin = find_dataset_bytes(record, source=source) + destination.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(dir=destination.parent, prefix=".materialize-") + os.close(handle) + temporary = Path(temporary) + try: + shutil.copyfile(origin, temporary) + if sha256_file(temporary) != dataset["sha256"]: + raise VariantError("dataset copy failed SHA-256 verification") + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + return destination + + +# --- years and namespaces ----------------------------------------------------- + + +def supported_years(record: dict, registry_years) -> tuple[list[int], list[dict]]: + """Partition an event's registry years by the variant's data years.""" + data = record["data_years"] + supported = sorted( + year for year in set(registry_years) if data[0] <= year <= data[-1] + ) + unsupported = [ + { + "year": year, + "reason": "before_data_year" if year < data[0] else "after_extension_end", + } + for year in sorted(set(registry_years)) + if not data[0] <= year <= data[-1] + ] + return supported, unsupported + + +def namespace_root(record: dict, *, root: Path = ROOT) -> Path: + return root / record["result_root"] + + +def event_output_dir(record: dict, event: str, *, root: Path = ROOT) -> Path: + return namespace_root(record, root=root) / event + + +def sealing_receipt(path: Path, *, root: Path = ROOT) -> Path | None: + """The REPLAY_RECEIPT.json sealing a directory or one of its parents.""" + path = path.resolve() + root = root.resolve() + for directory in (path, *path.parents): + if (directory / RECEIPT_NAME).is_file(): + return directory / RECEIPT_NAME + if directory == root: + break + return None + + +def assert_not_sealed(path: Path, *, root: Path = ROOT) -> None: + receipt = sealing_receipt(path, root=root) + if receipt is not None: + raise VariantError( + f"{path} is inside a sealed replay namespace ({receipt}); write a new namespace" + ) + + +def manifest_fields( + record: dict, registry_years, full_pairs, supported_pairs, artifact_pairs +) -> dict: + """RUN_MANIFEST additions for a non-legacy variant (RELEASE_REPLAY.md).""" + supported, unsupported = supported_years(record, registry_years) + return { + "variant_id": record["variant_id"], + "variant_identity_sha256": record["identity_sha256"], + "data_years": record["data_years"], + "supported_years": supported, + "unsupported_years": unsupported, + "full_supported_grid": [ + {"measure_key": key, "year": year} for key, year in sorted(supported_pairs) + ], + "full_supported_grid_complete": set(supported_pairs) <= set(artifact_pairs), + } diff --git a/tests/fixtures/uk_replay/DIGESTS.json b/tests/fixtures/uk_replay/DIGESTS.json new file mode 100644 index 00000000..c7f000d9 --- /dev/null +++ b/tests/fixtures/uk_replay/DIGESTS.json @@ -0,0 +1,8 @@ +{ + "qualification_request_golden.json": { + "request_id": "2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf" + }, + "replay_receipt_golden.json": { + "receipt_sha256": "5cbc44c1b79506a6410c12b437a7451a0e1387ab6a133b9cb8dc0bf71b396650" + } +} diff --git a/tests/fixtures/uk_replay/qualification_request_golden.jcs b/tests/fixtures/uk_replay/qualification_request_golden.jcs new file mode 100644 index 00000000..0e008c93 --- /dev/null +++ b/tests/fixtures/uk_replay/qualification_request_golden.jcs @@ -0,0 +1 @@ +{"attempt":1,"candidate":{"data":{"build_id":"policyengine-uk-data-1.58.0","dataset":"enhanced_frs_2024_25","path":"enhanced_frs_2024_25.h5","release_manifest_sha256":"ddd63e1f7755ee358c4ede9050bfdde09f3a80f48a5c0d8ee9d1fb531e4a1a07","repo_id":"policyengine/policyengine-uk-data-private","repo_type":"model","revision":"1.58.0","sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","size_bytes":128369110},"expected_release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"policyengine":{"packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","source":{"git":{"commit":"481940eca44fc75e41fbf9de4ff454e0a0d7c0e0","repository":"PolicyEngine/policyengine.py"}},"version":"6.2.6"},"policyengine_core":{"version":"3.33.0","wheel_sha256":"2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789"},"policyengine_uk":{"version":"2.125.1","wheel_sha256":"a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a"}},"compare_to":"latest_certified","kind":"candidate","requested_by":{"commit":"481940eca44fc75e41fbf9de4ff454e0a0d7c0e0","jcs_probe":[1e-7,15000000000000000,0.30000000000000004,0,1e+21,5e-324,123456789012345680000,9007199254740991],"note":"Golden fixture: JCS probes for non-ASCII (Budget £, café, 予算) and doubles.","repository":"PolicyEngine/policyengine.py","run_attempt":1,"workflow_run_id":1},"schema":"policyengine.uk-replay-qualification-request/v1"} \ No newline at end of file diff --git a/tests/fixtures/uk_replay/qualification_request_golden.json b/tests/fixtures/uk_replay/qualification_request_golden.json new file mode 100644 index 00000000..1f6d69c0 --- /dev/null +++ b/tests/fixtures/uk_replay/qualification_request_golden.json @@ -0,0 +1,72 @@ +{ + "schema": "policyengine.uk-replay-qualification-request/v1", + "kind": "candidate", + "attempt": 1, + "candidate": { + "policyengine": { + "version": "6.2.6", + "source": { + "git": { + "repository": "PolicyEngine/policyengine.py", + "commit": "481940eca44fc75e41fbf9de4ff454e0a0d7c0e0" + } + }, + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54" + }, + "policyengine_uk": { + "version": "2.125.1", + "wheel_sha256": "a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a" + }, + "policyengine_core": { + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "data": { + "build_id": "policyengine-uk-data-1.58.0", + "dataset": "enhanced_frs_2024_25", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "path": "enhanced_frs_2024_25.h5", + "sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "size_bytes": 128369110, + "release_manifest_sha256": "ddd63e1f7755ee358c4ede9050bfdde09f3a80f48a5c0d8ee9d1fb531e4a1a07" + }, + "expected_release_bundle": { + "bundle_id": "uk-6.2.6", + "country_id": "uk", + "policyengine_version": "6.2.6", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "data_build_model_version": "2.122.2", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "compatibility_basis": "legacy_compatible_model_package", + "certified_by": "policyengine.py bundle certification" + } + }, + "compare_to": "latest_certified", + "requested_by": { + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 1, + "run_attempt": 1, + "commit": "481940eca44fc75e41fbf9de4ff454e0a0d7c0e0", + "note": "Golden fixture: JCS probes for non-ASCII (Budget £, café, 予算) and doubles.", + "jcs_probe": [ + 1e-07, + 1.5e+16, + 0.30000000000000004, + -0.0, + 1e+21, + 5e-324, + 1.2345678901234568e+20, + 9007199254740991 + ] + } +} diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.jcs b/tests/fixtures/uk_replay/replay_receipt_golden.jcs new file mode 100644 index 00000000..d542e5bd --- /dev/null +++ b/tests/fixtures/uk_replay/replay_receipt_golden.jcs @@ -0,0 +1 @@ +{"corpus":{"corpus_version":"uk-fiscal-event-replay/1","events":{"spring_budget_2023":{"artifacts":8,"comparison_provenance_sha256":"8888888888888888888888888888888888888888888888888888888888888888","full_supported_grid_complete":true,"inert_measure_years":[],"registry_sha256":"5555555555555555555555555555555555555555555555555555555555555555","run_manifest_sha256":"6666666666666666666666666666666666666666666666666666666666666666","staging_manifest_sha256":"7777777777777777777777777777777777777777777777777777777777777777","supported_years":[2024,2025,2026,2027],"unsupported_years":[{"reason":"before_data_year","year":2023}]}}},"diff":{"base_variant_id":"pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z","diff_sha256":"9999999999999999999999999999999999999999999999999999999999999999","flag_counts":{"construction":0,"coverage":0,"inertness":0,"mapping":0,"numerical":2,"structural":1,"support":1},"flagged_rows":[{"base_gbp":-15000000000000000,"event":"spring_budget_2023","flags":["numerical"],"head_gbp":0.30000000000000004,"key":"universal_credit","level":"head","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2024},{"base_gbp":1e-7,"event":"spring_budget_2023","flags":["numerical","structural"],"head_gbp":-123456789.125,"key":"total","level":"measure_year","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2025},{"base_gbp":-2500000,"event":"spring_budget_2023","flags":["support"],"head_gbp":null,"key":"obr-pmd-row-123","level":"source_row","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2023}]},"execution":{"dataset_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","packages":{"policyengine":{"record_sha256":"1111111111111111111111111111111111111111111111111111111111111111","version":"6.2.6"},"policyengine-core":{"record_sha256":"3333333333333333333333333333333333333333333333333333333333333333","version":"3.33.0"},"policyengine-uk":{"record_sha256":"2222222222222222222222222222222222222222222222222222222222222222","version":"2.125.1"}},"platform":"Golden fixture (synthetic values; café £)","python":"3.12.14","release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"requirements_sha256":"4444444444444444444444444444444444444444444444444444444444444444","runner":"github-actions ubuntu-latest"},"integrity":{"failures":[],"status":"complete"},"rejection":null,"request_id":"2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf","results":{"commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","manifest_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","path":"requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/","ref":"replay-state","repository":"PolicyEngine/policyengine-scorecard"},"schema":"policyengine.uk-replay-receipt/v1","status":"complete","variant_id":"pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0","variant_identity_sha256":"0000000000000000000000000000000000000000000000000000000000000000"} \ No newline at end of file diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.json b/tests/fixtures/uk_replay/replay_receipt_golden.json new file mode 100644 index 00000000..87898936 --- /dev/null +++ b/tests/fixtures/uk_replay/replay_receipt_golden.json @@ -0,0 +1,138 @@ +{ + "schema": "policyengine.uk-replay-receipt/v1", + "request_id": "2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf", + "status": "complete", + "rejection": null, + "variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "variant_identity_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "execution": { + "packages": { + "policyengine": { + "version": "6.2.6", + "record_sha256": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "policyengine-uk": { + "version": "2.125.1", + "record_sha256": "2222222222222222222222222222222222222222222222222222222222222222" + }, + "policyengine-core": { + "version": "3.33.0", + "record_sha256": "3333333333333333333333333333333333333333333333333333333333333333" + } + }, + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", + "release_bundle": { + "bundle_id": "uk-6.2.6", + "country_id": "uk", + "policyengine_version": "6.2.6", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "data_build_model_version": "2.122.2", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "compatibility_basis": "legacy_compatible_model_package", + "certified_by": "policyengine.py bundle certification" + }, + "dataset_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "requirements_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "python": "3.12.14", + "platform": "Golden fixture (synthetic values; café £)", + "runner": "github-actions ubuntu-latest" + }, + "corpus": { + "corpus_version": "uk-fiscal-event-replay/1", + "events": { + "spring_budget_2023": { + "registry_sha256": "5555555555555555555555555555555555555555555555555555555555555555", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_years": [ + { + "year": 2023, + "reason": "before_data_year" + } + ], + "run_manifest_sha256": "6666666666666666666666666666666666666666666666666666666666666666", + "staging_manifest_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "comparison_provenance_sha256": "8888888888888888888888888888888888888888888888888888888888888888", + "artifacts": 8, + "full_supported_grid_complete": true, + "inert_measure_years": [] + } + } + }, + "integrity": { + "status": "complete", + "failures": [] + }, + "diff": { + "base_variant_id": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "diff_sha256": "9999999999999999999999999999999999999999999999999999999999999999", + "flag_counts": { + "numerical": 2, + "structural": 1, + "coverage": 0, + "mapping": 0, + "support": 1, + "inertness": 0, + "construction": 0 + }, + "flagged_rows": [ + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024, + "level": "head", + "key": "universal_credit", + "flags": [ + "numerical" + ], + "base_gbp": -1.5e+16, + "head_gbp": 0.30000000000000004 + }, + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025, + "level": "measure_year", + "key": "total", + "flags": [ + "numerical", + "structural" + ], + "base_gbp": 1e-07, + "head_gbp": -123456789.125 + }, + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023, + "level": "source_row", + "key": "obr-pmd-row-123", + "flags": [ + "support" + ], + "base_gbp": -2500000.0, + "head_gbp": null + } + ] + }, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "path": "requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/", + "manifest_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "receipt_sha256": "5cbc44c1b79506a6410c12b437a7451a0e1387ab6a133b9cb8dc0bf71b396650" +} diff --git a/tests/test_modal_uk_event.py b/tests/test_modal_uk_event.py index 4a75e473..ca032604 100644 --- a/tests/test_modal_uk_event.py +++ b/tests/test_modal_uk_event.py @@ -359,3 +359,66 @@ def concurrent(**kwargs): for key in recorded["env"] ) assert len(recorded["mounts"]) == len(mounts) + 1 # build-time frozen requirements + + +# Imported inside functions the replay worker never calls: the fiscal +# costings CLI's own restage and comparison steps (restage_existing_run and +# main in compute_uk_obr_costings.py). +NOT_ON_THE_WORKER_PATH = {"pipeline/compare_uk_obr_costings.py"} + + +def pipeline_imports(relative: str) -> set[str]: + """Every pipeline module a file imports, at module level or in a function.""" + import ast + + found = set() + for node in ast.walk(ast.parse((runner.ROOT / relative).read_text())): + names = [] + if isinstance(node, ast.ImportFrom) and node.module == "pipeline": + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "pipeline." + ): + names = [node.module.split(".", 1)[1]] + elif isinstance(node, ast.Import): + names = [ + alias.name.split(".", 1)[1] + for alias in node.names + if alias.name.startswith("pipeline.") + ] + for name in names: + candidate = f"pipeline/{name.replace('.', '/')}.py" + if (runner.ROOT / candidate).is_file(): + found.add(candidate) + return found + + +def pipeline_import_closure(start: str) -> set[str]: + seen, queue = set(), [start] + while queue: + relative = queue.pop() + if relative in seen or relative in NOT_ON_THE_WORKER_PATH: + continue + seen.add(relative) + queue.extend(pipeline_imports(relative)) + return seen + + +def test_the_remote_image_holds_everything_the_runner_imports_and_reads(): + # Review of #160: the worker imports compute_uk_event, which imports the + # variants module and reads the variants document; a file missing from + # the allowlist fails every remote run at import. + from pipeline import uk_replay_variants as variants + + allowed = set(runner.repository_files("autumn_budget_2024")) + needed = pipeline_import_closure("pipeline/modal_uk_event.py") + assert "pipeline/uk_replay_variants.py" in needed + assert needed <= allowed, sorted(needed - allowed) + legacy = variants.get_variant() + assert variants.is_legacy(legacy) + read = { + str(variants.VARIANTS_PATH.relative_to(runner.ROOT)), + str(variants.CERTIFIED_BUNDLE.relative_to(runner.ROOT)), + legacy["requirements"]["path"], + } + assert read <= allowed, sorted(read - allowed) diff --git a/tests/test_obr_costings_registry.py b/tests/test_obr_costings_registry.py index 24ed0f12..3958d824 100644 --- a/tests/test_obr_costings_registry.py +++ b/tests/test_obr_costings_registry.py @@ -371,6 +371,10 @@ def test_dividend_lag_overrides_match_installed_engine_dates_per_path(): from policyengine_uk import CountryTaxBenefitSystem except ImportError as exc: pytest.skip(f"policyengine-uk cannot be imported: {exc}") + # The overrides record the dates of the registry's pinned engine; a + # replay environment for another certified variant has other dates. + if compute.package_version("policyengine-uk") != "2.89.2": + pytest.skip("the OBR-costing registry's dividend overrides pin 2.89.2") system = CountryTaxBenefitSystem() rates = system.parameters.gov.hmrc.income_tax.rates diff --git a/tests/test_uk_event_compute.py b/tests/test_uk_event_compute.py index 9ece78ba..ed70ac15 100644 --- a/tests/test_uk_event_compute.py +++ b/tests/test_uk_event_compute.py @@ -16,6 +16,13 @@ from pipeline import stage_uk_event as stage +@pytest.fixture(autouse=True) +def engine_free_environment(monkeypatch): + """These tests stub the engine; the installed-environment check that + precedes every real run is tested in test_uk_replay_variants.py.""" + monkeypatch.setattr(compute, "check_variant_environment", lambda variant: {}) + + def test_optional_engine_provenance_preserves_legacy_metadata(monkeypatch): variable = SimpleNamespace( label="Income tax", @@ -557,7 +564,7 @@ def test_no_digest_no_simulation(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(RuntimeError("missing digest")), + lambda *args: (_ for _ in ()).throw(RuntimeError("missing digest")), ) called = [] monkeypatch.setattr(compute, "run_sim", lambda *args: called.append(args)) @@ -681,7 +688,7 @@ def test_changed_registry_refused_before_preflight(tmp_path, monkeypatch): lambda document: registry.write_text("{}"), ) called = [] - monkeypatch.setattr(compute, "preflight", lambda: called.append("preflight")) + monkeypatch.setattr(compute, "preflight", lambda *args: called.append("preflight")) with pytest.raises(ValueError, match="registry bytes changed"): compute.main(["--event", "test_event", "--registry", str(registry)]) assert not called @@ -707,7 +714,7 @@ def test_compute_reuses_baselines_and_reruns_produce_identical_bytes( monkeypatch.setattr( compute, "preflight", - lambda: { + lambda *args: { "sha256": "a" * 64, "release_bundle": {"certified_data_build_id": "certified-test"}, }, @@ -773,7 +780,7 @@ def test_cli_event_identity_checked_before_preflight(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(AssertionError("preflight must not run")), + lambda *args: (_ for _ in ()).throw(AssertionError("preflight must not run")), ) with pytest.raises(ValueError, match="event identity"): compute.main(["--event", "test_event", "--registry", str(path)]) @@ -794,7 +801,7 @@ def test_focused_default_rerun_guard_precedes_preflight(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(AssertionError("preflight reached")), + lambda *args: (_ for _ in ()).throw(AssertionError("preflight reached")), ) args = ["--event", "test_event", "--measures", measure()["measure_key"]] with pytest.raises(ValueError, match="focused reruns"): @@ -823,7 +830,7 @@ def test_interrupted_compute_resumes_only_receipted_artifacts(tmp_path, monkeypa monkeypatch.setattr( compute, "preflight", - lambda: { + lambda *args: { "sha256": "a" * 64, "release_bundle": {"certified_data_build_id": "certified-test"}, }, diff --git a/tests/test_uk_replay_interface.py b/tests/test_uk_replay_interface.py new file mode 100644 index 00000000..aa30afab --- /dev/null +++ b/tests/test_uk_replay_interface.py @@ -0,0 +1,270 @@ +"""The qualification request/receipt identities shared with policyengine.py.""" + +import copy +import json +import random +from pathlib import Path + +import jsonschema +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import uk_replay_interface as ui + +FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +DIGESTS = json.loads((FIXTURES / "DIGESTS.json").read_text()) + + +def golden(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_golden_request_bytes_and_id_are_pinned(): + request = golden("qualification_request_golden.json") + assert ( + ui.jcs_bytes(request) + == (FIXTURES / "qualification_request_golden.jcs").read_bytes() + ) + assert ( + ui.validate_request(request) + == DIGESTS["qualification_request_golden.json"]["request_id"] + ) + + +def test_golden_receipt_bytes_and_digest_are_pinned(): + receipt = golden("replay_receipt_golden.json") + ui.validate(receipt, ui.RECEIPT_SCHEMA_ID) + body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} + assert ui.jcs_bytes(body) == (FIXTURES / "replay_receipt_golden.jcs").read_bytes() + assert ( + receipt["receipt_sha256"] + == DIGESTS["replay_receipt_golden.json"]["receipt_sha256"] + ) + assert receipt["request_id"] == ui.request_id( + golden("qualification_request_golden.json") + ) + + +def test_jcs_numbers_and_strings_follow_rfc8785_not_json_dumps(): + probe = golden("qualification_request_golden.json")["requested_by"] + assert ui.jcs_bytes(probe["jcs_probe"]) == ( + b"[1e-7,15000000000000000,0.30000000000000004,0,1e+21,5e-324," + b"123456789012345680000,9007199254740991]" + ) + # json.dumps writes 1e-07 and escapes non-ASCII; JCS does neither. + assert json.dumps(1e-7) == "1e-07" + assert ui.jcs_bytes("café £") == '"café £"'.encode() + + +def test_integers_beyond_double_precision_are_refused(): + with pytest.raises(Exception): + ui.jcs_bytes([2**53 + 1]) + + +def test_request_id_is_never_a_request_field(): + request = golden("qualification_request_golden.json") + with pytest.raises(ValueError): + ui.request_id({**request, "request_id": "0" * 64}) + with pytest.raises(jsonschema.ValidationError): + ui.validate_request({**request, "request_id": "0" * 64}) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda r: r.update(kind="nightly"), + lambda r: r.update(attempt=0), + lambda r: r["candidate"]["data"].update(sha256="ABC"), + lambda r: r["candidate"]["data"].update(repo_type="space"), + lambda r: r["candidate"]["policyengine_uk"].pop("wheel_sha256"), + lambda r: r["candidate"].update(extra=True), + lambda r: r["requested_by"].update(commit="481940ec"), + # A released bundle is identified by its PyPI wheel, never a branch. + lambda r: r.update(kind="released_bundle"), + ], +) +def test_request_schema_rejects_malformed_requests(mutate): + request = golden("qualification_request_golden.json") + mutate(request) + with pytest.raises(jsonschema.ValidationError): + ui.validate_request(request) + + +def test_released_bundle_request_names_its_pypi_wheel(): + request = golden("qualification_request_golden.json") + request["kind"] = "released_bundle" + request["candidate"]["policyengine"]["source"] = { + "pypi": { + "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" + } + } + assert len(ui.validate_request(request)) == 64 + + +def test_requested_by_accepts_additional_provenance(): + request = golden("qualification_request_golden.json") + request["requested_by"]["actor"] = "github-actions[bot]" + ui.validate_request(request) + + +def test_rejected_receipt_carries_a_reason_and_no_results(): + receipt = golden("replay_receipt_golden.json") + receipt.update( + status="rejected", + rejection={"code": "release_bundle_mismatch", "detail": "model_version"}, + variant_id=None, + variant_identity_sha256=None, + execution=None, + corpus=None, + diff=None, + integrity={"status": "not_run", "failures": []}, + ) + ui.validate(ui.seal_receipt(receipt), ui.RECEIPT_SCHEMA_ID) + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "rejection": None}), ui.RECEIPT_SCHEMA_ID + ) + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt( + {**receipt, "corpus": golden("replay_receipt_golden.json")["corpus"]} + ), + ui.RECEIPT_SCHEMA_ID, + ) + + +def _rejected(): + receipt = golden("replay_receipt_golden.json") + receipt.update( + status="rejected", + rejection={"code": "incompatible_pair", "detail": "x"}, + variant_id=None, + variant_identity_sha256=None, + execution=None, + corpus=None, + diff=None, + integrity={"status": "not_run", "failures": []}, + ) + return receipt + + +@pytest.mark.parametrize( + "change", + [ + # A rejected request ran nothing and has no identity (review of #160, M6). + {"variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0"}, + {"variant_identity_sha256": "0" * 64}, + {"execution": golden("replay_receipt_golden.json")["execution"]}, + {"integrity": {"status": "complete", "failures": []}}, + {"integrity": {"status": "failed", "failures": []}}, + ], +) +def test_a_rejected_receipt_cannot_claim_a_run(change): + ui.validate(ui.seal_receipt(_rejected()), ui.RECEIPT_SCHEMA_ID) + with pytest.raises(jsonschema.ValidationError): + ui.validate(ui.seal_receipt({**_rejected(), **change}), ui.RECEIPT_SCHEMA_ID) + + +def test_a_complete_receipt_names_its_identity_and_sealed_namespace(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "variant_identity_sha256": None}), + ui.RECEIPT_SCHEMA_ID, + ) + results = {k: v for k, v in receipt["results"].items() if k != "manifest_sha256"} + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "results": results}), ui.RECEIPT_SCHEMA_ID + ) + + +def test_a_failed_receipt_cannot_report_complete_integrity(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + receipt["status"] = "failed" + for status in ("complete", "incomplete", "not_run"): + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt( + {**receipt, "integrity": {"status": status, "failures": []}} + ), + ui.RECEIPT_SCHEMA_ID, + ) + + +def test_execution_python_is_exactly_a_version(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + receipt["execution"] = {**receipt["execution"], "python": "3.12.13; rm -rf"} + with pytest.raises(jsonschema.ValidationError): + ui.validate(ui.seal_receipt(receipt), ui.RECEIPT_SCHEMA_ID) + + +@pytest.mark.parametrize("path", ["../x.h5", "a/../x.h5", "a/..", ".."]) +def test_request_dataset_path_has_no_parent_segments(path): + request = golden("qualification_request_golden.json") + request["candidate"]["data"]["path"] = path + with pytest.raises(jsonschema.ValidationError): + ui.validate_request(request) + + +def test_tampered_receipt_fails_its_digest(): + receipt = golden("replay_receipt_golden.json") + receipt["diff"]["flagged_rows"][0]["head_gbp"] = 0.3 + with pytest.raises(ValueError): + ui.validate(receipt, ui.RECEIPT_SCHEMA_ID) + + +def test_terminal_status_binds_receipt_digest_and_commit(): + status = { + "schema": ui.STATUS_SCHEMA_ID, + "request_id": DIGESTS["qualification_request_golden.json"]["request_id"], + "status": "complete", + "history": [ + { + "status": "queued", + "at": "2026-10-11T12:00:00Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/1", + } + ], + } + with pytest.raises(jsonschema.ValidationError): + ui.validate(status, ui.STATUS_SCHEMA_ID) + status.update(receipt_sha256="a" * 64, receipt_commit="b" * 40) + ui.validate(status, ui.STATUS_SCHEMA_ID) + + +def _reordered(value, rng): + if isinstance(value, dict): + items = list(value.items()) + rng.shuffle(items) + return {k: _reordered(v, rng) for k, v in items} + if isinstance(value, list): + return [_reordered(v, rng) for v in value] + return value + + +@settings(max_examples=60, deadline=None) +@given(st.integers(min_value=0, max_value=2**32)) +def test_request_id_depends_on_content_not_key_order(seed): + request = golden("qualification_request_golden.json") + shuffled = _reordered(request, random.Random(seed)) + assert ui.request_id(shuffled) == ui.request_id(request) + + +@settings(max_examples=60, deadline=None) +@given( + st.floats(allow_nan=False, allow_infinity=False), + st.floats(allow_nan=False, allow_infinity=False), +) +def test_receipt_digest_changes_with_any_flagged_value(a, b): + receipt = golden("replay_receipt_golden.json") + left = copy.deepcopy(receipt) + right = copy.deepcopy(receipt) + left["diff"]["flagged_rows"][0]["head_gbp"] = a + right["diff"]["flagged_rows"][0]["head_gbp"] = b + same = ui.jcs_bytes(a) == ui.jcs_bytes(b) + assert (ui.receipt_sha256(left) == ui.receipt_sha256(right)) == same diff --git a/tests/test_uk_replay_legacy.py b/tests/test_uk_replay_legacy.py index 19f8276e..1ef998ca 100644 --- a/tests/test_uk_replay_legacy.py +++ b/tests/test_uk_replay_legacy.py @@ -4,6 +4,8 @@ import json from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parent.parent @@ -30,3 +32,95 @@ def unavailable(): capsys.readouterr().out == (ROOT / "tests/fixtures/uk_replay_ab2025_dry_run.txt").read_text() ) + + +LEGACY_EVENTS = ( + "autumn_budget_2024", + "autumn_statement_2023", + "spring_budget_2024", + "spring_statement_2025", + "spring_budget_2023", +) + + +@pytest.mark.parametrize("event", LEGACY_EVENTS) +def test_committed_legacy_staging_and_comparison_regenerate_byte_for_byte( + event, tmp_path +): + """The variant work must not move a legacy byte: staging and comparison, + rerun from the committed artifacts, reproduce the committed files.""" + from pipeline import compare_uk_event as compare + from pipeline import compute_uk_event as compute + from pipeline import compute_uk_obr_costings as fiscal + from pipeline import stage_uk_event as stage + + registry_path, artifact_dir = compute.event_paths(event) + payload = registry_path.read_bytes() + manifest = json.loads((artifact_dir / "RUN_MANIFEST.json").read_bytes()) + assert "variant_id" not in manifest and "unsupported_years" not in manifest + rows, tally = stage.stage_event( + json.loads(payload), + manifest, + artifact_dir=artifact_dir, + registry_sha256=hashlib.sha256(payload).hexdigest(), + ) + staged = b"".join( + (json.dumps(row, sort_keys=True, allow_nan=False) + "\n").encode() + for row in rows + ) + assert staged == (artifact_dir / "STAGED.jsonl").read_bytes() + staging_manifest = compute.canonical_bytes( + { + **tally, + "event": event, + "staged_path": fiscal.relative_to_root(artifact_dir / "STAGED.jsonl"), + "staged_sha256": hashlib.sha256(staged).hexdigest(), + } + ) + assert staging_manifest == (artifact_dir / "STAGING_MANIFEST.json").read_bytes() + compare.write_comparison( + event, staged_path=artifact_dir / "STAGED.jsonl", output_dir=tmp_path + ) + written = sorted(path.name for path in tmp_path.iterdir()) + assert written == [ + "COMPARISON.csv", + "COMPARISON.json", + "COMPARISON.md", + "COMPARISON_PROVENANCE.json", + ] + for name in written: + assert (tmp_path / name).read_bytes() == (artifact_dir / name).read_bytes() + + +def test_committed_legacy_summary_regenerates_byte_for_byte(monkeypatch): + from pipeline import compare_uk_event as compare + + written = {} + monkeypatch.setattr( + compare, "atomic_write_bytes", lambda path, data: written.update({path: data}) + ) + root = ROOT / "results" / "uk" / "events" + compare.write_summary(root) + assert list(written) == [root / "SUMMARY.md"] + assert written[root / "SUMMARY.md"] == (root / "SUMMARY.md").read_bytes() + + +def test_legacy_artifacts_and_manifests_carry_no_variant_fields(): + """Every committed legacy artifact, manifest and progress receipt keeps + its pre-variant shape.""" + root = ROOT / "results" / "uk" / "events" + checked = 0 + for event in LEGACY_EVENTS: + for path in sorted((root / event).glob("*.json")): + if path.name.startswith("COMPARISON"): + continue + document = json.loads(path.read_bytes()) + assert isinstance(document, dict) + leaked = [ + key + for key in document + if "variant" in key or "supported" in key or key == "data_years" + ] + assert not leaked, (path.name, leaked) + checked += 1 + assert checked > 100 diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py new file mode 100644 index 00000000..679e8c3d --- /dev/null +++ b/tests/test_uk_replay_variants.py @@ -0,0 +1,1199 @@ +"""Certified replay variants: exact validation, support and namespaces. + +Engine-free and dataset-free: installed distributions are synthesised as +dist-info directories, simulations are stubbed, and the dataset is a few +bytes with a known digest. +""" + +import base64 +import copy +import hashlib +import importlib +import importlib.metadata +import json +import os +import sys +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import compute_uk_event as compute +from pipeline import stage_uk_event as stage +from pipeline import uk_replay_variants as rv + +LEGACY = "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" +CURRENT = "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" +DATASET_SHA256 = rv.get_variant(CURRENT)["identity"]["dataset"]["sha256"] + + +def record(variant_id=CURRENT): + return copy.deepcopy(rv.get_variant(variant_id)) + + +def reseal(r): + """Recompute a mutated record's own identity fields, so only the + consistency rule under test can refuse it.""" + r["identity_sha256"] = rv.identity_sha256(r["identity"]) + r["variant_id"] = rv.variant_slug(r["identity"]) + if r["layout"] == "replay_namespace": + r["result_root"] = f"{rv.REPLAY_RESULT_ROOT}/{r['variant_id']}" + return r + + +def test_committed_variants_validate_and_default_is_legacy(): + document = rv.load_document() + assert document["default_variant"] == LEGACY + assert {v["variant_id"] for v in document["variants"]} == {LEGACY, CURRENT} + for variant in document["variants"]: + rv.validate_record(variant) + assert rv.is_legacy(rv.get_variant()) + assert not rv.is_legacy(rv.get_variant(CURRENT)) + with pytest.raises(rv.VariantError, match="no replay variant"): + rv.get_variant("pe9.9.9_uk9.9.9_unknown") + + +def test_identity_bytes_equal_rfc8785_for_committed_identities(): + rfc8785 = pytest.importorskip("rfc8785") + for variant in rv.load_document()["variants"]: + assert rv.canonical_identity_bytes(variant["identity"]) == rfc8785.dumps( + variant["identity"] + ) + + +INCONSISTENT = [ + ( + "model version", + lambda r: r["identity"]["release_bundle"].update(model_version="2.126.8"), + ), + ( + "wrapper version", + lambda r: r["identity"]["release_bundle"].update(policyengine_version="6.2.7"), + ), + ( + "model package", + lambda r: r["identity"]["release_bundle"].update( + model_package="policyengine-us" + ), + ), + ( + "artifact digest", + lambda r: r["identity"]["release_bundle"].update( + certified_data_artifact_sha256="0" * 64 + ), + ), + ( + "dataset uri", + lambda r: r["identity"]["release_bundle"].update( + default_dataset_uri="hf://x/y.h5@1" + ), + ), + ("dataset bytes", lambda r: r["identity"]["dataset"].update(sha256="1" * 64)), + ("dataset size", lambda r: r["identity"]["dataset"].update(size_bytes=0)), + ( + "certified model", + lambda r: r["certification"].update(certified_for_model_version="2.125.0"), + ), + ( + "certified build", + lambda r: r["certification"].update( + data_build_id="policyengine-uk-data-1.57.0" + ), + ), + ( + "basis", + lambda r: r["certification"].update( + compatibility_basis="exact_build_model_version" + ), + ), + ("materialization", lambda r: r["identity"].update(materialization="download")), + ("python", lambda r: r["identity"].update(python="3")), + ("status", lambda r: r.update(status="promoted")), + ("executor", lambda r: r.update(executor=["kubernetes"])), + ("data years gap", lambda r: r.update(data_years=[2024, 2026])), + ("data years empty", lambda r: r.update(data_years=[])), + ("requirements bytes", lambda r: r["requirements"].update(sha256="2" * 64)), + ( + "requirements file", + lambda r: r["requirements"].update(path="docs/uk_replay/nope.txt"), + ), + ( + "installed hash", + lambda r: r["identity"]["packages"]["policyengine-uk"].update( + record_sha256="short" + ), + ), + ( + "extra package", + lambda r: r["identity"]["packages"].update( + {"policyengine-us": {"version": "1", "record_sha256": "3" * 64}} + ), + ), + ("result root", lambda r: r.update(result_root="results/uk/events")), +] + + +@pytest.mark.parametrize("label,mutate", INCONSISTENT, ids=[x[0] for x in INCONSISTENT]) +def test_each_inconsistent_record_is_refused(label, mutate): + r = record() + mutate(r) + if label != "result root": + reseal(r) + with pytest.raises(rv.VariantError): + rv.validate_record(r) + + +def test_identity_tampering_is_refused_without_resealing(): + r = record() + r["identity"]["packages"]["policyengine-uk"]["record_sha256"] = "4" * 64 + with pytest.raises(rv.VariantError, match="identity_sha256"): + rv.validate_record(r) + r = record() + r["variant_id"] = "pe6.2.6_uk2.125.1_other-data" + with pytest.raises(rv.VariantError, match="variant_id"): + rv.validate_record(r) + + +def test_legacy_record_must_agree_with_certified_bundle(tmp_path): + for relative in ( + "docs/uk_replay/requirements.txt", + "data/uk/certified_bundle.json", + ): + (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / relative).write_bytes((rv.ROOT / relative).read_bytes()) + rv.validate_record(record(LEGACY), root=tmp_path) + pinned = json.loads((tmp_path / "data/uk/certified_bundle.json").read_text()) + pinned["compatible_model_packages"][0]["specifier"] = "==2.90.0" + (tmp_path / "data/uk/certified_bundle.json").write_text(json.dumps(pinned)) + with pytest.raises(rv.VariantError, match="does not declare this model"): + rv.validate_record(record(LEGACY), root=tmp_path) + + +def installed_for(r): + identity = r["identity"] + return { + "packages": { + name: { + "version": identity["packages"][name]["version"], + "record_sha256": identity["packages"][name]["record_sha256"], + } + for name in rv.PACKAGES + }, + "packaged_bundle_manifest_sha256": identity["packaged_bundle_manifest_sha256"], + "python": identity["python"], + "distributions": rv.requirements_pins( + (rv.ROOT / r["requirements"]["path"]).read_text() + ), + } + + +def test_every_pinned_distribution_is_part_of_the_installed_check(): + r = record() + exact = installed_for(r) + for name in ("numpy", "pandas", "h5py"): + changed = copy.deepcopy(exact) + changed["distributions"][name] = "0.0.0" + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=changed) + missing = copy.deepcopy(exact) + del missing["distributions"][name] + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=missing) + extra = copy.deepcopy(exact) + extra["distributions"]["rfc8785"] = "0.1.4" + rv.verify_installed(r, installed=extra) + + +def test_the_identity_binds_the_requirements_freeze(): + r = record() + r["identity"]["requirements_sha256"] = "5" * 64 + reseal(r) + with pytest.raises(rv.VariantError, match="another requirements file"): + rv.validate_record(r) + + +def test_installed_environment_must_match_every_field(): + r = record() + exact = installed_for(r) + assert rv.verify_installed(r, installed=exact) == exact + mutations = [] + for name in rv.PACKAGES: + for field in ("version", "record_sha256"): + mutations.append(("packages", name, field)) + mutations += [("packaged_bundle_manifest_sha256",), ("python",)] + for path in mutations: + changed = copy.deepcopy(exact) + node = changed + for part in path[:-1]: + node = node[part] + node[path[-1]] = "different" + with pytest.raises(rv.VariantError, match="not the certified one"): + rv.verify_installed(r, installed=changed) + + +def test_one_environment_never_validates_two_variants(): + current, legacy = record(CURRENT), record(LEGACY) + assert rv.identity_mismatches(legacy, installed_for(current)) + assert rv.identity_mismatches(current, installed_for(legacy)) + + +def write_distribution(site: Path, files: dict[str, bytes], installer: bytes): + info = site / "demo-1.0.dist-info" + info.mkdir(parents=True) + rows = [] + for relative, payload in { + **files, + "demo-1.0.dist-info/INSTALLER": installer, + }.items(): + path = site / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=") + rows.append(f"{relative},sha256={digest.decode()},{len(payload)}") + rows += ["demo-1.0.dist-info/RECORD,,", "demo/__pycache__/x.cpython-312.pyc,,"] + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: demo\nVersion: 1.0\n") + (info / "RECORD").write_text("\n".join(rows) + "\n") + return importlib.metadata.PathDistribution(info) + + +def test_record_digest_binds_wheel_files_not_the_installer(tmp_path): + files = {"demo/__init__.py": b"x = 1\n", "demo/data/manifest.json": b"{}"} + pip = write_distribution(tmp_path / "pip", files, b"pip\n") + uv = write_distribution(tmp_path / "uv", files, b"uv\n") + assert rv.distribution_record_sha256(pip) == rv.distribution_record_sha256(uv) + other = write_distribution( + tmp_path / "other", {**files, "demo/__init__.py": b"x = 2\n"}, b"uv\n" + ) + assert rv.distribution_record_sha256(other) != rv.distribution_record_sha256(uv) + (tmp_path / "uv" / "demo" / "__init__.py").write_bytes(b"x = 3\n") + with pytest.raises(rv.VariantError, match="differs from its wheel RECORD"): + rv.distribution_record_sha256(uv) + + +def test_certification_is_isolated_from_tokens(): + environ = {"HF_TOKEN": "secret", "HUGGING_FACE_TOKEN": "secret", "PATH": "/bin"} + removed = rv.isolate_certification_environment(environ) + assert sorted(removed) == ["HF_TOKEN", "HUGGING_FACE_TOKEN"] + assert not set(rv.TOKEN_VARIABLES) & set(environ) + assert environ["HF_HUB_OFFLINE"] == "1" and environ["PATH"] == "/bin" + + +def test_release_bundle_must_match_exactly(): + r = record() + bundle = dict(r["identity"]["release_bundle"]) + rv.verify_release_bundle(r, bundle) + for change in ( + {"compatibility_basis": "exact_build_model_version"}, + {"model_version": "2.126.8"}, + {"extra": "field"}, + ): + with pytest.raises(rv.VariantError, match="runtime release bundle differs"): + rv.verify_release_bundle(r, {**bundle, **change}) + missing = dict(bundle) + missing.pop("data_build_fingerprint") + with pytest.raises(rv.VariantError): + rv.verify_release_bundle(r, missing) + + +def tiny_variant(payload: bytes): + r = record() + r["identity"]["dataset"].update( + sha256=hashlib.sha256(payload).hexdigest(), size_bytes=len(payload) + ) + return r + + +def test_materialisation_copies_only_verified_bytes(tmp_path): + payload = b"certified bytes" + r = tiny_variant(payload) + source = tmp_path / "cache" / "enhanced_frs_2024_25.h5" + source.parent.mkdir() + source.write_bytes(payload) + runtime = tmp_path / "runtime" + placed = rv.materialize_dataset(r, runtime, source=source) + assert placed == (runtime / "data" / "enhanced_frs_2024_25.h5").resolve() + assert placed.read_bytes() == payload + assert rv.materialize_dataset(r, runtime, source=tmp_path / "absent") == placed + impostor = tmp_path / "impostor" / "enhanced_frs_2024_25.h5" + impostor.parent.mkdir() + impostor.write_bytes(b"other bytes same len") + placed.unlink() + with pytest.raises(rv.VariantError, match="no local file holds"): + rv.materialize_dataset(r, runtime, source=impostor) + impostor.write_bytes(b"certified bytez") + with pytest.raises(rv.VariantError, match="no local file holds"): + rv.materialize_dataset(r, runtime, source=impostor) + placed.symlink_to(source) + with pytest.raises(rv.VariantError, match="symlink"): + rv.materialize_dataset(r, runtime, source=source) + with pytest.raises(rv.VariantError, match="does not materialise"): + rv.materialize_dataset(record(LEGACY), runtime, source=source) + + +def test_materialisation_finds_the_cache_copy_by_digest(tmp_path, monkeypatch): + payload = b"certified bytes" + r = tiny_variant(payload) + hub = tmp_path / "hub" + snapshots = hub / "models--policyengine--policyengine-uk-data-private" / "snapshots" + (snapshots / "aaa").mkdir(parents=True) + (snapshots / "aaa" / "enhanced_frs_2024_25.h5").write_bytes(b"older release") + (snapshots / "bbb").mkdir() + (snapshots / "bbb" / "enhanced_frs_2024_25.h5").write_bytes(payload) + monkeypatch.setenv("HF_HUB_CACHE", str(hub)) + placed = rv.materialize_dataset(r, tmp_path / "runtime") + assert placed.read_bytes() == payload + + +@settings(max_examples=200, deadline=None) +@given( + st.integers(2015, 2035), + st.integers(0, 10), + st.sets(st.integers(2010, 2040), min_size=1, max_size=12), +) +def test_supported_year_partition_is_exact(first, span, registry_years): + r = {"data_years": list(range(first, first + span + 1))} + supported, unsupported = rv.supported_years(r, registry_years) + unsupported_years = [row["year"] for row in unsupported] + assert sorted(supported + unsupported_years) == sorted(registry_years) + assert not set(supported) & set(unsupported_years) + assert supported == sorted(supported) + assert all(first <= year <= first + span for year in supported) + for row in unsupported: + assert row["reason"] == ( + "before_data_year" if row["year"] < first else "after_extension_end" + ) + + +def test_sealed_namespace_refuses_writes(tmp_path): + namespace = tmp_path / "results" / "uk" / "replays" / CURRENT + (namespace / "spring_budget_2023").mkdir(parents=True) + rv.assert_not_sealed(namespace / "spring_budget_2023", root=tmp_path) + (namespace / rv.RECEIPT_NAME).write_text("{}") + with pytest.raises(rv.VariantError, match="sealed"): + rv.assert_not_sealed(namespace / "spring_budget_2023", root=tmp_path) + with pytest.raises(rv.VariantError, match="sealed"): + rv.assert_not_sealed(namespace / "spring_budget_2023" / "deeper", root=tmp_path) + + +def test_variant_years_refuse_an_explicit_unsupported_year(): + r = record() + registry = {"calendar_years": [2023, 2024, 2025]} + assert compute.validate_variant_years([], registry, r) == [2024, 2025] + with pytest.raises(ValueError, match="outside .*data years"): + compute.validate_variant_years([2023], registry, r) + with pytest.raises(ValueError, match="supports none"): + compute.validate_variant_years([], {"calendar_years": [2022, 2023]}, r) + with pytest.raises(ValueError, match="documented support"): + compute.validate_variant_years([2026], registry, r) + + +def measure(): + rows = [] + for fy, tax, benefit in (("2023-24", 90, -20), ("2024-25", 100, -30)): + rows += [ + { + "source_row_id": f"tax_{fy}", + "fy": fy, + "metric": "revenue_change", + "tax_head": "Income tax", + "value_gbp": tax, + }, + { + "source_row_id": f"benefit_{fy}", + "fy": fy, + "metric": "exchequer_impact", + "tax_head": "Welfare inside cap", + "value_gbp": benefit, + }, + ] + rows.append( + { + "source_row_id": "corporation_2023-24", + "fy": "2023-24", + "metric": "revenue_change", + "tax_head": "Corporation tax", + "value_gbp": 4, + "classification": "out_of_household_scope", + } + ) + return { + "measure_key": "test_event__tax_and_benefit", + "title": "An announcement", + "computability": "partial", + "construction": "reversal_on_certified_world", + "commences_fy": "2023-24", + "pe_baseline_modifier": {"gov.tax.rate": 0.2}, + "head_variables": ["income_tax", "universal_credit"], + "heads": [ + { + "obr_head": "Income tax", + "pe_variables": ["income_tax"], + "channel": "tax", + }, + { + "obr_head": "Welfare inside cap", + "pe_variables": ["universal_credit"], + "channel": "spending", + }, + ], + "source_rows": rows, + } + + +def simulation(tax, benefit): + return { + "aggregates_gbp": { + "income_tax": float(tax), + "universal_credit": float(benefit), + }, + "variable_metadata": { + "income_tax": {"entity": "person", "definition_period": "year"}, + "universal_credit": {"entity": "benunit", "definition_period": "year"}, + }, + "policyengine_bundle": { + "model_version": "2.125.1", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + }, + "dataset_sha256_before": DATASET_SHA256, + "dataset_sha256_after": DATASET_SHA256, + "performance": {"wall_seconds": 1}, + } + + +def test_variant_run_skips_unsupported_years_and_stages_them_visibly( + tmp_path, monkeypatch +): + monkeypatch.setattr(compute, "ROOT", tmp_path) + registry_path = tmp_path / "registry.json" + registry_document = {"calendar_years": [2023, 2024], "measures": [measure()]} + registry_path.write_text(json.dumps(registry_document)) + monkeypatch.setattr(compute.registry_builder, "validate_registry", lambda d: None) + monkeypatch.setattr(compute.worlds, "engine_resolver", lambda: lambda p, d: None) + monkeypatch.setattr( + compute, + "preflight", + lambda *args: (_ for _ in ()).throw( + AssertionError("legacy preflight for a variant") + ), + ) + monkeypatch.setattr( + compute, + "variant_preflight", + lambda variant, dataset_source=None: { + "sha256": variant["identity"]["dataset"]["sha256"], + "release_bundle": { + "certified_data_build_id": "policyengine-uk-data-1.58.0" + }, + }, + ) + years = [] + + def fake_run(year, variables, reform, pre): + years.append(year) + return simulation(900, 1000) if reform else simulation(1000, 1030) + + monkeypatch.setattr(compute, "run_sim", fake_run) + output = tmp_path / "results" / "uk" / "replays" / CURRENT / "test_event" + args = [ + "--event", + "test_event", + "--variant", + CURRENT, + "--registry", + str(registry_path), + "--output-dir", + str(output), + ] + assert compute.main(args) == 0 + assert set(years) == {2024} + manifest = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + assert manifest["variant_id"] == CURRENT + assert manifest["variant_identity_sha256"] == record()["identity_sha256"] + assert manifest["supported_years"] == [2024] + assert manifest["unsupported_years"] == [ + {"year": 2023, "reason": "before_data_year"} + ] + assert manifest["full_event_complete"] is False + assert manifest["full_supported_grid_complete"] is True + artifact = json.loads( + (output / "test_event__tax_and_benefit_2024.json").read_bytes() + ) + assert artifact["variant_id"] == CURRENT + rows, tally = stage.stage_event( + registry_document, + manifest, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + by_id = {row["source_row_id"]: row for row in rows} + assert by_id["tax_2024-25"]["pe_value"] == 100 + assert by_id["tax_2023-24"]["status"] == "unsupported_on_variant" + assert by_id["tax_2023-24"]["pe_value"] is None + assert "2024–2030" in by_id["tax_2023-24"]["reason"] + assert by_id["corporation_2023-24"]["computability"] == "out_of_household_scope" + assert by_id["corporation_2023-24"]["status"] == "not_computed" + assert tally["unsupported_source_rows"] == 2 + assert tally["unsupported_years"] == [2023] + assert tally["full_supported_grid_complete"] is True + assert tally["source_rows"] == tally["staged_rows"] == len(rows) == 5 + # Resume within the same variant reuses receipted artifacts. + years.clear() + assert compute.main(args + ["--resume"]) == 0 + assert years == [] + # A forged unsupported-year artifact is refused by staging. + forged = copy.deepcopy(manifest) + forged["unsupported_years"] = [{"year": 2024, "reason": "before_data_year"}] + with pytest.raises( + ValueError, match="does not support|supported grid|supported years differ" + ): + stage.stage_event( + registry_document, + forged, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + other = copy.deepcopy(manifest) + other["variant_identity_sha256"] = "f" * 64 + with pytest.raises(ValueError, match="variant identity"): + stage.stage_event( + registry_document, + other, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + (output.parent / rv.RECEIPT_NAME).write_text("{}") + with pytest.raises(rv.VariantError, match="sealed"): + compute.main(args + ["--resume"]) + + +def test_dataset_year_guard_refuses_years_without_inputs(monkeypatch, tmp_path): + from pipeline import compute_uk_obr_costings as fiscal + + data_file = tmp_path / "data.h5" + data_file.write_bytes(b"x") + digest = hashlib.sha256(b"x").hexdigest() + + class Sim: + policyengine_bundle = { + "certified_data_build_id": "build", + "runtime_dataset_source": str(data_file), + } + tax_benefit_system = type("System", (), {"variables": {"income_tax": None}})() + dataset = type("Dataset", (), {"years": [2024, 2025]})() + + def calculate(self, name, period): + import numpy as np + + return np.array([1.0]) + + class Uk: + @staticmethod + def managed_microsimulation(**kwargs): + return Sim() + + import sys + import types + + monkeypatch.setitem(sys.modules, "policyengine", types.SimpleNamespace(uk=Uk)) + monkeypatch.setattr(fiscal, "variable_metadata", lambda *a, **k: {}) + ok = fiscal.run_managed_simulation( + year=2024, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=digest, + ) + assert ok["aggregates_gbp"] == {"income_tax": 1.0} + with pytest.raises(RuntimeError, match="outside the simulation dataset's years"): + fiscal.run_managed_simulation( + year=2023, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=digest, + ) + + +# --- review of #160 ------------------------------------------------------------ + + +def staged_variant_run(tmp_path, monkeypatch): + """A one-measure variant run over CY2023–2024 (2023 unsupported).""" + monkeypatch.setattr(compute, "ROOT", tmp_path) + registry_path = tmp_path / "registry.json" + registry = {"calendar_years": [2023, 2024], "measures": [measure()]} + registry_path.write_text(json.dumps(registry)) + monkeypatch.setattr(compute.registry_builder, "validate_registry", lambda d: None) + monkeypatch.setattr(compute.worlds, "engine_resolver", lambda: lambda p, d: None) + monkeypatch.setattr( + compute, + "variant_preflight", + lambda variant, dataset_source=None: { + "sha256": variant["identity"]["dataset"]["sha256"], + "release_bundle": { + "certified_data_build_id": "policyengine-uk-data-1.58.0" + }, + }, + ) + monkeypatch.setattr( + compute, + "run_sim", + lambda year, variables, reform, pre: ( + simulation(900, 1000) if reform else simulation(1000, 1030) + ), + ) + output = tmp_path / "results" / "uk" / "replays" / CURRENT / "test_event" + args = [ + "--event", + "test_event", + "--variant", + CURRENT, + "--registry", + str(registry_path), + "--output-dir", + str(output), + ] + assert compute.main(args) == 0 + manifest = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + digest = hashlib.sha256(registry_path.read_bytes()).hexdigest() + return registry, manifest, output, digest, args + + +@pytest.mark.parametrize( + "label,forge", + [ + # B1: a self-consistent manifest that claims 2023 is supported. + ( + "no unsupported years", + lambda m: m.update(unsupported_years=[], supported_years=[2023, 2024]), + ), + ("other data years", lambda m: m.update(data_years=[2023, 2030])), + ("other identity", lambda m: m.update(variant_identity_sha256="f" * 64)), + ("unknown variant", lambda m: m.update(variant_id="pe9_uk9_x")), + ("other dataset", lambda m: m.update(certified_dataset_sha256="b" * 64)), + ("other data bundle", lambda m: m.update(data_bundle="another-build")), + # Deleting variant_id must not turn the identity check off. + ("variant fields dropped", lambda m: m.pop("variant_id")), + ], +) +def test_staging_checks_a_manifests_variant_claims_against_the_record( + tmp_path, monkeypatch, label, forge +): + registry, manifest, output, digest, _ = staged_variant_run(tmp_path, monkeypatch) + stage.stage_event(registry, manifest, artifact_dir=output, registry_sha256=digest) + forged = copy.deepcopy(manifest) + forge(forged) + with pytest.raises((ValueError, rv.VariantError)): + stage.stage_event(registry, forged, artifact_dir=output, registry_sha256=digest) + + +def test_a_legacy_manifest_lists_no_variant_fields(): + registry = {"calendar_years": [2023], "measures": []} + for fields in ( + {"unsupported_years": [{"year": 2023, "reason": "before_data_year"}]}, + {"variant_identity_sha256": "a" * 64}, + ): + with pytest.raises(ValueError, match="legacy compute manifest"): + stage.manifest_variant(fields, registry, variants_path=rv.VARIANTS_PATH) + assert stage.manifest_variant({}, registry, variants_path=rv.VARIANTS_PATH) is None + + +def test_resume_refuses_a_receipt_for_an_unsupported_year(tmp_path, monkeypatch): + # B2: a receipt (from an older revision, or edited) naming a 2023 + # artifact must not put that year into a variant's manifest. + registry, manifest, output, digest, args = staged_variant_run(tmp_path, monkeypatch) + forged = output / "test_event__tax_and_benefit_2023.json" + forged.write_bytes( + (output / "test_event__tax_and_benefit_2024.json") + .read_bytes() + .replace(b'"year": 2024', b'"year": 2023') + ) + manifest["artifacts"][str(forged.relative_to(tmp_path))] = hashlib.sha256( + forged.read_bytes() + ).hexdigest() + (output / "RUN_MANIFEST.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="outside the variant's supported years"): + compute.main(args + ["--resume"]) + # A progress receipt for 2023 is never read, even one whose identity + # fields match this run exactly (it would be accepted for 2024). + (output / "RUN_MANIFEST.json").unlink() + genuine = json.loads((output / "RUN_PROGRESS_2024.json").read_bytes()) + (output / "RUN_PROGRESS_2023.json").write_text( + json.dumps( + { + **genuine, + "year": 2023, + "artifacts": { + str(forged.relative_to(tmp_path)): hashlib.sha256( + forged.read_bytes() + ).hexdigest() + }, + } + ) + ) + assert compute.main(args + ["--resume"]) == 0 + rerun = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + assert all("_2023.json" not in path for path in rerun["artifacts"]) + + +def test_staging_and_comparison_refuse_a_sealed_namespace(tmp_path, monkeypatch): + # B3: the staged rows and comparison are part of what the receipt binds. + from pipeline import compare_uk_event as compare + + registry, manifest, output, digest, _ = staged_variant_run(tmp_path, monkeypatch) + (output.parent / rv.RECEIPT_NAME).write_text("{}") + monkeypatch.setattr(rv, "ROOT", tmp_path) + monkeypatch.setattr( + stage.compute, "event_paths", lambda event: (tmp_path / "registry.json", output) + ) + with pytest.raises(rv.VariantError, match="sealed"): + stage.main(["--event", "test_event", "--artifact-dir", str(output)]) + assert not (output / "STAGED.jsonl").exists() + with pytest.raises(rv.VariantError, match="sealed"): + compare.write_summary(output.parent, artifact_root=tmp_path) + + +def test_population_note_names_the_variant_population(): + assert stage.population_note(None) == stage.LEGACY_POPULATION_NOTE + assert "single certified 2023 population" in stage.LEGACY_POPULATION_NOTE + note = stage.population_note(record()) + assert "enhanced_frs_2024_25.h5" in note and "1.58.0" in note + assert "2023" not in note + + +def test_only_certified_variants_run_unless_qualifying(tmp_path): + document = rv.load_document() + candidate = copy.deepcopy(document) + for r in candidate["variants"]: + if r["variant_id"] == CURRENT: + r["status"] = "candidate" + path = tmp_path / "variants.json" + path.write_text(json.dumps(candidate)) + with pytest.raises(rv.VariantError, match="not certified"): + rv.get_variant(CURRENT, path=path) + assert rv.get_variant(CURRENT, path=path, allow_uncertified=True)["status"] == ( + "candidate" + ) + candidate["default_variant"] = CURRENT + path.write_text(json.dumps(candidate)) + with pytest.raises(rv.VariantError, match="default_variant must be"): + rv.load_document(path) + + +def test_the_cli_refuses_a_candidate_without_the_qualification_flag( + tmp_path, monkeypatch +): + document = rv.load_document() + for r in document["variants"]: + if r["variant_id"] == CURRENT: + r["status"] = "candidate" + path = tmp_path / "variants.json" + path.write_text(json.dumps(document)) + args = ["--event", "autumn_budget_2024", "--variant", CURRENT, "--dry-run"] + with pytest.raises(rv.VariantError, match="not certified"): + compute.main(args + ["--variants-file", str(path)]) + + +def test_the_legacy_variant_refuses_a_dataset_file(): + # M5: the legacy variant reads its Hugging Face cache; a file would be + # silently ignored. + with pytest.raises(ValueError, match="--dataset-file"): + compute.main( + ["--event", "autumn_budget_2024", "--dataset-file", "x.h5", "--dry-run"] + ) + + +def test_a_legacy_run_checks_its_installed_environment_first(monkeypatch): + # M4: the legacy record's digests are checked before its preflight. + calls = [] + monkeypatch.setattr( + compute, + "check_variant_environment", + lambda variant: ( + calls.append(variant["variant_id"]) + or (_ for _ in ()).throw(rv.VariantError("not the certified environment")) + ), + ) + monkeypatch.setattr( + compute, + "preflight", + lambda *a: (_ for _ in ()).throw(AssertionError("preflight before check")), + ) + monkeypatch.setattr(rv, "assert_not_sealed", lambda path: None) + with pytest.raises(rv.VariantError, match="not the certified environment"): + compute.main(["--event", "autumn_budget_2024"]) + assert calls == [LEGACY] + + +@pytest.mark.parametrize( + "label,mutate", + [ + ("dataset repo", lambda r: r["identity"]["dataset"].update(repo_id="a/b")), + ("dataset revision", lambda r: r["identity"]["dataset"].update(revision="9")), + ("dataset path", lambda r: r["identity"]["dataset"].update(path="other.h5")), + ( + "dataset repo type", + lambda r: r["identity"]["dataset"].update(repo_type="dataset"), + ), + ("layout", lambda r: r.update(layout="flat")), + ("bool data year", lambda r: r.update(data_years=[True, 2])), + ("certification", lambda r: r.update(certification="certified")), + ("identity", lambda r: r.update(identity=[])), + ], +) +def test_single_field_changes_are_refused(label, mutate): + r = record() + mutate(r) + with pytest.raises(rv.VariantError): + rv.validate_record(r) + + +def test_a_wrong_legacy_result_root_is_refused(): + r = record(LEGACY) + r["result_root"] = f"{rv.REPLAY_RESULT_ROOT}/{LEGACY}" + with pytest.raises(rv.VariantError, match="result_root"): + rv.validate_record(r) + + +def test_a_requirements_pin_that_disagrees_is_refused(tmp_path): + r = record() + text = (rv.ROOT / r["requirements"]["path"]).read_text() + changed = text.replace("policyengine-uk==2.125.1", "policyengine-uk==2.125.2") + assert changed != text + path = tmp_path / "docs" / "requirements.txt" + path.parent.mkdir(parents=True) + path.write_text(changed) + r["requirements"] = { + "path": "docs/requirements.txt", + "sha256": hashlib.sha256(changed.encode()).hexdigest(), + } + r["identity"]["requirements_sha256"] = r["requirements"]["sha256"] + reseal(r) + with pytest.raises(rv.VariantError, match="requirements pin policyengine-uk"): + rv.validate_record(r, root=tmp_path) + + +def demo_site(tmp_path, monkeypatch): + files = { + "demo/__init__.py": b"x = 1\n", + "demo/parameters/rate.yaml": b"values: {2024-01-01: 0.2}\n", + } + site = tmp_path / "site" + write_distribution(site, files, b"uv\n") + monkeypatch.syspath_prepend(str(site)) + importlib.invalidate_caches() + return site + + +def test_an_unlisted_file_in_a_package_directory_is_found(tmp_path, monkeypatch): + # B4: core loads every file in a parameter directory, so a file no RECORD + # lists can change results while every listed file still verifies. + site = demo_site(tmp_path, monkeypatch) + assert rv.unlisted_package_files(("demo",)) == {} + (site / "demo" / "__pycache__").mkdir() + (site / "demo" / "__pycache__" / "y.cpython-312.pyc").write_bytes(b"\0") + assert rv.unlisted_package_files(("demo",)) == {} # bytecode is never read + (site / "demo" / "parameters" / "extra.yaml").write_text("values: {}\n") + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/extra.yaml"] + } + + +def test_the_scan_sees_what_core_would_load(tmp_path, monkeypatch): + # Review of dc040de: core's loaders recurse into __pycache__ and follow + # symlinked directories, and only the package's own RECORD is hash-checked. + site = demo_site(tmp_path, monkeypatch) + cache = site / "demo" / "parameters" / "__pycache__" + cache.mkdir() + (cache / "ok.cpython-312.pyc").write_bytes(b"\0") + (cache / "x.py").write_text("raise SystemExit\n") + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/__pycache__/x.py"] + } + (cache / "x.py").unlink() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "rate.yaml").write_text("values: {}\n") + (site / "demo" / "parameters" / "linked").symlink_to( + outside, target_is_directory=True + ) + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/linked (symlink)"] + } + (site / "demo" / "parameters" / "linked").unlink() + # A symlink to a listed file is still a symlink, not that file. + (site / "demo" / "alias.py").symlink_to(site / "demo" / "__init__.py") + assert rv.unlisted_package_files(("demo",)) == {"demo": ["demo/alias.py (symlink)"]} + (site / "demo" / "alias.py").unlink() + # Another distribution listing a file inside this package does not count. + extra = site / "demo" / "parameters" / "from_other.yaml" + extra.write_text("values: {}\n") + info = site / "other-1.0.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: other\nVersion: 1.0\n") + (info / "RECORD").write_text("demo/parameters/from_other.yaml,,\n") + importlib.invalidate_caches() + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/from_other.yaml"] + } + + +def test_startup_files_must_belong_to_a_pinned_distribution(tmp_path, monkeypatch): + # site.py runs a .pth file's import lines before any check here. + site = demo_site(tmp_path, monkeypatch) + (site / "_virtualenv.pth").write_text("import _virtualenv\n") + (site / "hook.pth").write_text("import os; os.environ['X'] = '1'\n") + files = rv.startup_files(("demo",)) + assert [f["name"] for f in files] == ["_virtualenv.pth", "hook.pth"] + assert rv.startup_file_problems(files, {"demo": "1.0"}) == [ + "startup file hook.pth belongs to no pinned distribution" + ] + # The shim's name with other content is not the shim. + (site / "_virtualenv.pth").write_text("import evil\n") + assert len(rv.startup_file_problems(rv.startup_files(("demo",)), {})) == 2 + # A .pth a pinned distribution lists is that distribution's own. + owned = [{"name": "x.pth", "owner": "demo", "text": "import x", "symlink": False}] + assert rv.startup_file_problems(owned, {"demo": "1.0"}) == [] + assert rv.startup_file_problems(owned, {}) != [] + r = record() + installed = installed_for(r) + installed["startup_files"] = files + with pytest.raises(rv.VariantError, match="startup file hook.pth"): + rv.verify_installed(r, installed=installed) + + +def test_unlisted_files_fail_the_installed_check(): + r = record() + installed = installed_for(r) + rv.verify_installed(r, installed=installed) + installed["unlisted_files"] = {"policyengine-uk": ["policyengine_uk/x.yaml"]} + with pytest.raises(rv.VariantError, match="files outside its RECORD"): + rv.verify_installed(r, installed=installed) + + +def test_bytecode_isolation_precedes_engine_import(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + monkeypatch.setattr(sys, "dont_write_bytecode", sys.dont_write_bytecode) + environ = {} + for module in rv.ENGINE_MODULES.values(): + monkeypatch.delitem(sys.modules, module, raising=False) + stale = tmp_path / "no-bytecode" / "policyengine_uk" + stale.mkdir(parents=True) + (stale / "x.pyc").write_bytes(b"\0") + prefix = rv.isolate_bytecode(tmp_path, environ) + assert sys.pycache_prefix == str(prefix) == environ["PYTHONPYCACHEPREFIX"] + assert sys.dont_write_bytecode and environ["PYTHONDONTWRITEBYTECODE"] == "1" + assert list(prefix.iterdir()) == [] # a stale cache is cleared + monkeypatch.setitem(sys.modules, "policyengine_uk", object()) + with pytest.raises(rv.VariantError, match="imported before isolation"): + rv.isolate_bytecode(tmp_path, environ) + + +def test_bytecode_isolation_refuses_a_symlinked_directory(tmp_path, monkeypatch): + # The guard runs before anything resolves or removes the path: a + # symlinked no-bytecode directory must not get its target deleted. + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + monkeypatch.setattr(sys, "dont_write_bytecode", sys.dont_write_bytecode) + for module in rv.ENGINE_MODULES.values(): + monkeypatch.delitem(sys.modules, module, raising=False) + precious = tmp_path / "precious" + precious.mkdir() + (precious / "keep.txt").write_text("keep") + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "no-bytecode").symlink_to(precious, target_is_directory=True) + with pytest.raises(rv.VariantError, match="symlink"): + rv.isolate_bytecode(runtime, {}) + assert (precious / "keep.txt").read_text() == "keep" + + +def test_an_engine_imported_from_another_tree_is_refused(tmp_path, monkeypatch): + site = demo_site(tmp_path, monkeypatch) + monkeypatch.setitem(rv.ENGINE_MODULES, "demo", "demo") + module = type(sys)("demo") + module.__file__ = str(site / "demo" / "__init__.py") + monkeypatch.setitem(sys.modules, "demo", module) + rv.verify_import_origins(("demo",)) + shadow = tmp_path / "shadow" / "demo" / "__init__.py" + shadow.parent.mkdir(parents=True) + shadow.write_text("x = 2\n") + module.__file__ = str(shadow) + with pytest.raises(rv.VariantError, match="not its verified distribution"): + rv.verify_import_origins(("demo",)) + + +def test_isolation_routes_every_http_request_to_a_closed_port(): + # M2: requests ignores HF_HUB_OFFLINE but reads these variables + # (urllib.request.getproxies), so the release-manifest fetch fails and + # the wrapper uses its packaged certification. + import urllib.request + + environ = {"NO_PROXY": "*", "no_proxy": "huggingface.co", "PATH": "/bin"} + removed = rv.isolate_certification_environment(environ) + assert {"NO_PROXY", "no_proxy"} <= set(removed) + original = dict(os.environ) + try: + os.environ.clear() + os.environ.update(environ) + proxies = urllib.request.getproxies() + finally: + os.environ.clear() + os.environ.update(original) + assert proxies["https"] == proxies["http"] == "http://127.0.0.1:9" + assert "no" not in proxies + + +def test_a_symlinked_data_directory_is_refused(tmp_path): + payload = b"certified bytes" + r = tiny_variant(payload) + source = tmp_path / "source.h5" + source.write_bytes(payload) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "data").symlink_to(elsewhere, target_is_directory=True) + with pytest.raises(rv.VariantError, match="symlink"): + rv.materialize_dataset(r, runtime, source=source) + assert list(elsewhere.iterdir()) == [] + + +def test_a_dataset_that_does_not_state_its_years_fails_when_required( + monkeypatch, tmp_path +): + # M3: a variant's simulation fails closed when its dataset hides its years. + from pipeline import compute_uk_obr_costings as fiscal + + data_file = tmp_path / "data.h5" + data_file.write_bytes(b"x") + + class Sim: + policyengine_bundle = { + "certified_data_build_id": "build", + "runtime_dataset_source": str(data_file), + } + tax_benefit_system = type("System", (), {"variables": {"income_tax": None}})() + dataset = object() + + def calculate(self, name, period): + import numpy as np + + return np.array([1.0]) + + class Uk: + @staticmethod + def managed_microsimulation(**kwargs): + return Sim() + + import types + + monkeypatch.setitem(sys.modules, "policyengine", types.SimpleNamespace(uk=Uk)) + monkeypatch.setattr(fiscal, "variable_metadata", lambda *a, **k: {}) + kwargs = dict( + year=2024, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=hashlib.sha256(b"x").hexdigest(), + ) + assert fiscal.run_managed_simulation(**kwargs)["aggregates_gbp"] # legacy + with pytest.raises(RuntimeError, match="does not state its years"): + fiscal.run_managed_simulation(**kwargs, require_dataset_years=True) + + +def test_the_installed_engine_is_a_committed_variant_when_present(): + """In a pinned venv (.venv-candidate, .venv-replay, and the CI + replay-engine job once the receiver change adds it), the real installed + environment reproduces a committed record's digests, with no unlisted + files. Without an engine installed this test skips.""" + try: + version = importlib.metadata.version("policyengine-uk") + except importlib.metadata.PackageNotFoundError: + pytest.skip("no engine installed") + matches = [ + r + for r in rv.load_document()["variants"] + if r["identity"]["packages"]["policyengine-uk"]["version"] == version + ] + if not matches: + pytest.skip(f"policyengine-uk {version} is no committed variant") + installed = rv.installed_identity() + assert installed["unlisted_files"] == {} + rv.verify_installed(matches[0], installed=installed) + + +def test_the_legacy_preflight_is_isolated_and_checks_its_release_bundle(monkeypatch): + # Review of dc040de, M4: the legacy run uses the packaged certification + # and must match its record's release bundle exactly, like any variant. + calls = [] + legacy = record(LEGACY) + bundle = dict(legacy["identity"]["release_bundle"]) + monkeypatch.setattr(compute.fiscal, "configure_offline", lambda: None) + monkeypatch.setattr(compute.worlds, "configure_offline", lambda: None) + monkeypatch.setattr( + rv, "isolate_certification_environment", lambda: calls.append("certification") + ) + monkeypatch.setattr(rv, "isolate_bytecode", lambda root: calls.append("bytecode")) + monkeypatch.setattr( + compute.worlds, + "preflight", + lambda: { + "sha256": legacy["identity"]["dataset"]["sha256"], + "revision": bundle["certified_data_build_id"], + }, + ) + managed = { + "sha256": legacy["identity"]["dataset"]["sha256"], + "release_bundle": bundle, + } + monkeypatch.setattr( + compute.fiscal, + "preflight_certified_dataset", + lambda: calls.append("engine") or dict(managed), + ) + monkeypatch.setattr(compute, "import_engine", lambda: calls.append("origins")) + pre = compute.preflight(legacy) + assert calls == ["certification", "bytecode", "engine", "origins"] + assert pre["check_import_origins"] is True + managed["release_bundle"] = {**bundle, "compatibility_basis": "other"} + with pytest.raises(rv.VariantError, match="runtime release bundle differs"): + compute.preflight(legacy) + # Without a record (the Modal preflight-only path) it is the original check. + calls.clear() + managed["release_bundle"] = bundle + assert "check_import_origins" not in compute.preflight() + assert calls == ["engine"] + + +def test_guards_check_the_environment_and_compile_from_source(monkeypatch): + calls = [] + monkeypatch.setattr( + compute, "check_variant_environment", lambda variant: calls.append("installed") + ) + monkeypatch.setattr(rv, "isolate_bytecode", lambda root: calls.append("bytecode")) + monkeypatch.setattr( + compute.worlds, + "engine_resolver", + lambda: calls.append("engine") or (lambda path, date: None), + ) + monkeypatch.setattr(compute.worlds, "reversal_delta_mismatches", lambda *a, **k: []) + monkeypatch.setattr(compute, "import_engine", lambda: calls.append("origins")) + assert compute.main(["--event", "autumn_budget_2024", "--guards"]) == 0 + assert calls == ["installed", "bytecode", "engine", "origins"] + + +def test_a_shadowed_distribution_copy_does_not_count(tmp_path, monkeypatch): + # Found on Modal: the container's own client packages sit behind the + # pinned environment on sys.path; Python imports the first copy. + for index, version in enumerate(("2.0", "1.0")): + info = tmp_path / f"site{index}" / f"shadowed-{version}.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowed\nVersion: {version}\n" + ) + monkeypatch.syspath_prepend(str(tmp_path / "site1")) + monkeypatch.syspath_prepend(str(tmp_path / "site0")) # first on the path + importlib.invalidate_caches() + assert importlib.metadata.version("shadowed") == "2.0" + assert rv.installed_distributions()["shadowed"] == "2.0"