From c9ab576926d44d76810283e916b847d80cbffbbd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:05:47 -0400 Subject: [PATCH 1/9] Design: replay the event corpus on every certified release Contract for certified replay variants, exact validation, unsupported years, per-release namespaces, version-scoped diagnoses, construction premises, the release diff and the W4 qualification interface. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/QUALIFICATION_INTERFACE.md | 189 +++++++++++++++ docs/uk_replay/RELEASE_REPLAY.md | 282 ++++++++++++++++++++++ 2 files changed, 471 insertions(+) create mode 100644 docs/uk_replay/QUALIFICATION_INTERFACE.md create mode 100644 docs/uk_replay/RELEASE_REPLAY.md diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md new file mode 100644 index 0000000..0402d6b --- /dev/null +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -0,0 +1,189 @@ +# Qualification request and replay receipt (v1, proposed) + +Status: proposal from W6 (policyengine-scorecard) to W4 (policyengine.py +qualification), 11 October 2026, for agreement by 15 October. JSON Schemas: +[`schemas/qualification_request.v1.json`](schemas/qualification_request.v1.json) +and [`schemas/replay_receipt.v1.json`](schemas/replay_receipt.v1.json). + +Two flows use the same receiver: + +* **Pre-promotion qualification.** W4's certify workflow, holding a candidate + model/data pair it has not yet promoted, sends an immutable request. W6 + validates the candidate's identity and compatibility, reruns the supported + event corpus, diffs it against the latest certified variant, and returns an + artifact-bound receipt that qualification consumes. +* **Post-promotion consumer dispatch.** After a policyengine.py release, the + existing dispatcher notifies consumers. Scorecard treats a released bundle + it has not replayed as a request with `kind: "released_bundle"`, so it + ends up with a certified variant and a sealed namespace. + +## Transport + +* W4 sends `repository_dispatch` to `PolicyEngine/policyengine-scorecard` + with `event_type: "uk-replay-qualification-request"` and + `client_payload: {"request": }`. W4 alone edits the policyengine.py + dispatcher and its GitHub App token scope; scorecard needs + `contents: read` on policyengine.py (public) and nothing else from W4. +* GitHub keeps at most one pending run per concurrency group and cancels + older pending runs, so the receiver's first step writes the request to the + persistent queue on the `replay-state` branch before anything else. A + scheduled catch-up (every 3 hours) drains the queue and discovers + policyengine.py releases that were never dispatched. +* State for request `` lives at + `replay-state:requests//`: + `REQUEST.json` (the bytes received), `STATUS.json` + (`queued` → `running` → `complete` | `rejected` | `failed`, with + timestamps and the Actions run URL), and on completion + `REPLAY_RECEIPT.json`. +* W4 polls `GET /repos/PolicyEngine/policyengine-scorecard/contents/requests//STATUS.json?ref=replay-state` + (the repository is public). A receipt is addressed by the request's own + digest, so W4 needs no callback and no scorecard token. +* Duplicate requests (same `request_id`) are idempotent: an existing + terminal status is returned, never rerun. A failed request can be retried + only by a new request (a different `attempt`). + +## Request + +```json +{ + "schema": "policyengine.uk-replay-qualification-request/v1", + "kind": "candidate" , + "attempt": 1, + "candidate": { + "policyengine": { + "version": "6.2.7", + "source": {"git": {"repository": "PolicyEngine/policyengine.py", "commit": "<40-hex>"}}, + "packaged_bundle_manifest_sha256": "<64-hex>" + }, + "policyengine_uk": {"version": "2.126.8", "wheel_sha256": "<64-hex>"}, + "policyengine_core": {"version": "3.33.0", "wheel_sha256": "<64-hex>"}, + "data": { + "build_id": "policyengine-uk-data-1.58.0", + "dataset": "enhanced_frs_2024_25", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "path": "enhanced_frs_2024_25.h5", + "sha256": "<64-hex>", + "size_bytes": 128369110, + "release_manifest_sha256": "<64-hex>" + }, + "expected_release_bundle": { "...": "the exact pe.uk.uk_latest.release_bundle under packaged certification" } + }, + "compare_to": "latest_certified", + "requested_by": { + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 1234567890, + "commit": "<40-hex>" + } +} +``` + +* `request_id` is not a field: it is the SHA-256 of the canonical request + JSON (sorted keys, no whitespace, UTF-8). Both sides compute it the same + way; `tests/fixtures/` holds a shared golden example. +* The wrapper is identified by `version` plus either a PyPI wheel + (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose + regenerated manifest is not on PyPI yet, a public git commit. Scorecard + installs exactly that, then checks the installed packaged manifest's SHA-256. +* Model and core are PyPI wheels pinned by SHA-256. +* `kind` is `candidate` or `released_bundle`. +* `compare_to` is `latest_certified` (default) or a `variant_id`. + +## What scorecard validates before running + +The candidate becomes a `candidate` variant record (RELEASE_REPLAY.md). It is +**rejected**, with a code, when: + +| Code | Condition | +|---|---| +| `schema_invalid` | The request fails the v1 schema. | +| `artifact_hash_mismatch` | A downloaded wheel, the installed packaged manifest or the dataset bytes differ from the request. | +| `release_bundle_mismatch` | `pe.uk.uk_latest.release_bundle` under packaged certification differs from `expected_release_bundle`. | +| `incompatible_pair` | The packaged certification does not name the candidate model version, or the dataset is not the bundle's certified artifact. | +| `data_access_failed` | The dataset could not be fetched with the receiver's credential. | +| `no_supported_years` | No event year falls in the dataset's supported years. | + +A rejected request has no numerical results; its receipt says why. + +## Receipt + +```json +{ + "schema": "policyengine.uk-replay-receipt/v1", + "request_id": "<64-hex>", + "status": "complete", + "rejection": null, + "variant_id": "pe6.2.7_uk2.126.8_policyengine-uk-data-1.58.0", + "variant_identity_sha256": "<64-hex>", + "execution": { + "packages": {"policyengine": {"version": "6.2.7", "record_sha256": "…"}, "...": "..."}, + "packaged_bundle_manifest_sha256": "…", + "release_bundle": {"...": "..."}, + "dataset_sha256": "…", + "requirements_sha256": "…", + "python": "3.12.13", + "platform": "Linux-…", + "runner": "github-actions ubuntu-latest" + }, + "corpus": { + "corpus_version": "uk-fiscal-event-replay/1", + "events": { + "autumn_budget_2024": { + "registry_sha256": "…", + "supported_years": [2024, 2025, 2026, 2027, 2028, 2029], + "unsupported_years": [], + "run_manifest_sha256": "…", + "staging_manifest_sha256": "…", + "comparison_provenance_sha256": "…", + "artifacts": 30, + "full_supported_grid_complete": true, + "inert_measure_years": [] + } + } + }, + "integrity": { + "status": "complete", + "failures": [] + }, + "diff": { + "base_variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "diff_sha256": "…", + "flag_counts": {"numerical": 0, "structural": 0, "coverage": 0, "mapping": 0, "support": 0, "inertness": 0, "construction": 0}, + "flagged_rows": [ {"event": "…", "measure_key": "…", "year": 2026, "level": "head", "key": "income_tax", "flags": ["numerical"], "base_gbp": 0.0, "head_gbp": 0.0} ] + }, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "commit": "<40-hex>", + "path": "requests//results/" + }, + "receipt_sha256": "<64-hex of the canonical receipt without this field>" +} +``` + +### Division of judgement + +* `integrity.status` is `complete`, `incomplete` or `failed`. `failed` means + a crash, a non-finite aggregate, an identity mismatch after the run started, + or an accounting identity broken in staging. These are structural failures + in the PLAN's sense; W4 blocks promotion on them. +* `diff.flag_counts` and `flagged_rows` describe movements against the base + variant. They carry no verdict. Whether an unexplained material move blocks + promotion, and who reviews it, is W4's qualification policy under d1344. +* Unsupported years are listed per event and are never counted as failures. + +## Open points for W4 + +1. Does the candidate wrapper arrive as a git commit (as proposed) or as a + wheel artifact? Scorecard can consume either; a wheel needs a URL it can + read without a token. +2. Receipt delivery: polling `replay-state` (proposed) or an additional + callback dispatch to policyengine.py. +3. Expected turnaround: not yet measured for a full sweep. One probe on + 6.2.6/2.125.1/1.58.0 loaded the baseline and computed five annual + aggregates in 10.6s at 2.3GB peak, so a five-event sweep should take + minutes rather than hours on a 16GB runner; the first full run will + replace this estimate. The queue serialises runs, so a backlog adds wait. +4. The receipt schema versioning rule: additive fields keep `v1`; renamed or + removed fields bump to `v2`. diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md new file mode 100644 index 0000000..aa8f67d --- /dev/null +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -0,0 +1,282 @@ +# Replaying the event corpus on every certified release + +Status: design contract for workstream W6 of the Autumn Budget 2026 CI plan, +11 October 2026. It extends the track 1 replay ([RECIPE.md](RECIPE.md), +[YEARS.md](YEARS.md)) from one hard-pinned world to any number of +**certified replay variants**, and turns each variant's run into a receipt +that release qualification (W4, policyengine.py) can consume before a +model/data pair is promoted. + +Nothing here changes a registered construction. A variant reruns the same +frozen registries; where a construction's premise no longer holds on a +variant, that is reported as a construction revision for review, never +silently repaired. Methodology changes (counterfactual intent, annualisation +semantics, acceptance policy) wait on cos d1344. + +## Why the existing runner cannot rerun + +Read in code at `61c13e38`: + +* `pipeline/uk_engine_registry.py` refuses any engine except `2.89.2`. +* `compute_uk_ab2025.preflight()` and `compute_uk_obr_costings.assert_engine_matches_bundle()` + read `data/uk/certified_bundle.json`, whose only compatible model is + `policyengine-uk ==2.89.2`, and require the installed engine to equal it. +* `modal_uk_event.py` hardcodes `ENGINE_PINS` (`policyengine 5.0.2`, + core `3.27.1`, UK `2.89.2`), the populace Hugging Face cache layout and + the audited HF commit. +* `compute_uk_event.validate_years` allows CY2023–2030 for every run. +* `compare_uk_event.review_diagnosis` attaches ten 2.89.2 findings to every + row with `"engine_version": "2.89.2"` whatever engine produced the row; + `construction_note` states 2.89.2 facts (the £3.15 Class 2 rate, "the pin + still has the 3% surcharge scale") unconditionally. +* `diagnose_uk_event_models.investigate` refuses any engine but 2.89.2. + +The newest certified pair is policyengine.py **6.2.6** (PR #555 and its +version bump `481940ec`): `policyengine-uk 2.125.1`, `policyengine-core +3.33.0`, data `policyengine-uk-data-1.58.0`, default dataset +`enhanced_frs_2024_25` (SHA-256 `5ba399ea…60b6`, 128,369,110 bytes, +`hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0`). + +Measured on that pair on 11 October with the installed wheels (one managed +simulation, offline, no token): + +* The HDF5's `time_period` table holds `["2024"]`; the managed simulation's + `dataset.years` is `[2024, …, 2030]` (`extend_single_year_dataset`, + `end_year=2030`, in `policyengine_uk/data/economic_assumptions.py`). +* **Calculating 2023 does not raise.** `income_tax` for 2023 sums to £0.0bn, + against £275.5bn in 2024: the simulation has no 2023 inputs and returns + defaults. Year support therefore has to be enforced from the data; a + successful calculation proves nothing. +* `pe.uk.managed_microsimulation()` materialises the default dataset at + `./data/` relative to the working directory (policyengine.py + `provenance/dataset_materialization.py`, `DEFAULT_DATA_DIR = Path("./data")`), + reusing a file whose SHA-256 matches and otherwise downloading over HTTPS. + The 5.0.2 HF-cache mirror path does not apply. +* With no Hugging Face token the release-manifest fetch fails and the + wrapper uses its packaged certification (`compatibility_basis + "legacy_compatible_model_package"`, `certified_for_model_version + "2.125.1"`). With a token, `certify_data_release_compatibility` derives a + certification from the remote release manifest instead, which changes + `release_bundle` fields. The 5.0.2 audit recorded the same split + (`OFFLINE_BUNDLE_AUDIT.json`, `release_bundle_field_changes_if_remote_manifest_is_available`). +* Peak resident memory for the baseline load plus five annual aggregates was + 2.3GB; the whole probe took 10.6s. + +## Certified replay variants + +`data/uk/replay_variants.json` lists every variant the corpus may run on. +`data/uk/certified_bundle.json` is unchanged and remains the legacy +variant's identity: AB2025, the OBR-costing runner and every existing +receipt keep reading it byte-for-byte. + +```json +{ + "schema_version": 1, + "default_variant": "", + "variants": [ , ... ] +} +``` + +A variant record: + +| Field | Meaning | +|---|---| +| `variant_id` | `pe_uk_`; must equal the slug recomputed from `identity`. | +| `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`, the SHA-256 of the installed distribution's `RECORD` file (it lists every installed file with its hash, so it binds the wheel contents). Optional `wheel_sha256` when the wheel itself was hashed. | +| `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | +| `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | +| `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`, `uri`. | +| `identity.materialization` | `hf_cache_mirror` (policyengine 5.x: the existing HF-cache plus `data/uk/policyengine-local` mirror) or `policyengine_data_dir` (6.x: a verified copy at `/data/`). | +| `identity.python` | Interpreter minor version, e.g. `3.12`. | +| `requirements` | `{path, sha256}`: the exact freeze installed for this variant. Legacy: `docs/uk_replay/requirements.txt`. Others: `docs/uk_replay/requirements/.txt`. | +| `data_years` | Calendar years the population supports: from the dataset's first `time_period` through the engine's extension end, as observed in `simulation.dataset.years`. | +| `data_years_evidence` | The observed `time_period`, `dataset.years` and the engine file/line that sets the extension end. | +| `certification` | Where the pairing is certified: `source` (e.g. `policyengine.py 6.2.6 packaged manifest`), `compatibility_basis`, `certified_for_model_version`, `data_build_id`. | +| `status` | `certified` (a released policyengine.py bundle), `candidate` (a W4 qualification request; never a default), or `retired`. | +| `qualification_request_sha256` | For candidates, the request the variant was built from. | +| `executor` | `local` (GitHub runner or workstation) and/or `modal`. The populace 1.3GB dataset needs Modal's 32GiB; the 128MB enhanced FRS runs on a 16GB runner. | + +### What "validated exactly" means + +Before any simulation the runner checks, in order, and refuses on the first +failure: + +1. **Record consistency (engine-free).** `variant_id` and `identity_sha256` + recompute; `release_bundle.model_version == packages["policyengine-uk"].version`; + `release_bundle.policyengine_version == packages["policyengine"].version`; + `certification.certified_for_model_version == packages["policyengine-uk"].version`; + `release_bundle.certified_data_artifact_sha256 == dataset.sha256`; + `release_bundle.certified_data_build_id == certification.data_build_id`; + `release_bundle.default_dataset_uri` names `dataset.repo_id`, `path` and + `revision`; the requirements file hashes to `requirements.sha256` and pins + exactly the three package versions. For the legacy variant, additionally + `certified_bundle.json`'s `revision`, `sha256`, `size_bytes`, `repo_id` + and `compatible_model_packages` agree with the record. +2. **Installed environment.** Installed versions of the three packages equal + the record; each installed `RECORD` hashes to `record_sha256`; the + installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; + the interpreter minor version matches. +3. **Certification path.** Before importing the engine the runner removes + `HUGGING_FACE_TOKEN`, `HF_TOKEN` and `HUGGINGFACE_HUB_TOKEN` from the + environment and forces Hugging Face offline mode, so the wrapper uses its + packaged certification on every host. Then + `dict(pe.uk.uk_latest.release_bundle) == identity.release_bundle`, exactly. +4. **Dataset bytes.** The materialised file hashes to `dataset.sha256` and + has `size_bytes`, before construction and after the last aggregate + (existing `run_managed_simulation` checks), and the simulation's reported + runtime dataset source is that file. +5. **Year support.** Requested years must lie in `data_years`, and inside + every simulation `year in simulation.dataset.years` (when the attribute + exists) or the run stops. Registry years outside `data_years` are not + requested by default; they are reported as unsupported (below). + +A model/data combination is **incompatible** when no certified variant +records it. There is no flag that relaxes a check: running a newer engine +means adding a variant whose certification names it. + +## Unsupported years + +Each event registry's `calendar_years` is the construction's window; a +variant runs `calendar_years ∩ data_years`. The rest are **unsupported on the +variant**, with a reason (`before_data_year` or `after_extension_end`). +`RUN_MANIFEST.json` records `supported_years`, `unsupported_years` and +`full_supported_grid_complete`; `full_event_complete` keeps its meaning +(every registry year). Staging marks a source row whose year is unsupported +as `status: "unsupported_on_variant"` with no numerical counterpart, kept in +the row universe and counted separately from `not_computed`. On +`pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0`, CY2023 is unsupported, so +Spring Budget 2023, Autumn Statement 2023 and Spring Budget 2024 lose their +FY2023–24 counterparts; that loss is shown, never hidden. + +## Result namespaces + +* Legacy results stay where they are, in `results/uk/events//`. +* Every other variant writes to `results/uk/replays//`: + `VARIANT.json` (the validated record plus the runtime receipt), + `SUPPORT.json`, `CONSTRUCTION_PREMISES.json`, `MODEL_DIAGNOSTICS.json`, + `/` (the same files as a legacy event directory), `SUMMARY.md`, and + finally `REPLAY_RECEIPT.json`. +* `results/uk/replays/INDEX.json` maps each variant to its root (legacy: + `results/uk/events`), status and receipt. +* A namespace with `REPLAY_RECEIPT.json` is **sealed**: the receipt binds the + SHA-256 of every file under it, the runner refuses to write there, and CI + fails if any sealed byte changes. A rerun of the same variant goes to a new + namespace suffix (`/rerun-`) and its determinism against the + sealed run is itself a check. +* Release differences live in `results/uk/replays/diffs/__/`. + +## Version-scoped diagnoses + +Every diagnosis and construction note carries the engine version it was +established on. A comparison row attaches a diagnosis as current only when +the artifact's `engine_version` is in that set. On any other engine the row +carries `prior_diagnoses` with `established_on` and `status: +"not_reverified"` until `diagnose_uk_event_models.py` has been rerun on that +engine, whose per-case predicate records `persists`, `resolved` or +`inconclusive`. The legacy event outputs remain byte-identical. + +## Construction premises + +`pipeline/audit_uk_event_premises.py` records, for each executable measure, +year and parameter path the construction touches, on a given engine: +current-law value at the annual lookup (1 January of the year), the raw +value at 30 April, the executed baseline-world and reform-world values, and +these checks: + +| Check | Fails when | +|---|---| +| `forward_delta_changes_current_law` | A forward delta equals current law in every in-force supported year (inert by construction). | +| `reversal_differs_from_current_law` | A reversal modifier equals current law in every in-force supported year. | +| `derived_counterfactual_reproduces` | Re-deriving `counterfactual_derivation` with this engine's parameters differs from the frozen registry values. | +| `annual_value_matches_30_april_sample` | The processed annual value differs from the raw 30 April value (the engine blends or annualises differently from the 2.89.2 note). | + +A failed check is a **construction revision candidate**, reported in the +release diff's construction flags and summarised in +`docs/uk_replay/CONSTRUCTION_REVISIONS.md` with a proposed revision. Running +a revised construction is a methodology change and waits on d1344. + +Observed on 2.125.1 on 11 October (parameter tree only): + +* `gov.contrib.labour.private_school_vat` is still 0 in every year, so the + AB2024 VAT forward delta still changes current law. +* The SDLT additional-dwelling scale is still 3/5/8/13/15%, so the AB2024 + forward 2pp delta still applies. +* CGT `basic_rate` at the 2024 annual lookup is 0.1346. In 2.125.1, + `parameters/gov/hmrc/cgt/basic_rate.yaml` dates the 18% rate from + 2024-10-30 and sets `metadata.fiscal_year_blend: true`, and + `utils/parameters.convert_to_fiscal_year_parameters` day-weights such + parameters over 6 April–5 April (`fiscal_year_average`) instead of + sampling 30 April: 0.10 × 207/365 + 0.18 × 158/365 = 0.1346. The AB2024 + note assumes 2.89.2's raw 2025-04-06 date and 30 April sampling. The + reversal, first selected in January 2025, leaves the blended 2024 increase + in both worlds, so CY2024 shows no effect although current law has one. +* The UC childcare cap path (2024 £1,014.63 for one child) and the benefit + CPI path feed the SB2023 and SS2025 frozen counterfactuals, which were + derived from 2.89.2's values. +* The Class 2 flat rate for 2023 is £3.45, so the "pin uses £3.15" note is a + 2.89.2 fact only. + +## Release differences + +`pipeline/diff_uk_event_runs.py --base --head ` +compares two complete namespaces, engine-free, after re-verifying each side's +staging and comparison receipts. Keys: + +* source row: `(event, source_row_id)`; +* measure-year total: `(event, measure_key, year)`; +* PE head: `(event, measure_key, year, variable)` from artifact `head_effects`. + +Flags are descriptive; none is a pass or a failure. A flag names a move to +investigate, and the existing signed PE/OBR bins are reported unchanged as +bin transitions. + +| Flag | Raised when | +|---|---| +| `numerical` | Both sides numeric and `abs(head − base) > max(£1,000,000, 0.01 × max(abs(head), abs(base)))`. Strictly greater. | +| `structural` | Sign reversal or a zero crossing (zero means `abs(x) ≤ £0.50`, accounting roundoff); artifact schema or orientation changed; non-finite value. Regardless of size. | +| `coverage` | A counterpart, artifact pair or source row present on one side only. | +| `mapping` | A row's `head_variables`, an artifact's `head_channels` or `head_definitions`, or the registry SHA-256 differs. | +| `support` | A year supported on one side only. | +| `inertness` | `inert_construction` on one side only, or identical worlds in an in-force year on one side only. | +| `construction` | Executed `baseline_world`/`reform_world` reform dicts differ, or a premise check differs. | + +Identity changes (wrapper, engine, core, data build, dataset digest) are +recorded once per diff; they explain moves and are not flags on rows. + +Outputs: `DIFF.json` (canonical, sorted), `DIFF.md`, and one issue packet per +flagged `(event, measure, year)` under `packets/`. A packet holds the source +cells (table, column, row number, source-row id, OBR value), both execution +identities, registry and artifact digests, the executed reform dicts, the +affected heads with base/head values, focused rerun commands for each +variant, the statutory sources the registry already cites, and empty +`minimal_case` and `statutory_evidence` sections for the investigating lane. +Packets are evidence for the coordinating session, which files upstream +issues; this pipeline files none. + +## Qualification requests and receipts + +See [QUALIFICATION_INTERFACE.md](QUALIFICATION_INTERFACE.md) for the request +W4 sends before promotion, the receipt returned, and the persistent queue. + +## Invariants + +Each has deterministic boundary cases and Hypothesis properties in +`tests/`: + +* **Exact variant validation.** For any record and environment differing from + it in one field, validation refuses; for the matching pair it accepts. +* **Support honesty.** No artifact exists for a year outside the variant's + `data_years`; every registry source row appears exactly once in staging, + with unsupported years as `unsupported_on_variant`. +* **Replay determinism.** For the same complete execution fingerprint + (variant identity, registry bytes, measure, year), numerical artifact bytes + are identical; timings stay in `RUN_LOG.json`. +* **Legacy preservation.** Legacy event registries, staging and comparison + outputs regenerate byte-identically after these changes. +* **Diff flags.** `diff(a, a)` has no row flags. Flag sets are symmetric under + swapping sides (with signs reversed). No numerical flag at or below the + threshold; one strictly above it. Row order and file order do not change + output bytes. +* **Sealed namespaces.** Every byte under a sealed namespace matches its + receipt. From 7cfa3f9c5e2bc848edc76067d9b5a0669592fe88 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:15:50 -0400 Subject: [PATCH 2/9] Pin the qualification request/receipt interface with RFC 8785 digests JSON Schemas for the W4 qualification request, the replay receipt and the replay-state status file; RFC 8785 (JCS) request_id and receipt_sha256; golden fixtures with pinned bytes and digests, including the JCS number and non-ASCII probes W4 asked for. Adds rfc8785 and jsonschema to the CI test environment. Records the amendments W4 accepted on 11 October and the current certified variant's exact requirements freeze. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- docs/uk_replay/QUALIFICATION_INTERFACE.md | 100 ++++++-- docs/uk_replay/RELEASE_REPLAY.md | 4 +- ..._uk2.125.1_policyengine-uk-data-1.58.0.txt | 84 +++++++ .../schemas/qualification_request.v1.json | 145 +++++++++++ docs/uk_replay/schemas/replay_receipt.v1.json | 229 ++++++++++++++++++ docs/uk_replay/schemas/replay_status.v1.json | 36 +++ pipeline/uk_replay_interface.py | 81 +++++++ tests/fixtures/uk_replay/DIGESTS.json | 8 + .../qualification_request_golden.jcs | 1 + .../qualification_request_golden.json | 72 ++++++ .../uk_replay/replay_receipt_golden.jcs | 1 + .../uk_replay/replay_receipt_golden.json | 137 +++++++++++ tests/test_uk_replay_interface.py | 190 +++++++++++++++ 14 files changed, 1061 insertions(+), 29 deletions(-) create mode 100644 docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt create mode 100644 docs/uk_replay/schemas/qualification_request.v1.json create mode 100644 docs/uk_replay/schemas/replay_receipt.v1.json create mode 100644 docs/uk_replay/schemas/replay_status.v1.json create mode 100644 pipeline/uk_replay_interface.py create mode 100644 tests/fixtures/uk_replay/DIGESTS.json create mode 100644 tests/fixtures/uk_replay/qualification_request_golden.jcs create mode 100644 tests/fixtures/uk_replay/qualification_request_golden.json create mode 100644 tests/fixtures/uk_replay/replay_receipt_golden.jcs create mode 100644 tests/fixtures/uk_replay/replay_receipt_golden.json create mode 100644 tests/test_uk_replay_interface.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6bc6f..fd5132e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: # pypdf + pyyaml are NOT optional extras: the HMT lane's strongest # honesty guarantee (every registry title and figure anchor verbatim # in the real PDF) runs as a test, and a bare env silently skipped it. - - run: uv run --with pytest --with hypothesis --with pypdf --with pyyaml --with numpy --with psutil pytest tests/ -q + - run: uv run --with pytest --with hypothesis --with pypdf --with pyyaml --with numpy --with psutil --with rfc8785==0.1.4 --with jsonschema pytest tests/ -q env: PYTHONPATH: . - run: git diff --exit-code diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index 0402d6b..d616748 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -1,9 +1,26 @@ -# Qualification request and replay receipt (v1, proposed) - -Status: proposal from W6 (policyengine-scorecard) to W4 (policyengine.py -qualification), 11 October 2026, for agreement by 15 October. JSON Schemas: -[`schemas/qualification_request.v1.json`](schemas/qualification_request.v1.json) -and [`schemas/replay_receipt.v1.json`](schemas/replay_receipt.v1.json). +# Qualification request and replay receipt (v1) + +Status: proposed by W6 (policyengine-scorecard) to W4 (policyengine.py +qualification) on 11 October 2026 and accepted in principle by W4 the same +day, with the amendments folded in below (RFC 8785 canonicalisation, the +receipt digest and commit in `STATUS.json`, extra `requested_by` fields, and +the post-release flow). JSON Schemas: +[`schemas/qualification_request.v1.json`](schemas/qualification_request.v1.json), +[`schemas/replay_receipt.v1.json`](schemas/replay_receipt.v1.json) and +[`schemas/replay_status.v1.json`](schemas/replay_status.v1.json). +Implementation: `pipeline/uk_replay_interface.py`. Golden fixtures with +pinned digests: `tests/fixtures/uk_replay/` (`DIGESTS.json`, and the exact +JCS bytes in `*.jcs`). + +## Canonical bytes + +Both sides compute digests over **RFC 8785 (JCS)** bytes, using the +`rfc8785` package (0.1.4 here). `json.dumps(sort_keys=True)` is not +equivalent: it writes `1e-07` where JCS writes `1e-7`, `1.5e+16` where JCS +writes `15000000000000000`, and escapes non-ASCII characters that JCS +writes as UTF-8. Integers must lie within ±(2^53 − 1); the package refuses +larger ones. The golden request carries these probes in +`requested_by.jcs_probe` and a non-ASCII `requested_by.note`. Two flows use the same receiver: @@ -12,10 +29,11 @@ Two flows use the same receiver: validates the candidate's identity and compatibility, reruns the supported event corpus, diffs it against the latest certified variant, and returns an artifact-bound receipt that qualification consumes. -* **Post-promotion consumer dispatch.** After a policyengine.py release, the - existing dispatcher notifies consumers. Scorecard treats a released bundle - it has not replayed as a request with `kind: "released_bundle"`, so it - ends up with a certified variant and a sealed namespace. +* **Post-promotion consumer dispatch.** After a policyengine.py release, + W4's NotifyConsumers sends the same event with `kind: "released_bundle"`, + the policyengine PyPI wheel SHA-256 and the packaged manifest SHA-256. The + same receiver handles it, so scorecard ends up with a certified variant + and a sealed namespace for every release; there is no separate handler. ## Transport @@ -32,9 +50,12 @@ Two flows use the same receiver: * State for request `` lives at `replay-state:requests//`: `REQUEST.json` (the bytes received), `STATUS.json` - (`queued` → `running` → `complete` | `rejected` | `failed`, with - timestamps and the Actions run URL), and on completion - `REPLAY_RECEIPT.json`. + (`queued` → `running` → `complete` | `rejected` | `failed`, with a + history of timestamps and Actions run URLs), and on completion + `REPLAY_RECEIPT.json`. The receipt is committed first; a terminal + `STATUS.json` is written in a later commit and names `receipt_sha256` and + `receipt_commit`, so one contents read binds the status to the receipt + bytes. * W4 polls `GET /repos/PolicyEngine/policyengine-scorecard/contents/requests//STATUS.json?ref=replay-state` (the repository is public). A receipt is addressed by the request's own digest, so W4 needs no callback and no scorecard token. @@ -79,9 +100,10 @@ Two flows use the same receiver: } ``` -* `request_id` is not a field: it is the SHA-256 of the canonical request - JSON (sorted keys, no whitespace, UTF-8). Both sides compute it the same - way; `tests/fixtures/` holds a shared golden example. +* `request_id` is not a field: it is the SHA-256 of the request's JCS + bytes. Both sides test against the golden fixture. +* `requested_by` accepts additional provenance fields (for example + `run_attempt`); every other object is closed. * The wrapper is identified by `version` plus either a PyPI wheel (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose regenerated manifest is not on PyPI yet, a public git commit. Scorecard @@ -164,6 +186,19 @@ A rejected request has no numerical results; its receipt says why. ### Division of judgement +How flags arise, so a gate built on them sees the intended events: + +* A year unsupported on one side raises `support` on that year's rows, not + `coverage`. A lost counterpart is `coverage` only where both sides support + the year. +* `construction` means the executed reform dictionaries differ or a premise + check changed its result. A change in the current-law value of a parameter + the construction touches is recorded as context in the row's details, not + as a flag, because it is what a model release is expected to change. +* For a candidate, the base is the latest certified variant, normally on the + same data build; the first certified-variant run diffs against the legacy + populace variant, where the data family itself changes. + * `integrity.status` is `complete`, `incomplete` or `failed`. `failed` means a crash, a non-finite aggregate, an identity mismatch after the run started, or an accounting identity broken in staging. These are structural failures @@ -173,17 +208,30 @@ A rejected request has no numerical results; its receipt says why. promotion, and who reviews it, is W4's qualification policy under d1344. * Unsupported years are listed per event and are never counted as failures. -## Open points for W4 - -1. Does the candidate wrapper arrive as a git commit (as proposed) or as a - wheel artifact? Scorecard can consume either; a wheel needs a URL it can - read without a token. -2. Receipt delivery: polling `replay-state` (proposed) or an additional - callback dispatch to policyengine.py. -3. Expected turnaround: not yet measured for a full sweep. One probe on +## Agreed points + +1. The candidate wrapper is a git commit on a policyengine.py + `candidate/uk/-` branch whose packaged manifest, pins, + `uv.lock` and TRO certify the pair; that commit becomes the promotion PR + head, so the replayed bytes are the promoted bytes. +2. Receipt delivery is by polling `replay-state`; no callback. +3. W4's qualification receipt records `request_id`, `receipt_sha256` and + `receipt_commit`; nothing of this receipt nests inside W4's execution + receipt. +4. W4 computes `expected_release_bundle` in a subprocess with the three + Hugging Face token variables removed. +5. W4's proposed gate on these fields (pending d1344): `rejected`, `failed`, + integrity `failed` or `incomplete`, and structural, coverage, mapping, + inertness or construction flags fail the replay gate; numerical and + support flags go to review; `no_supported_years` fails; unsupported years + alone never fail. + +## Open points + +1. Expected turnaround: not yet measured for a full sweep. One probe on 6.2.6/2.125.1/1.58.0 loaded the baseline and computed five annual aggregates in 10.6s at 2.3GB peak, so a five-event sweep should take minutes rather than hours on a 16GB runner; the first full run will replace this estimate. The queue serialises runs, so a backlog adds wait. -4. The receipt schema versioning rule: additive fields keep `v1`; renamed or - removed fields bump to `v2`. +2. The receipt schema versioning rule: additive optional fields keep `v1`; + renamed or removed fields bump to `v2`. diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index aa8f67d..829cdc2 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -235,11 +235,11 @@ bin transitions. |---|---| | `numerical` | Both sides numeric and `abs(head − base) > max(£1,000,000, 0.01 × max(abs(head), abs(base)))`. Strictly greater. | | `structural` | Sign reversal or a zero crossing (zero means `abs(x) ≤ £0.50`, accounting roundoff); artifact schema or orientation changed; non-finite value. Regardless of size. | -| `coverage` | A counterpart, artifact pair or source row present on one side only. | +| `coverage` | A counterpart, artifact pair or source row present on one side only, in a year both sides support. | | `mapping` | A row's `head_variables`, an artifact's `head_channels` or `head_definitions`, or the registry SHA-256 differs. | | `support` | A year supported on one side only. | | `inertness` | `inert_construction` on one side only, or identical worlds in an in-force year on one side only. | -| `construction` | Executed `baseline_world`/`reform_world` reform dicts differ, or a premise check differs. | +| `construction` | Executed `baseline_world`/`reform_world` reform dicts differ, or a premise check's result differs. A changed current-law value of a touched parameter is recorded as context on the row, not flagged: it is what a model release is expected to change. | Identity changes (wrapper, engine, core, data build, dataset digest) are recorded once per diff; they explain moves and are not flags on rows. diff --git a/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt b/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt new file mode 100644 index 0000000..8cf183d --- /dev/null +++ b/docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt @@ -0,0 +1,84 @@ +annotated-types==0.8.0 +anyio==4.15.1 +asttokens==3.0.2 +attrs==26.1.0 +blosc2==4.14.1 +certifi==2026.7.22 +charset-normalizer==3.5.2 +click==8.5.0 +decorator==5.3.1 +dpath==2.2.0 +executing==2.3.0 +filelock==4.1.0 +fsspec==2026.9.0 +h11==0.16.0 +h2==4.4.1 +h5py==3.16.0 +hf-xet==1.7.0 +hpack==4.2.0 +httpcore==1.0.9 +httpcore2==2.13.1 +httpx==0.28.1 +httpx2==2.13.1 +huggingface-hub==2.2.0 +hyperframe==6.1.0 +hypothesis==6.169.0 +idna==3.20 +iniconfig==2.3.1 +ipython==8.39.0 +jedi==0.20.0 +jinja2==3.1.6 +jsonpickle==4.1.3 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +markdown-it-py==4.2.0 +markupsafe==3.0.4 +matplotlib-inline==0.2.2 +mdurl==0.1.2 +microdf-python==1.3.10 +msgpack==1.2.3 +ndindex==1.10.1 +networkx==3.7 +numexpr==2.14.2 +numpy==2.5.4 +packaging==26.3 +pandas==3.0.6 +parso==0.8.7 +pexpect==4.9.0 +plotly==5.24.1 +pluggy==1.6.0 +policyengine==6.2.6 +policyengine-core==3.33.0 +policyengine-uk==2.125.1 +prompt-toolkit==3.0.53 +psutil==6.1.1 +ptyprocess==0.7.0 +pure-eval==0.2.4 +py-cpuinfo==9.0.0 +pydantic==2.14.0 +pydantic-core==2.50.0 +pygments==2.21.0 +pypdf==6.20.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +pyvis==0.3.2 +pyyaml==6.0.3 +referencing==0.37.0 +requests==2.34.2 +rich==15.0.0 +rpds-py==2026.9.1 +six==1.17.0 +sortedcontainers==2.4.0 +stack-data==0.6.3 +standard-imghdr==3.13.0 +tables==3.11.1 +tenacity==9.2.1 +threadpoolctl==3.7.0 +tqdm==4.70.1 +traitlets==5.16.1 +truststore==0.10.4 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.8.0 +wcwidth==0.9.2 +wheel==0.48.0 diff --git a/docs/uk_replay/schemas/qualification_request.v1.json b/docs/uk_replay/schemas/qualification_request.v1.json new file mode 100644 index 0000000..735b66c --- /dev/null +++ b/docs/uk_replay/schemas/qualification_request.v1.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/qualification_request.v1.json", + "title": "UK fiscal-event replay qualification request, v1", + "description": "Sent by policyengine.py release qualification (W4) to policyengine-scorecard (W6). request_id is the SHA-256 of the RFC 8785 (JCS) bytes of the whole request; it is never a field. Additive optional fields keep v1; renamed or removed fields bump the version.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "kind", "attempt", "candidate", "compare_to", "requested_by"], + "properties": { + "schema": {"const": "policyengine.uk-replay-qualification-request/v1"}, + "kind": { + "description": "candidate: a pair not yet promoted; released_bundle: a published policyengine.py release.", + "enum": ["candidate", "released_bundle"] + }, + "attempt": { + "description": "A retry of a failed request is a new request with a higher attempt.", + "type": "integer", + "minimum": 1 + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": ["policyengine", "policyengine_uk", "policyengine_core", "data", "expected_release_bundle"], + "properties": { + "policyengine": { + "type": "object", + "additionalProperties": false, + "required": ["version", "source", "packaged_bundle_manifest_sha256"], + "properties": { + "version": {"$ref": "#/$defs/version"}, + "source": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["git"], + "properties": { + "git": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "commit"], + "properties": { + "repository": {"const": "PolicyEngine/policyengine.py"}, + "commit": {"$ref": "#/$defs/sha1"} + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["pypi"], + "properties": { + "pypi": { + "type": "object", + "additionalProperties": false, + "required": ["wheel_sha256"], + "properties": {"wheel_sha256": {"$ref": "#/$defs/sha256"}} + } + } + } + ] + }, + "packaged_bundle_manifest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "policyengine_uk": {"$ref": "#/$defs/pypi_package"}, + "policyengine_core": {"$ref": "#/$defs/pypi_package"}, + "data": { + "type": "object", + "additionalProperties": false, + "required": ["build_id", "dataset", "repo_id", "repo_type", "revision", "path", "sha256", "size_bytes", "release_manifest_sha256"], + "properties": { + "build_id": {"type": "string", "minLength": 1}, + "dataset": {"type": "string", "minLength": 1}, + "repo_id": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "repo_type": {"enum": ["model", "dataset"]}, + "revision": {"type": "string", "minLength": 1}, + "path": {"type": "string", "minLength": 1, "pattern": "^[^/][^\\\\]*$"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "size_bytes": {"type": "integer", "minimum": 1}, + "release_manifest_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "expected_release_bundle": { + "description": "The exact dict(pe.uk.uk_latest.release_bundle) under packaged certification, computed with HF_TOKEN, HUGGING_FACE_TOKEN and HUGGINGFACE_HUB_TOKEN removed from the environment.", + "type": "object", + "required": ["policyengine_version", "model_version", "certified_data_build_id", "certified_data_artifact_sha256", "default_dataset_uri"], + "additionalProperties": {"type": ["string", "null"]}, + "properties": { + "policyengine_version": {"$ref": "#/$defs/version"}, + "model_version": {"$ref": "#/$defs/version"}, + "certified_data_build_id": {"type": "string", "minLength": 1}, + "certified_data_artifact_sha256": {"$ref": "#/$defs/sha256"}, + "default_dataset_uri": {"type": "string", "pattern": "^hf://"} + } + } + } + }, + "compare_to": { + "description": "latest_certified, or an explicit scorecard variant_id.", + "type": "string", + "minLength": 1 + }, + "requested_by": { + "type": "object", + "additionalProperties": true, + "required": ["repository", "workflow_run_id", "commit"], + "properties": { + "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "workflow_run_id": {"type": "integer", "minimum": 1}, + "run_attempt": {"type": "integer", "minimum": 1}, + "commit": {"$ref": "#/$defs/sha1"} + } + } + }, + "allOf": [ + { + "if": {"properties": {"kind": {"const": "released_bundle"}}}, + "then": { + "properties": { + "candidate": { + "properties": { + "policyengine": {"properties": {"source": {"required": ["pypi"]}}} + } + } + } + } + } + ], + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "sha1": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "version": {"type": "string", "pattern": "^[0-9]+(\\.[0-9]+)*([a-z0-9.+-]*)$"}, + "pypi_package": { + "type": "object", + "additionalProperties": false, + "required": ["version", "wheel_sha256"], + "properties": { + "version": {"$ref": "#/$defs/version"}, + "wheel_sha256": {"$ref": "#/$defs/sha256"} + } + } + } +} diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json new file mode 100644 index 0000000..2aac386 --- /dev/null +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -0,0 +1,229 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/replay_receipt.v1.json", + "title": "UK fiscal-event replay receipt, v1", + "description": "Returned by policyengine-scorecard for one qualification request. receipt_sha256 is the SHA-256 of the RFC 8785 (JCS) bytes of the receipt without that field. Flags describe movements against the base variant; they carry no verdict.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "request_id", "status", "rejection", "variant_id", "variant_identity_sha256", "execution", "corpus", "integrity", "diff", "results", "receipt_sha256"], + "properties": { + "schema": {"const": "policyengine.uk-replay-receipt/v1"}, + "request_id": {"$ref": "#/$defs/sha256"}, + "status": {"enum": ["complete", "rejected", "failed"]}, + "rejection": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["code", "detail"], + "properties": { + "code": {"enum": ["schema_invalid", "artifact_hash_mismatch", "release_bundle_mismatch", "incompatible_pair", "data_access_failed", "no_supported_years"]}, + "detail": {"type": "string", "minLength": 1} + } + } + ] + }, + "variant_id": {"type": ["string", "null"], "minLength": 1}, + "variant_identity_sha256": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/sha256"}]}, + "execution": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["packages", "packaged_bundle_manifest_sha256", "release_bundle", "dataset_sha256", "requirements_sha256", "python", "platform", "runner"], + "properties": { + "packages": { + "type": "object", + "additionalProperties": false, + "required": ["policyengine", "policyengine-uk", "policyengine-core"], + "properties": { + "policyengine": {"$ref": "#/$defs/installed_package"}, + "policyengine-uk": {"$ref": "#/$defs/installed_package"}, + "policyengine-core": {"$ref": "#/$defs/installed_package"} + } + }, + "packaged_bundle_manifest_sha256": {"$ref": "#/$defs/sha256"}, + "release_bundle": {"type": "object", "additionalProperties": {"type": ["string", "null"]}}, + "dataset_sha256": {"$ref": "#/$defs/sha256"}, + "requirements_sha256": {"$ref": "#/$defs/sha256"}, + "python": {"type": "string", "pattern": "^3\\.[0-9]+\\.[0-9]+"}, + "platform": {"type": "string", "minLength": 1}, + "runner": {"type": "string", "minLength": 1} + } + } + ] + }, + "corpus": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["corpus_version", "events"], + "properties": { + "corpus_version": {"type": "string", "minLength": 1}, + "events": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"$ref": "#/$defs/event_result"} + } + } + } + ] + }, + "integrity": { + "type": "object", + "additionalProperties": false, + "required": ["status", "failures"], + "properties": { + "status": { + "description": "complete: every supported measure-year ran and every receipt verified. incomplete: some supported pairs are missing. failed: a crash, a non-finite value, an identity mismatch after the run started, or a broken accounting identity.", + "enum": ["complete", "incomplete", "failed", "not_run"] + }, + "failures": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "detail"], + "properties": { + "code": {"type": "string", "minLength": 1}, + "detail": {"type": "string", "minLength": 1}, + "event": {"type": "string"}, + "measure_key": {"type": "string"}, + "year": {"type": "integer"} + } + } + } + } + }, + "diff": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "additionalProperties": false, + "required": ["base_variant_id", "diff_sha256", "flag_counts", "flagged_rows"], + "properties": { + "base_variant_id": {"type": "string", "minLength": 1}, + "diff_sha256": {"$ref": "#/$defs/sha256"}, + "flag_counts": { + "type": "object", + "additionalProperties": false, + "required": ["numerical", "structural", "coverage", "mapping", "support", "inertness", "construction"], + "properties": { + "numerical": {"type": "integer", "minimum": 0}, + "structural": {"type": "integer", "minimum": 0}, + "coverage": {"type": "integer", "minimum": 0}, + "mapping": {"type": "integer", "minimum": 0}, + "support": {"type": "integer", "minimum": 0}, + "inertness": {"type": "integer", "minimum": 0}, + "construction": {"type": "integer", "minimum": 0} + } + }, + "flagged_rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["event", "measure_key", "year", "level", "key", "flags", "base_gbp", "head_gbp"], + "properties": { + "event": {"type": "string", "minLength": 1}, + "measure_key": {"type": "string", "minLength": 1}, + "year": {"type": "integer"}, + "level": {"enum": ["source_row", "measure_year", "head"]}, + "key": {"type": "string", "minLength": 1}, + "flags": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"enum": ["numerical", "structural", "coverage", "mapping", "support", "inertness", "construction"]} + }, + "base_gbp": {"type": ["number", "null"]}, + "head_gbp": {"type": ["number", "null"]} + } + } + } + } + } + ] + }, + "results": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "ref", "path"], + "properties": { + "repository": {"const": "PolicyEngine/policyengine-scorecard"}, + "ref": {"type": "string", "minLength": 1}, + "commit": {"$ref": "#/$defs/sha1"}, + "path": {"type": "string", "minLength": 1} + } + }, + "receipt_sha256": {"$ref": "#/$defs/sha256"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "rejected"}}}, + "then": {"properties": {"rejection": {"type": "object"}, "corpus": {"type": "null"}, "diff": {"type": "null"}}} + }, + { + "if": {"properties": {"status": {"enum": ["complete", "failed"]}}}, + "then": {"properties": {"rejection": {"type": "null"}, "variant_id": {"type": "string"}, "execution": {"type": "object"}}} + }, + { + "if": {"properties": {"status": {"const": "complete"}}}, + "then": {"properties": {"integrity": {"properties": {"status": {"enum": ["complete", "incomplete"]}}}, "corpus": {"type": "object"}}} + } + ], + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "sha1": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "installed_package": { + "type": "object", + "additionalProperties": false, + "required": ["version", "record_sha256"], + "properties": { + "version": {"type": "string", "minLength": 1}, + "record_sha256": {"$ref": "#/$defs/sha256"}, + "wheel_sha256": {"$ref": "#/$defs/sha256"} + } + }, + "event_result": { + "type": "object", + "additionalProperties": false, + "required": ["registry_sha256", "supported_years", "unsupported_years", "run_manifest_sha256", "staging_manifest_sha256", "comparison_provenance_sha256", "artifacts", "full_supported_grid_complete", "inert_measure_years"], + "properties": { + "registry_sha256": {"$ref": "#/$defs/sha256"}, + "supported_years": {"type": "array", "items": {"type": "integer"}, "uniqueItems": true}, + "unsupported_years": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["year", "reason"], + "properties": { + "year": {"type": "integer"}, + "reason": {"enum": ["before_data_year", "after_extension_end"]} + } + } + }, + "run_manifest_sha256": {"$ref": "#/$defs/sha256"}, + "staging_manifest_sha256": {"$ref": "#/$defs/sha256"}, + "comparison_provenance_sha256": {"$ref": "#/$defs/sha256"}, + "artifacts": {"type": "integer", "minimum": 0}, + "full_supported_grid_complete": {"type": "boolean"}, + "inert_measure_years": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["measure_key", "year"], + "properties": {"measure_key": {"type": "string"}, "year": {"type": "integer"}} + } + } + } + } + } +} diff --git a/docs/uk_replay/schemas/replay_status.v1.json b/docs/uk_replay/schemas/replay_status.v1.json new file mode 100644 index 0000000..3694ca4 --- /dev/null +++ b/docs/uk_replay/schemas/replay_status.v1.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PolicyEngine/policyengine-scorecard/blob/main/docs/uk_replay/schemas/replay_status.v1.json", + "title": "UK fiscal-event replay request status, v1", + "description": "replay-state:requests//STATUS.json. A terminal status names the receipt's digest and the replay-state commit that added REPLAY_RECEIPT.json, so one contents read binds the status to the receipt bytes.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "request_id", "status", "history"], + "properties": { + "schema": {"const": "policyengine.uk-replay-status/v1"}, + "request_id": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "status": {"enum": ["queued", "running", "complete", "rejected", "failed"]}, + "history": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["status", "at", "run_url"], + "properties": { + "status": {"enum": ["queued", "running", "complete", "rejected", "failed"]}, + "at": {"type": "string", "format": "date-time"}, + "run_url": {"type": "string", "pattern": "^https://github\\.com/PolicyEngine/policyengine-scorecard/actions/runs/[0-9]+"} + } + } + }, + "receipt_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "receipt_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"enum": ["complete", "rejected", "failed"]}}}, + "then": {"required": ["receipt_sha256", "receipt_commit"]} + } + ] +} diff --git a/pipeline/uk_replay_interface.py b/pipeline/uk_replay_interface.py new file mode 100644 index 0000000..3788f5a --- /dev/null +++ b/pipeline/uk_replay_interface.py @@ -0,0 +1,81 @@ +"""Canonical identities for qualification requests and replay receipts. + +Release qualification (policyengine.py) sends a request; this repository +returns a receipt. Both sides address them by digest, so both must produce +the same bytes: RFC 8785 JSON Canonicalization Scheme (JCS), via the +``rfc8785`` package. JCS fixes what ``json.dumps(sort_keys=True)`` leaves to +the implementation: number serialisation (``1e-7``, not ``1e-07``; +``15000000000000000``, not ``1.5e+16``) and literal UTF-8 for non-ASCII +strings. See docs/uk_replay/QUALIFICATION_INTERFACE.md. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_DIR = ROOT / "docs" / "uk_replay" / "schemas" +REQUEST_SCHEMA_ID = "policyengine.uk-replay-qualification-request/v1" +RECEIPT_SCHEMA_ID = "policyengine.uk-replay-receipt/v1" +STATUS_SCHEMA_ID = "policyengine.uk-replay-status/v1" +SCHEMA_FILES = { + REQUEST_SCHEMA_ID: "qualification_request.v1.json", + RECEIPT_SCHEMA_ID: "replay_receipt.v1.json", + STATUS_SCHEMA_ID: "replay_status.v1.json", +} + + +def jcs_bytes(value: Any) -> bytes: + """RFC 8785 canonical UTF-8 bytes; the package refuses non-finite numbers.""" + import rfc8785 + + return rfc8785.dumps(value) + + +def jcs_sha256(value: Any) -> str: + return hashlib.sha256(jcs_bytes(value)).hexdigest() + + +def request_id(request: dict) -> str: + """The request's own digest; it is never a field of the request.""" + if "request_id" in request: + raise ValueError("request_id is derived from the request, not carried in it") + return jcs_sha256(request) + + +def receipt_sha256(receipt: dict) -> str: + """Digest of the receipt without its own receipt_sha256 field.""" + return jcs_sha256({k: v for k, v in receipt.items() if k != "receipt_sha256"}) + + +def seal_receipt(receipt: dict) -> dict: + sealed = {k: v for k, v in receipt.items() if k != "receipt_sha256"} + sealed["receipt_sha256"] = receipt_sha256(sealed) + return sealed + + +def load_schema(schema_id: str) -> dict: + return json.loads((SCHEMA_DIR / SCHEMA_FILES[schema_id]).read_bytes()) + + +def validate(document: dict, schema_id: str) -> None: + """JSON Schema validation, then the digest identity a schema cannot express.""" + import jsonschema + + jsonschema.Draft202012Validator( + load_schema(schema_id), + format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER, + ).validate(document) + if schema_id == RECEIPT_SCHEMA_ID and ( + document["receipt_sha256"] != receipt_sha256(document) + ): + raise ValueError("receipt_sha256 does not match the receipt's canonical bytes") + + +def validate_request(request: dict) -> str: + """Validate a request and return its request_id.""" + validate(request, REQUEST_SCHEMA_ID) + return request_id(request) diff --git a/tests/fixtures/uk_replay/DIGESTS.json b/tests/fixtures/uk_replay/DIGESTS.json new file mode 100644 index 0000000..b7e61f6 --- /dev/null +++ b/tests/fixtures/uk_replay/DIGESTS.json @@ -0,0 +1,8 @@ +{ + "qualification_request_golden.json": { + "request_id": "2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf" + }, + "replay_receipt_golden.json": { + "receipt_sha256": "a070ac40cb4e6d7daef520cfc1be4ed4958cdd4fe275975dc6d55e856d4ff515" + } +} diff --git a/tests/fixtures/uk_replay/qualification_request_golden.jcs b/tests/fixtures/uk_replay/qualification_request_golden.jcs new file mode 100644 index 0000000..0e008c9 --- /dev/null +++ b/tests/fixtures/uk_replay/qualification_request_golden.jcs @@ -0,0 +1 @@ +{"attempt":1,"candidate":{"data":{"build_id":"policyengine-uk-data-1.58.0","dataset":"enhanced_frs_2024_25","path":"enhanced_frs_2024_25.h5","release_manifest_sha256":"ddd63e1f7755ee358c4ede9050bfdde09f3a80f48a5c0d8ee9d1fb531e4a1a07","repo_id":"policyengine/policyengine-uk-data-private","repo_type":"model","revision":"1.58.0","sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","size_bytes":128369110},"expected_release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"policyengine":{"packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","source":{"git":{"commit":"481940eca44fc75e41fbf9de4ff454e0a0d7c0e0","repository":"PolicyEngine/policyengine.py"}},"version":"6.2.6"},"policyengine_core":{"version":"3.33.0","wheel_sha256":"2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789"},"policyengine_uk":{"version":"2.125.1","wheel_sha256":"a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a"}},"compare_to":"latest_certified","kind":"candidate","requested_by":{"commit":"481940eca44fc75e41fbf9de4ff454e0a0d7c0e0","jcs_probe":[1e-7,15000000000000000,0.30000000000000004,0,1e+21,5e-324,123456789012345680000,9007199254740991],"note":"Golden fixture: JCS probes for non-ASCII (Budget £, café, 予算) and doubles.","repository":"PolicyEngine/policyengine.py","run_attempt":1,"workflow_run_id":1},"schema":"policyengine.uk-replay-qualification-request/v1"} \ No newline at end of file diff --git a/tests/fixtures/uk_replay/qualification_request_golden.json b/tests/fixtures/uk_replay/qualification_request_golden.json new file mode 100644 index 0000000..1f6d69c --- /dev/null +++ b/tests/fixtures/uk_replay/qualification_request_golden.json @@ -0,0 +1,72 @@ +{ + "schema": "policyengine.uk-replay-qualification-request/v1", + "kind": "candidate", + "attempt": 1, + "candidate": { + "policyengine": { + "version": "6.2.6", + "source": { + "git": { + "repository": "PolicyEngine/policyengine.py", + "commit": "481940eca44fc75e41fbf9de4ff454e0a0d7c0e0" + } + }, + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54" + }, + "policyengine_uk": { + "version": "2.125.1", + "wheel_sha256": "a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a" + }, + "policyengine_core": { + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "data": { + "build_id": "policyengine-uk-data-1.58.0", + "dataset": "enhanced_frs_2024_25", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "path": "enhanced_frs_2024_25.h5", + "sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "size_bytes": 128369110, + "release_manifest_sha256": "ddd63e1f7755ee358c4ede9050bfdde09f3a80f48a5c0d8ee9d1fb531e4a1a07" + }, + "expected_release_bundle": { + "bundle_id": "uk-6.2.6", + "country_id": "uk", + "policyengine_version": "6.2.6", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "data_build_model_version": "2.122.2", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "compatibility_basis": "legacy_compatible_model_package", + "certified_by": "policyengine.py bundle certification" + } + }, + "compare_to": "latest_certified", + "requested_by": { + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 1, + "run_attempt": 1, + "commit": "481940eca44fc75e41fbf9de4ff454e0a0d7c0e0", + "note": "Golden fixture: JCS probes for non-ASCII (Budget £, café, 予算) and doubles.", + "jcs_probe": [ + 1e-07, + 1.5e+16, + 0.30000000000000004, + -0.0, + 1e+21, + 5e-324, + 1.2345678901234568e+20, + 9007199254740991 + ] + } +} diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.jcs b/tests/fixtures/uk_replay/replay_receipt_golden.jcs new file mode 100644 index 0000000..871f1d3 --- /dev/null +++ b/tests/fixtures/uk_replay/replay_receipt_golden.jcs @@ -0,0 +1 @@ +{"corpus":{"corpus_version":"uk-fiscal-event-replay/1","events":{"spring_budget_2023":{"artifacts":8,"comparison_provenance_sha256":"8888888888888888888888888888888888888888888888888888888888888888","full_supported_grid_complete":true,"inert_measure_years":[],"registry_sha256":"5555555555555555555555555555555555555555555555555555555555555555","run_manifest_sha256":"6666666666666666666666666666666666666666666666666666666666666666","staging_manifest_sha256":"7777777777777777777777777777777777777777777777777777777777777777","supported_years":[2024,2025,2026,2027],"unsupported_years":[{"reason":"before_data_year","year":2023}]}}},"diff":{"base_variant_id":"pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z","diff_sha256":"9999999999999999999999999999999999999999999999999999999999999999","flag_counts":{"construction":0,"coverage":0,"inertness":0,"mapping":0,"numerical":2,"structural":1,"support":1},"flagged_rows":[{"base_gbp":-15000000000000000,"event":"spring_budget_2023","flags":["numerical"],"head_gbp":0.30000000000000004,"key":"universal_credit","level":"head","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2024},{"base_gbp":1e-7,"event":"spring_budget_2023","flags":["numerical","structural"],"head_gbp":-123456789.125,"key":"total","level":"measure_year","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2025},{"base_gbp":-2500000,"event":"spring_budget_2023","flags":["support"],"head_gbp":null,"key":"obr-pmd-row-123","level":"source_row","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2023}]},"execution":{"dataset_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","packages":{"policyengine":{"record_sha256":"1111111111111111111111111111111111111111111111111111111111111111","version":"6.2.6"},"policyengine-core":{"record_sha256":"3333333333333333333333333333333333333333333333333333333333333333","version":"3.33.0"},"policyengine-uk":{"record_sha256":"2222222222222222222222222222222222222222222222222222222222222222","version":"2.125.1"}},"platform":"Golden fixture (synthetic values; café £)","python":"3.12.14","release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"requirements_sha256":"4444444444444444444444444444444444444444444444444444444444444444","runner":"github-actions ubuntu-latest"},"integrity":{"failures":[],"status":"complete"},"rejection":null,"request_id":"2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf","results":{"commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","path":"requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/","ref":"replay-state","repository":"PolicyEngine/policyengine-scorecard"},"schema":"policyengine.uk-replay-receipt/v1","status":"complete","variant_id":"pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0","variant_identity_sha256":"0000000000000000000000000000000000000000000000000000000000000000"} \ No newline at end of file diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.json b/tests/fixtures/uk_replay/replay_receipt_golden.json new file mode 100644 index 0000000..433f095 --- /dev/null +++ b/tests/fixtures/uk_replay/replay_receipt_golden.json @@ -0,0 +1,137 @@ +{ + "schema": "policyengine.uk-replay-receipt/v1", + "request_id": "2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf", + "status": "complete", + "rejection": null, + "variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "variant_identity_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "execution": { + "packages": { + "policyengine": { + "version": "6.2.6", + "record_sha256": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "policyengine-uk": { + "version": "2.125.1", + "record_sha256": "2222222222222222222222222222222222222222222222222222222222222222" + }, + "policyengine-core": { + "version": "3.33.0", + "record_sha256": "3333333333333333333333333333333333333333333333333333333333333333" + } + }, + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", + "release_bundle": { + "bundle_id": "uk-6.2.6", + "country_id": "uk", + "policyengine_version": "6.2.6", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "data_build_model_version": "2.122.2", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "compatibility_basis": "legacy_compatible_model_package", + "certified_by": "policyengine.py bundle certification" + }, + "dataset_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "requirements_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "python": "3.12.14", + "platform": "Golden fixture (synthetic values; café £)", + "runner": "github-actions ubuntu-latest" + }, + "corpus": { + "corpus_version": "uk-fiscal-event-replay/1", + "events": { + "spring_budget_2023": { + "registry_sha256": "5555555555555555555555555555555555555555555555555555555555555555", + "supported_years": [ + 2024, + 2025, + 2026, + 2027 + ], + "unsupported_years": [ + { + "year": 2023, + "reason": "before_data_year" + } + ], + "run_manifest_sha256": "6666666666666666666666666666666666666666666666666666666666666666", + "staging_manifest_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "comparison_provenance_sha256": "8888888888888888888888888888888888888888888888888888888888888888", + "artifacts": 8, + "full_supported_grid_complete": true, + "inert_measure_years": [] + } + } + }, + "integrity": { + "status": "complete", + "failures": [] + }, + "diff": { + "base_variant_id": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "diff_sha256": "9999999999999999999999999999999999999999999999999999999999999999", + "flag_counts": { + "numerical": 2, + "structural": 1, + "coverage": 0, + "mapping": 0, + "support": 1, + "inertness": 0, + "construction": 0 + }, + "flagged_rows": [ + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2024, + "level": "head", + "key": "universal_credit", + "flags": [ + "numerical" + ], + "base_gbp": -1.5e+16, + "head_gbp": 0.30000000000000004 + }, + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2025, + "level": "measure_year", + "key": "total", + "flags": [ + "numerical", + "structural" + ], + "base_gbp": 1e-07, + "head_gbp": -123456789.125 + }, + { + "event": "spring_budget_2023", + "measure_key": "spring_budget_2023__uc_childcare_cap_increase", + "year": 2023, + "level": "source_row", + "key": "obr-pmd-row-123", + "flags": [ + "support" + ], + "base_gbp": -2500000.0, + "head_gbp": null + } + ] + }, + "results": { + "repository": "PolicyEngine/policyengine-scorecard", + "ref": "replay-state", + "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "path": "requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/" + }, + "receipt_sha256": "a070ac40cb4e6d7daef520cfc1be4ed4958cdd4fe275975dc6d55e856d4ff515" +} diff --git a/tests/test_uk_replay_interface.py b/tests/test_uk_replay_interface.py new file mode 100644 index 0000000..7cd4c78 --- /dev/null +++ b/tests/test_uk_replay_interface.py @@ -0,0 +1,190 @@ +"""The qualification request/receipt identities shared with policyengine.py.""" + +import copy +import json +import random +from pathlib import Path + +import jsonschema +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import uk_replay_interface as ui + +FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +DIGESTS = json.loads((FIXTURES / "DIGESTS.json").read_text()) + + +def golden(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_golden_request_bytes_and_id_are_pinned(): + request = golden("qualification_request_golden.json") + assert ( + ui.jcs_bytes(request) + == (FIXTURES / "qualification_request_golden.jcs").read_bytes() + ) + assert ( + ui.validate_request(request) + == DIGESTS["qualification_request_golden.json"]["request_id"] + ) + + +def test_golden_receipt_bytes_and_digest_are_pinned(): + receipt = golden("replay_receipt_golden.json") + ui.validate(receipt, ui.RECEIPT_SCHEMA_ID) + body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} + assert ui.jcs_bytes(body) == (FIXTURES / "replay_receipt_golden.jcs").read_bytes() + assert ( + receipt["receipt_sha256"] + == DIGESTS["replay_receipt_golden.json"]["receipt_sha256"] + ) + assert receipt["request_id"] == ui.request_id( + golden("qualification_request_golden.json") + ) + + +def test_jcs_numbers_and_strings_follow_rfc8785_not_json_dumps(): + probe = golden("qualification_request_golden.json")["requested_by"] + assert ui.jcs_bytes(probe["jcs_probe"]) == ( + b"[1e-7,15000000000000000,0.30000000000000004,0,1e+21,5e-324," + b"123456789012345680000,9007199254740991]" + ) + # json.dumps writes 1e-07 and escapes non-ASCII; JCS does neither. + assert json.dumps(1e-7) == "1e-07" + assert ui.jcs_bytes("café £") == '"café £"'.encode() + + +def test_integers_beyond_double_precision_are_refused(): + with pytest.raises(Exception): + ui.jcs_bytes([2**53 + 1]) + + +def test_request_id_is_never_a_request_field(): + request = golden("qualification_request_golden.json") + with pytest.raises(ValueError): + ui.request_id({**request, "request_id": "0" * 64}) + with pytest.raises(jsonschema.ValidationError): + ui.validate_request({**request, "request_id": "0" * 64}) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda r: r.update(kind="nightly"), + lambda r: r.update(attempt=0), + lambda r: r["candidate"]["data"].update(sha256="ABC"), + lambda r: r["candidate"]["data"].update(repo_type="space"), + lambda r: r["candidate"]["policyengine_uk"].pop("wheel_sha256"), + lambda r: r["candidate"].update(extra=True), + lambda r: r["requested_by"].update(commit="481940ec"), + # A released bundle is identified by its PyPI wheel, never a branch. + lambda r: r.update(kind="released_bundle"), + ], +) +def test_request_schema_rejects_malformed_requests(mutate): + request = golden("qualification_request_golden.json") + mutate(request) + with pytest.raises(jsonschema.ValidationError): + ui.validate_request(request) + + +def test_released_bundle_request_names_its_pypi_wheel(): + request = golden("qualification_request_golden.json") + request["kind"] = "released_bundle" + request["candidate"]["policyengine"]["source"] = { + "pypi": { + "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" + } + } + assert len(ui.validate_request(request)) == 64 + + +def test_requested_by_accepts_additional_provenance(): + request = golden("qualification_request_golden.json") + request["requested_by"]["actor"] = "github-actions[bot]" + ui.validate_request(request) + + +def test_rejected_receipt_carries_a_reason_and_no_results(): + receipt = golden("replay_receipt_golden.json") + receipt.update( + status="rejected", + rejection={"code": "release_bundle_mismatch", "detail": "model_version"}, + corpus=None, + diff=None, + integrity={"status": "not_run", "failures": []}, + ) + ui.validate(ui.seal_receipt(receipt), ui.RECEIPT_SCHEMA_ID) + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "rejection": None}), ui.RECEIPT_SCHEMA_ID + ) + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt( + {**receipt, "corpus": golden("replay_receipt_golden.json")["corpus"]} + ), + ui.RECEIPT_SCHEMA_ID, + ) + + +def test_tampered_receipt_fails_its_digest(): + receipt = golden("replay_receipt_golden.json") + receipt["diff"]["flagged_rows"][0]["head_gbp"] = 0.3 + with pytest.raises(ValueError): + ui.validate(receipt, ui.RECEIPT_SCHEMA_ID) + + +def test_terminal_status_binds_receipt_digest_and_commit(): + status = { + "schema": ui.STATUS_SCHEMA_ID, + "request_id": DIGESTS["qualification_request_golden.json"]["request_id"], + "status": "complete", + "history": [ + { + "status": "queued", + "at": "2026-10-11T12:00:00Z", + "run_url": "https://github.com/PolicyEngine/policyengine-scorecard/actions/runs/1", + } + ], + } + with pytest.raises(jsonschema.ValidationError): + ui.validate(status, ui.STATUS_SCHEMA_ID) + status.update(receipt_sha256="a" * 64, receipt_commit="b" * 40) + ui.validate(status, ui.STATUS_SCHEMA_ID) + + +def _reordered(value, rng): + if isinstance(value, dict): + items = list(value.items()) + rng.shuffle(items) + return {k: _reordered(v, rng) for k, v in items} + if isinstance(value, list): + return [_reordered(v, rng) for v in value] + return value + + +@settings(max_examples=60, deadline=None) +@given(st.integers(min_value=0, max_value=2**32)) +def test_request_id_depends_on_content_not_key_order(seed): + request = golden("qualification_request_golden.json") + shuffled = _reordered(request, random.Random(seed)) + assert ui.request_id(shuffled) == ui.request_id(request) + + +@settings(max_examples=60, deadline=None) +@given( + st.floats(allow_nan=False, allow_infinity=False), + st.floats(allow_nan=False, allow_infinity=False), +) +def test_receipt_digest_changes_with_any_flagged_value(a, b): + receipt = golden("replay_receipt_golden.json") + left = copy.deepcopy(receipt) + right = copy.deepcopy(receipt) + left["diff"]["flagged_rows"][0]["head_gbp"] = a + right["diff"]["flagged_rows"][0]["head_gbp"] = b + same = ui.jcs_bytes(a) == ui.jcs_bytes(b) + assert (ui.receipt_sha256(left) == ui.receipt_sha256(right)) == same From e12e3e5e0395f13cd7d8279cee25b7ff8b77056c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:43:40 -0400 Subject: [PATCH 3/9] Run the event replay on any certified variant, validated exactly data/uk/replay_variants.json records two certified replay variants: the legacy populace-2023 world (policyengine 5.0.2, policyengine-uk 2.89.2) and policyengine.py 6.2.6's pair (policyengine-uk 2.125.1, uk-data 1.58.0). pipeline/uk_replay_variants.py checks a record's consistency, the installed wheels (versions, every installed file against RECORD, the packaged manifest), the release bundle under packaged certification, and the dataset bytes, and refuses any other combination. compute_uk_event.py takes --variant. A non-legacy variant runs the registry years its population supports, records the rest as unsupported, writes to results/uk/replays// and refuses sealed namespaces. Every managed simulation now refuses a year outside its dataset's years: on the 2024 enhanced FRS a 2023 calculation returns defaults without raising. Staging keeps unsupported-year rows as unsupported_on_variant. The legacy variant's outputs are byte-identical. Co-Authored-By: Claude Opus 5.5 --- data/uk/replay_variants.json | 195 +++++++++ docs/uk_replay/RECIPE.md | 22 + docs/uk_replay/RELEASE_REPLAY.md | 5 +- pipeline/compute_uk_event.py | 185 +++++++-- pipeline/compute_uk_obr_costings.py | 11 + pipeline/stage_uk_event.py | 54 +++ pipeline/uk_engine_registry.py | 14 +- pipeline/uk_replay_variants.py | 599 ++++++++++++++++++++++++++++ tests/test_obr_costings_registry.py | 4 + tests/test_uk_replay_variants.py | 579 +++++++++++++++++++++++++++ 10 files changed, 1642 insertions(+), 26 deletions(-) create mode 100644 data/uk/replay_variants.json create mode 100644 pipeline/uk_replay_variants.py create mode 100644 tests/test_uk_replay_variants.py diff --git a/data/uk/replay_variants.json b/data/uk/replay_variants.json new file mode 100644 index 0000000..ab0a90a --- /dev/null +++ b/data/uk/replay_variants.json @@ -0,0 +1,195 @@ +{ + "default_variant": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "schema_version": 1, + "variants": [ + { + "certification": { + "certified_for_model_version": "2.89.2", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "source": "policyengine.py 5.0.2 packaged bundle manifest; docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json" + }, + "data_years": [ + 2023, + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "data_years_evidence": { + "dataset_time_period": [ + "2023" + ], + "engine_extension": "policyengine_uk 2.89.2 data/economic_assumptions.py extend_single_year_dataset end_year=2030", + "simulation_dataset_years": [ + 2023, + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "source": "docs/uk_replay/YEARS.md" + }, + "executor": [ + "local", + "modal" + ], + "identity": { + "dataset": { + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "size_bytes": 1315880118 + }, + "materialization": "hf_cache_mirror", + "packaged_bundle_manifest_sha256": "255dbb893f455d078a6bcece87399f0a22478f56bc4c6e526b758c4fd5b4db24", + "packages": { + "policyengine": { + "record_sha256": "5d5de97a40dbc525bc41a6347fc793ba772bd78f9e0630f2f1d382bc92575e68", + "version": "5.0.2" + }, + "policyengine-core": { + "record_sha256": "dd6570fe988159d89e45df66ccc43355f70abddb8a3a58d246ef38ce711e5687", + "version": "3.27.1" + }, + "policyengine-uk": { + "record_sha256": "4628bbb8ad80eec4624fc4c18214c186d1405098fa8cf40894a4437e071e1acc", + "version": "2.89.2" + } + }, + "python": "3.12", + "release_bundle": { + "bundle_id": "uk-5.0.2", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "certified_data_build_id": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "compatibility_basis": "built_with_model_package", + "country_id": "uk", + "data_build_fingerprint": null, + "data_build_model_git_sha": null, + "data_build_model_version": "2.89.2", + "data_package": "populace-data", + "data_version": "0.1.0", + "default_dataset": "populace_uk_2023", + "default_dataset_uri": "hf://policyengine/populace-uk-private/populace_uk_2023.h5@populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "model_package": "policyengine-uk", + "model_version": "2.89.2", + "policyengine_version": "5.0.2" + } + }, + "identity_sha256": "5c642304a4fff8f30074899b2bb530197920703add1b92304a4c3c3a48c5d176", + "layout": "legacy_events", + "note": "The bundle the track 1 replays were computed against. Its 1.3GB populace dataset needs Modal's 32GiB containers for full events.", + "requirements": { + "path": "docs/uk_replay/requirements.txt", + "sha256": "450e64b173b66333027ac36e056e19928d23df1cbe2690e462bab09b31ddff9d" + }, + "result_root": "results/uk/events", + "status": "certified", + "variant_id": "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" + }, + { + "certification": { + "certified_for_model_version": "2.125.1", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_id": "policyengine-uk-data-1.58.0", + "source": "policyengine.py 6.2.6 packaged bundle manifest (PolicyEngine/policyengine.py#555, 481940eca44fc75e41fbf9de4ff454e0a0d7c0e0)" + }, + "data_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "data_years_evidence": { + "dataset_time_period": [ + "2024" + ], + "engine_extension": "policyengine_uk 2.125.1 data/economic_assumptions.py:53-56 extend_single_year_dataset end_year=2030", + "simulation_dataset_years": [ + 2024, + 2025, + 2026, + 2027, + 2028, + 2029, + 2030 + ], + "source": "Measured 2026-10-11 with the installed wheels: pandas.read_hdf(, 'time_period') and pe.uk.managed_microsimulation().dataset.years; a 2023 calculation returned defaults (income_tax 0.0) without raising." + }, + "executor": [ + "local", + "modal" + ], + "identity": { + "dataset": { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + "sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "size_bytes": 128369110 + }, + "materialization": "policyengine_data_dir", + "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", + "packages": { + "policyengine": { + "record_sha256": "072606d8f554123484caff5a1e0f12c3203b96d03f5990ebc4d80d6491441fd0", + "version": "6.2.6", + "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" + }, + "policyengine-core": { + "record_sha256": "7a4e8a82c3467681971f5160428296b06e65357b8a70801f2cd631ea0943394a", + "version": "3.33.0", + "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" + }, + "policyengine-uk": { + "record_sha256": "c39b8d44d042125cb380ff8d9f8bf75516aba129b877c416e536e63c49ce0b72", + "version": "2.125.1", + "wheel_sha256": "a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a" + } + }, + "python": "3.12", + "release_bundle": { + "bundle_id": "uk-6.2.6", + "certified_by": "policyengine.py bundle certification", + "certified_data_artifact_sha256": "5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + "compatibility_basis": "legacy_compatible_model_package", + "country_id": "uk", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_model_git_sha": "4cbedbecca352f52dffe47e849553c2759a00711", + "data_build_model_version": "2.122.2", + "data_package": "policyengine-uk-data", + "data_version": "1.58.0", + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0", + "model_package": "policyengine-uk", + "model_version": "2.125.1", + "policyengine_version": "6.2.6" + } + }, + "identity_sha256": "f241374dea3ca00aeba8849437481da34e6c6c9ddb0bb9b75d3e72bbbe22fd69", + "layout": "replay_namespace", + "note": "The newest certified pair on 11 October 2026; data built with policyengine-uk 2.122.2 and certified for 2.125.1.", + "requirements": { + "path": "docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt", + "sha256": "5c333faef091e6780b2c178ac7584249ac7db0159841921727b913505577bcdb" + }, + "result_root": "results/uk/replays/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0", + "status": "certified", + "variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" + } + ] +} diff --git a/docs/uk_replay/RECIPE.md b/docs/uk_replay/RECIPE.md index 380aa25..a274129 100644 --- a/docs/uk_replay/RECIPE.md +++ b/docs/uk_replay/RECIPE.md @@ -407,6 +407,28 @@ receipts; corrected construction explanations are supplied by this recipe and comparison annotations. A current-builder `--check` applies to a rebuilt event registry, rather than regenerating those preserved input receipts. +## Running on another certified variant + +`data/uk/replay_variants.json` lists the certified replay variants; the +default is the populace-2023 world above. `--variant ` selects another +one, validated exactly (see [RELEASE_REPLAY.md](RELEASE_REPLAY.md)): build an +environment from the variant's requirements file, then + +```bash +UV_CACHE_DIR=.venv-uv-cache uv venv --python 3.12 .venv-candidate +UV_CACHE_DIR=.venv-uv-cache uv pip install --python .venv-candidate/bin/python -r docs/uk_replay/requirements/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0.txt +PYTHONPATH=. .venv-candidate/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0 --guards +PYTHONPATH=. .venv-candidate/bin/python -m pipeline.compute_uk_event --event autumn_budget_2024 --variant pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0 --workers 2 +PYTHONPATH=. .venv-candidate/bin/python pipeline/stage_uk_event.py --event autumn_budget_2024 --artifact-dir results/uk/replays/pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0/autumn_budget_2024 +``` + +The dataset must already be local: the runner finds the certified bytes by +SHA-256 in the Hugging Face cache, or takes `--dataset-file `, and +copies them into `.venv-replay-checks/runtime//data/`. Results +default to `results/uk/replays///`. Years outside the +variant's data years are not run; they are recorded in the manifest and +staged as `unsupported_on_variant`. + ## Commands and compute budget Use the existing `.venv-replay` and `.venv-replay-checks/modal-control` for diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 829cdc2..769cdcb 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -83,7 +83,7 @@ A variant record: |---|---| | `variant_id` | `pe_uk_`; must equal the slug recomputed from `identity`. | | `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | -| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`, the SHA-256 of the installed distribution's `RECORD` file (it lists every installed file with its hash, so it binds the wheel contents). Optional `wheel_sha256` when the wheel itself was hashed. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`) and bytecode. Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | | `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | | `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | | `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`, `uri`. | @@ -114,7 +114,8 @@ failure: `certified_bundle.json`'s `revision`, `sha256`, `size_bytes`, `repo_id` and `compatible_model_packages` agree with the record. 2. **Installed environment.** Installed versions of the three packages equal - the record; each installed `RECORD` hashes to `record_sha256`; the + the record; every installed file matches its `RECORD` hash and the + normalised `RECORD` hashes to `record_sha256`; the installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; the interpreter minor version matches. 3. **Certification path.** Before importing the engine the runner removes diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index f5d02e3..aab3fca 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -25,6 +25,7 @@ from pipeline import build_uk_event_registry as registry_builder from pipeline import compute_uk_ab2025 as worlds from pipeline import compute_uk_obr_costings as fiscal +from pipeline import uk_replay_variants as variants ROOT = worlds.ROOT EVENT_PATTERN = re.compile(r"^[a-z0-9]+(?:_[a-z0-9]+)*$") @@ -189,6 +190,7 @@ def build_artifact( certified: dict, registry_sha256: str, event: str, + variant_fields: dict | None = None, ) -> dict: """Numerical facts only: identical inputs produce identical artifact bytes. @@ -275,6 +277,8 @@ def build_artifact( } if reversal: artifact["literal_reversal_minus_certified_gbp"] = literal + # Legacy artifacts carry no variant fields, so their bytes are unchanged. + artifact.update(variant_fields or {}) return artifact @@ -292,6 +296,49 @@ def preflight() -> dict: return managed +def variant_identity_fields(variant: dict) -> dict: + """Artifact and receipt identity of a non-legacy variant; legacy: none.""" + if variants.is_legacy(variant): + return {} + return { + "variant_id": variant["variant_id"], + "variant_identity_sha256": variant["identity_sha256"], + } + + +def check_variant_environment(variant: dict) -> dict: + """Engine-free: the record is certified and the installed wheels are its own.""" + variants.validate_record(variant) + return variants.verify_installed(variant) + + +def variant_preflight(variant: dict, *, dataset_source: Path | None = None) -> dict: + """Validate a non-legacy variant exactly before any simulation. + + Order matters: the certification environment is isolated before the + wrapper is imported, and the working directory moves to the runtime + directory whose ./data holds the verified dataset, where policyengine.py + 6.x materialises its default dataset. Spawned year workers inherit both. + """ + installed = check_variant_environment(variant) + variants.isolate_certification_environment() + fiscal.configure_offline() + worlds.configure_offline() + runtime_root = (variants.RUNTIME_ROOT / variant["variant_id"]).resolve() + dataset = variants.materialize_dataset(variant, runtime_root, source=dataset_source) + os.chdir(runtime_root) + import policyengine as pe + + release_bundle = dict(pe.uk.uk_latest.release_bundle) + variants.verify_release_bundle(variant, release_bundle) + return { + "release_bundle": release_bundle, + "sha256": variant["identity"]["dataset"]["sha256"], + "runtime_dataset_source": str(dataset), + "installed": installed, + } + + def run_sim(year: int, variables: list[str], reform: dict | None, pre: dict) -> dict: reform = annual_reform(reform) scenario = ( @@ -365,6 +412,7 @@ def run_year_job(job: dict) -> dict: certified=baseline, registry_sha256=job["registry_sha256"], event=job["event"], + variant_fields=job.get("variant_fields"), ) path = job["output_dir"] / f"{key}_{year}.json" fiscal.atomic_write_bytes(path, canonical_bytes(artifact)) @@ -397,6 +445,7 @@ def progress_receipt(job: dict, artifacts: dict[str, str]) -> dict: "data_bundle": fiscal.data_bundle_id(job["pre"]["release_bundle"]), "engine_version": fiscal.package_version("policyengine-uk"), **runtime_versions(), + **(job.get("variant_fields") or {}), "artifacts": artifacts, } @@ -408,8 +457,10 @@ def retained_artifact_digests( years: list[int], registry_sha256: str, pre: dict, + variant_fields: dict | None = None, ) -> dict[str, str]: """Read only SHA-bound completed outputs; unreceipted files are ignored.""" + variant_fields = variant_fields or {} receipts = [] manifest_path = output_dir / "RUN_MANIFEST.json" if manifest_path.exists(): @@ -418,6 +469,7 @@ def retained_artifact_digests( manifest.get("event") == event and manifest.get("registry_sha256") == registry_sha256 and manifest.get("certified_dataset_sha256") == pre["sha256"] + and all(manifest.get(k) == v for k, v in variant_fields.items()) ): receipts.append(manifest) for year in years: @@ -431,6 +483,7 @@ def retained_artifact_digests( "year": year, "registry_sha256": registry_sha256, "pre": pre, + "variant_fields": variant_fields, }, {}, ) @@ -474,6 +527,38 @@ def validate_years(requested: list[int], registry: dict) -> list[int]: return years +def validate_variant_years( + requested: list[int], registry: dict, variant: dict +) -> list[int]: + """Registry years the variant's population supports. + + A year before the dataset's first time period computes without error but + from absent inputs (a 2023 income tax of zero on a 2024 dataset), so an + unsupported year is refused, never run; by default it is reported. + """ + allowed = registry.get("calendar_years") + if not allowed: + raise ValueError("registry must record the supported calendar_years") + supported, unsupported = variants.supported_years(variant, allowed) + if not supported: + raise ValueError( + f"{variant['variant_id']} supports none of this event's years {allowed}" + ) + years = sorted(set(requested or supported)) + if set(years) - set(allowed): + raise ValueError( + f"years {years} outside this event's documented support {allowed}" + ) + refused = sorted(set(years) - set(supported)) + if refused: + raise ValueError( + f"years {refused} are outside {variant['variant_id']}'s data years " + f"{variant['data_years'][0]}-{variant['data_years'][-1]}: " + f"{[row for row in unsupported if row['year'] in refused]}" + ) + return years + + def verify_registry_identity(path: Path, expected_sha256: str) -> None: if fiscal.sha256_file(path) != expected_sha256: raise ValueError( @@ -510,6 +595,15 @@ def main(argv: list[str] | None = None) -> int: worlds.configure_offline() ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--event", required=True) + ap.add_argument( + "--variant", + help="certified replay variant (data/uk/replay_variants.json); default: the file's default", + ) + ap.add_argument( + "--dataset-file", + type=Path, + help="a local file holding the variant's certified dataset bytes (verified by SHA-256)", + ) ap.add_argument("--registry", type=Path) ap.add_argument("--output-dir", type=Path) ap.add_argument("--measures", nargs="*") @@ -525,9 +619,17 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) + variant = variants.get_variant(args.variant) + legacy = variants.is_legacy(variant) + variant_fields = variant_identity_fields(variant) default_registry, default_output = event_paths(args.event) - registry_path = args.registry or default_registry + if not legacy: + default_output = variants.event_output_dir(variant, args.event) + # Absolute before a variant preflight moves into its runtime directory. + registry_path = (args.registry or default_registry).resolve() output_dir = (args.output_dir or default_output).resolve() + if args.dataset_file is not None: + args.dataset_file = args.dataset_file.resolve() registry_payload = registry_path.read_bytes() registry = json.loads(registry_payload) validate_event_identity(registry, args.event) @@ -544,7 +646,11 @@ def main(argv: list[str] | None = None) -> int: "or an explicit --output-dir" ) registry_sha256 = fiscal.hashlib.sha256(registry_payload).hexdigest() - years = validate_years(args.years or [], registry) + years = ( + validate_years(args.years or [], registry) + if legacy + else validate_variant_years(args.years or [], registry, variant) + ) index = {measure["measure_key"]: measure for measure in registry["measures"]} planned = worlds.computable(index, years=years) runnable = {key: world for key, world in planned.items() if "alias_of" not in world} @@ -559,6 +665,8 @@ def main(argv: list[str] | None = None) -> int: if coverage: raise ValueError("\n".join(coverage)) if args.dry_run or args.guards: + if not legacy and args.guards: + check_variant_environment(variant) try: resolve = worlds.engine_resolver() except ImportError: @@ -570,31 +678,41 @@ def main(argv: list[str] | None = None) -> int: if gaps: raise ValueError("\n".join(gaps)) guard = "every reversal delta restates certified current law" - print( - json.dumps( - { - "event": args.event, - "years": years, - "selected": selected, - "not_computable": worlds.not_computable(index, years=years), - "guard": guard, - "annual_lookup_reforms": { - key: { - field: annual_reform(runnable[key][field]) - for field in ("baseline_reform", "reform_reform") - } - for key in selected - }, - }, - indent=1, - sort_keys=True, + plan = { + "event": args.event, + "years": years, + "selected": selected, + "not_computable": worlds.not_computable(index, years=years), + "guard": guard, + "annual_lookup_reforms": { + key: { + field: annual_reform(runnable[key][field]) + for field in ("baseline_reform", "reform_reform") + } + for key in selected + }, + } + if not legacy: + supported, unsupported = variants.supported_years( + variant, registry["calendar_years"] ) - ) + plan.update( + variant_fields, + supported_years=supported, + unsupported_years=unsupported, + output_dir=fiscal.relative_to_root(output_dir), + ) + print(json.dumps(plan, indent=1, sort_keys=True)) return 0 started = time.perf_counter() + variants.assert_not_sealed(output_dir) verify_registry_identity(registry_path, registry_sha256) print(f"[pid {os.getpid()}] starting offline bundle preflight", flush=True) - pre = preflight() + pre = ( + preflight() + if legacy + else variant_preflight(variant, dataset_source=args.dataset_file) + ) print( f"[pid {os.getpid()}] offline bundle preflight complete; starting processed-world reversal guard", flush=True, @@ -614,11 +732,21 @@ def main(argv: list[str] | None = None) -> int: years=registry["calendar_years"], registry_sha256=registry_sha256, pre=pre, + variant_fields=variant_fields, ) requested_pairs = {(key, year) for key in selected for year in years} full_pairs = { (key, year) for key in runnable for year in registry["calendar_years"] } + supported_pairs = ( + full_pairs + if legacy + else { + (key, year) + for key, year in full_pairs + if year in variants.supported_years(variant, registry["calendar_years"])[0] + } + ) expected_paths = { str((output_dir / f"{key}_{year}.json").relative_to(ROOT)): (key, year) for key, year in full_pairs @@ -648,7 +776,7 @@ def main(argv: list[str] | None = None) -> int: or previous.get("year") != year or any( previous.get(field) != value - for field, value in runtime_versions().items() + for field, value in {**runtime_versions(), **variant_fields}.items() ) ): raise ValueError("retained artifact identity differs from its receipt") @@ -679,6 +807,7 @@ def main(argv: list[str] | None = None) -> int: "registry_sha256": registry_sha256, "event": args.event, "output_dir": output_dir, + "variant_fields": variant_fields, "retained_artifacts": { str( (output_dir / f"{key}_{year}.json").relative_to(ROOT) @@ -723,6 +852,16 @@ def main(argv: list[str] | None = None) -> int: **runtime_versions(), "not_computable": worlds.not_computable(index, years=years), } + if not legacy: + manifest.update( + variants.manifest_fields( + variant, + registry["calendar_years"], + full_pairs, + supported_pairs, + artifact_pairs, + ) + ) fiscal.atomic_write_bytes( output_dir / "RUN_MANIFEST.json", canonical_bytes(manifest) ) diff --git a/pipeline/compute_uk_obr_costings.py b/pipeline/compute_uk_obr_costings.py index c95e099..eaef9c6 100644 --- a/pipeline/compute_uk_obr_costings.py +++ b/pipeline/compute_uk_obr_costings.py @@ -1070,6 +1070,17 @@ def run_managed_simulation( ] if missing: raise RuntimeError(f"managed sim lacks variables: {missing}") + # A year outside the dataset's projected years has no inputs: it + # calculates defaults without raising (a 2023 income tax of zero + # on a 2024 dataset). Both pinned engines expose dataset.years. + dataset_years = getattr(getattr(sim, "dataset", None), "years", None) + if dataset_years is not None and int(year) not in { + int(value) for value in dataset_years + }: + raise RuntimeError( + f"{year} is outside the simulation dataset's years " + f"{sorted(int(value) for value in dataset_years)}" + ) aggregates: dict[str, float] = {} for name in variables: calculated = sim.calculate(name, str(year)) diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index 02e247f..b0b0d62 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -105,6 +105,10 @@ def stage_event( """Engine-free staging; missing measures and unsupported FYs stay visible.""" artifacts = {} paths = {} + # A non-legacy variant's manifest names its identity and the registry + # years its population cannot supply (RELEASE_REPLAY.md). + variant_id = (manifest or {}).get("variant_id") + unsupported = {row["year"] for row in (manifest or {}).get("unsupported_years", [])} if manifest is not None: event = registry.get("event_slug", registry.get("event")) if event is not None and manifest.get("event", event) != event: @@ -142,6 +146,18 @@ def stage_event( registry_sha256, manifest["certified_dataset_sha256"], ) + if variant_id is not None and ( + artifact.get("variant_id") != variant_id + or artifact.get("variant_identity_sha256") + != manifest["variant_identity_sha256"] + ): + raise ValueError( + "artifact variant identity differs from the compute manifest" + ) + if year in unsupported: + raise ValueError( + f"{key} {year}: artifact in a year the variant's population does not support" + ) planned = compute.worlds.computable(index, years=registry["calendar_years"]) full_pairs = { (key, year) @@ -150,6 +166,7 @@ def stage_event( for year in registry["calendar_years"] } computed_pairs = set(artifacts) + supported_pairs = {pair for pair in full_pairs if pair[1] not in unsupported} if computed_pairs - full_pairs: raise ValueError( "committed artifact is outside the event's executable measure/year grid" @@ -217,6 +234,17 @@ def declared_pairs(field): raise ValueError( "compute manifest full-event completeness is inaccurate" ) + if variant_id is not None: + if declared_pairs("full_supported_grid") != supported_pairs: + raise ValueError( + "compute manifest supported grid differs from the variant's years" + ) + if manifest["full_supported_grid_complete"] != ( + supported_pairs <= computed_pairs + ): + raise ValueError( + "compute manifest supported-grid completeness is inaccurate" + ) elif "measures" in manifest and "years" in manifest: requested = { (key, year) @@ -308,6 +336,18 @@ def declared_pairs(field): row["reason"] = ( "FY start outside this event's documented certified-bundle support" ) + elif year in unsupported: + data_years = manifest["data_years"] + reason = next( + r["reason"] + for r in manifest["unsupported_years"] + if r["year"] == year + ) + row["status"] = "unsupported_on_variant" + row["reason"] = ( + f"CY{year} is outside {variant_id}'s supported data years " + f"{data_years[0]}–{data_years[-1]} ({reason})" + ) elif artifact is None: row["reason"] = ( "no numerical artifact for this measure and calendar year" @@ -392,6 +432,20 @@ def declared_pairs(field): } if inert_pairs: tally["inert_measure_years"] = compute.grid_rows(inert_pairs) + if variant_id is not None: + tally.update( + { + "variant_id": variant_id, + "unsupported_years": sorted(unsupported), + "unsupported_measure_years": compute.grid_rows( + full_pairs - supported_pairs + ), + "supported_grid_size": len(supported_pairs), + "full_supported_grid_complete": supported_pairs <= computed_pairs + and not inert_pairs, + "unsupported_source_rows": counts.get("unsupported_on_variant", 0), + } + ) if tally["source_rows"] != tally["staged_rows"]: raise ValueError( "staging inventory does not preserve the source-row accounting identity" diff --git a/pipeline/uk_engine_registry.py b/pipeline/uk_engine_registry.py index 625d429..11cdb22 100644 --- a/pipeline/uk_engine_registry.py +++ b/pipeline/uk_engine_registry.py @@ -4,11 +4,23 @@ import math -def engine_dumps(pin="2.89.2", system=None): +def variant_engine_pin(variant_id=None): + """The policyengine-uk version of a certified replay variant (default: the + variant registries are authored against).""" + try: + from pipeline import uk_replay_variants as variants + except ModuleNotFoundError: + import uk_replay_variants as variants + record = variants.get_variant(variant_id) + return record["identity"]["packages"]["policyengine-uk"]["version"] + + +def engine_dumps(pin=None, system=None, variant=None): """(parameter path -> 2026 value, variable name -> info, year probe).""" import policyengine_uk from policyengine_core.parameters import Parameter, ParameterNode, ParameterScale + pin = pin or variant_engine_pin(variant) installed = importlib.metadata.version("policyengine-uk") if installed != pin: raise SystemExit( diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py new file mode 100644 index 0000000..a2f1ee4 --- /dev/null +++ b/pipeline/uk_replay_variants.py @@ -0,0 +1,599 @@ +"""Certified replay variants: the exact worlds the event corpus may run on. + +A variant is one certified policyengine.py bundle's execution identity: the +wrapper, model and core distributions, the release bundle the wrapper reports +under its packaged certification, and the certified dataset's bytes. The +runner validates the selected variant exactly before any simulation and +refuses every other combination; there is no flag that relaxes a check. +Running a newer engine means adding a variant whose certification names it. +See docs/uk_replay/RELEASE_REPLAY.md. + +Engine-free: nothing here imports policyengine. Installed distributions are +read through importlib.metadata and their RECORD files. +""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib.metadata +import json +import os +import re +import shutil +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VARIANTS_PATH = ROOT / "data" / "uk" / "replay_variants.json" +CERTIFIED_BUNDLE = ROOT / "data" / "uk" / "certified_bundle.json" +LEGACY_RESULT_ROOT = "results/uk/events" +REPLAY_RESULT_ROOT = "results/uk/replays" +RUNTIME_ROOT = ROOT / ".venv-replay-checks" / "runtime" +RECEIPT_NAME = "REPLAY_RECEIPT.json" +PACKAGES = ("policyengine", "policyengine-uk", "policyengine-core") +PACKAGED_MANIFEST = "policyengine/data/bundle/manifest.json" +TOKEN_VARIABLES = ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN") +OFFLINE_VARIABLES = { + "HF_HUB_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", +} +MATERIALIZATIONS = ("hf_cache_mirror", "policyengine_data_dir") +LAYOUTS = ("legacy_events", "replay_namespace") +STATUSES = ("certified", "candidate", "retired") +EXECUTORS = ("local", "modal") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +VERSION = re.compile(r"^[0-9]+(?:\.[0-9]+)*[a-z0-9.+-]*$") +PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)") +# Installer bookkeeping differs between pip and uv for the same wheel; only +# the distribution's own files identify what was installed. +INSTALLER_METADATA = ("INSTALLER", "REQUESTED", "RECORD", "direct_url.json") + + +class VariantError(ValueError): + """A variant record, installed environment or dataset is not the certified one.""" + + +def canonical_identity_bytes(value: object) -> bytes: + """Sorted-key compact JSON. Identities hold strings, integers and nulls + only, for which this equals RFC 8785; a test checks that equality.""" + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + + +def identity_sha256(identity: dict) -> str: + return hashlib.sha256(canonical_identity_bytes(identity)).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def variant_slug(identity: dict) -> str: + packages = identity["packages"] + build = identity["release_bundle"]["certified_data_build_id"] + return ( + f"pe{packages['policyengine']['version']}" + f"_uk{packages['policyengine-uk']['version']}_{build}" + ) + + +def hf_uri(dataset: dict) -> str: + return f"hf://{dataset['repo_id']}/{dataset['path']}@{dataset['revision']}" + + +def requirements_pins(text: str) -> dict[str, str]: + """Exact `name==version` lines only, normalised names; anything else refuses.""" + pins = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + match = PIN.fullmatch(line) + if not match: + raise VariantError(f"requirements must be exact pins, found {line!r}") + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() + if name in pins: + raise VariantError(f"duplicate requirement {name}") + pins[name] = match.group(2) + return pins + + +# --- the variant file ------------------------------------------------------- + + +def load_document(path: Path = VARIANTS_PATH) -> dict: + document = json.loads(path.read_bytes()) + if document.get("schema_version") != 1: + raise VariantError("replay_variants.json schema_version must be 1") + variants = document.get("variants") + if not isinstance(variants, list) or not variants: + raise VariantError("replay_variants.json lists no variants") + ids = [variant.get("variant_id") for variant in variants] + if len(ids) != len(set(ids)): + raise VariantError("replay_variants.json repeats a variant_id") + if document.get("default_variant") not in ids: + raise VariantError("default_variant is not a listed variant") + return document + + +def get_variant( + variant_id: str | None = None, *, path: Path = VARIANTS_PATH, root: Path = ROOT +) -> dict: + """The validated record; None selects the file's default variant.""" + document = load_document(path) + wanted = variant_id or document["default_variant"] + for record in document["variants"]: + if record["variant_id"] == wanted: + validate_record(record, root=root) + return record + raise VariantError(f"no certified replay variant {wanted!r}") + + +def is_legacy(record: dict) -> bool: + """The populace-2023 world whose results keep their original layout.""" + return record["layout"] == "legacy_events" + + +def validate_record(record: dict, *, root: Path = ROOT) -> None: + """Engine-free consistency: the record is one certified pairing.""" + problems = [] + + def need(condition: bool, message: str) -> None: + if not condition: + problems.append(message) + + try: + identity = record["identity"] + packages = identity["packages"] + bundle = identity["release_bundle"] + dataset = identity["dataset"] + certification = record["certification"] + except (KeyError, TypeError) as exc: + raise VariantError(f"variant record is missing {exc}") from exc + need( + set(packages) == set(PACKAGES), + "identity.packages must name exactly the three packages", + ) + for name in PACKAGES: + package = packages.get(name) or {} + need( + isinstance(package.get("version"), str) + and bool(VERSION.fullmatch(package.get("version", ""))), + f"{name}: version", + ) + need( + bool(SHA256.fullmatch(str(package.get("record_sha256")))), + f"{name}: record_sha256", + ) + if "wheel_sha256" in package: + need( + bool(SHA256.fullmatch(str(package["wheel_sha256"]))), + f"{name}: wheel_sha256", + ) + if problems: + raise VariantError("; ".join(problems)) + uk = packages["policyengine-uk"]["version"] + wrapper = packages["policyengine"]["version"] + need( + record.get("variant_id") == variant_slug(identity), + "variant_id differs from its identity", + ) + need( + record.get("identity_sha256") == identity_sha256(identity), + "identity_sha256 differs", + ) + need( + bool(SHA256.fullmatch(str(identity.get("packaged_bundle_manifest_sha256")))), + "packaged_bundle_manifest_sha256", + ) + need( + isinstance(bundle, dict) + and all(value is None or isinstance(value, str) for value in bundle.values()), + "release_bundle values must be strings or null", + ) + need( + bundle.get("model_package") == "policyengine-uk", "release_bundle model_package" + ) + need( + bundle.get("model_version") == uk, + "release_bundle model_version differs from the installed model pin", + ) + need( + bundle.get("policyengine_version") == wrapper, + "release_bundle policyengine_version differs from the wrapper pin", + ) + need( + certification.get("certified_for_model_version") == uk, + "the certification does not name this model version", + ) + need( + certification.get("data_build_id") == bundle.get("certified_data_build_id"), + "the certification names another data build", + ) + need( + certification.get("compatibility_basis") == bundle.get("compatibility_basis"), + "certification compatibility_basis differs from the release bundle", + ) + for field in ("repo_id", "repo_type", "revision", "path", "sha256", "size_bytes"): + need(field in dataset, f"dataset.{field}") + if problems: + raise VariantError("; ".join(problems)) + need(dataset["repo_type"] in ("model", "dataset"), "dataset.repo_type") + need(bool(SHA256.fullmatch(str(dataset["sha256"]))), "dataset.sha256") + need( + isinstance(dataset["size_bytes"], int) and dataset["size_bytes"] > 0, + "dataset.size_bytes", + ) + need( + bundle.get("certified_data_artifact_sha256") == dataset["sha256"], + "the certified artifact digest differs from the dataset", + ) + need( + bundle.get("default_dataset_uri") == hf_uri(dataset), + "default_dataset_uri differs from the dataset", + ) + need( + identity.get("materialization") in MATERIALIZATIONS, "identity.materialization" + ) + need( + isinstance(identity.get("python"), str) + and bool(re.fullmatch(r"3\.[0-9]+", identity.get("python", ""))), + "identity.python", + ) + need(record.get("status") in STATUSES, "status") + need(record.get("layout") in LAYOUTS, "layout") + need( + isinstance(record.get("executor"), list) + and bool(record["executor"]) + and set(record["executor"]) <= set(EXECUTORS), + "executor", + ) + years = record.get("data_years") + need( + isinstance(years, list) + and bool(years) + and all(isinstance(year, int) and not isinstance(year, bool) for year in years) + and years == list(range(years[0], years[-1] + 1)), + "data_years must be consecutive calendar years", + ) + expected_root = ( + LEGACY_RESULT_ROOT + if record.get("layout") == "legacy_events" + else f"{REPLAY_RESULT_ROOT}/{record.get('variant_id')}" + ) + need( + record.get("result_root") == expected_root, + f"result_root must be {expected_root}", + ) + requirements = record.get("requirements") or {} + requirements_path = root / str(requirements.get("path", "")) + if not requirements.get("path") or not requirements_path.is_file(): + problems.append("requirements file is missing") + else: + payload = requirements_path.read_bytes() + need( + hashlib.sha256(payload).hexdigest() == requirements.get("sha256"), + "requirements sha256 differs", + ) + try: + pins = requirements_pins(payload.decode()) + except VariantError as exc: + problems.append(str(exc)) + else: + for name in PACKAGES: + need( + pins.get(name) == packages[name]["version"], + f"requirements pin {name}", + ) + if record.get("layout") == "legacy_events": + legacy = json.loads( + (root / "data" / "uk" / "certified_bundle.json").read_bytes() + ) + need( + legacy.get("repo_id") == dataset["repo_id"] + and legacy.get("artifact") == dataset["path"] + and legacy.get("revision") == dataset["revision"] + and legacy.get("revision") == bundle.get("certified_data_build_id") + and legacy.get("sha256") == dataset["sha256"] + and legacy.get("size_bytes") == dataset["size_bytes"], + "the legacy variant differs from data/uk/certified_bundle.json", + ) + need( + any( + row.get("name") == "policyengine-uk" + and row.get("specifier") == f"=={uk}" + for row in legacy.get("compatible_model_packages", []) + ), + "data/uk/certified_bundle.json does not declare this model version", + ) + if problems: + raise VariantError(f"{record.get('variant_id')}: " + "; ".join(problems)) + + +# --- the installed environment --------------------------------------------- + + +def _record_rows(distribution) -> list[tuple[str, str, str]]: + text = distribution.read_text("RECORD") + if text is None: + raise VariantError(f"{distribution.metadata['Name']}: no RECORD file") + rows = [] + for line in text.splitlines(): + if not line.strip(): + continue + path, digest, size = (line.rsplit(",", 2) + ["", ""])[:3] + name = path.rsplit("/", 1)[-1] + if ( + ".dist-info/" in path + and name in INSTALLER_METADATA + or "__pycache__/" in path + or path.endswith(".pyc") + ): + continue + rows.append((path, digest, size)) + return sorted(rows) + + +def distribution_record_sha256(distribution, *, verify_files: bool = True) -> str: + """Digest of the distribution's own RECORD entries, after checking every + listed file on disk against its recorded hash. Installer bookkeeping and + bytecode are excluded, so pip and uv installs of one wheel agree.""" + rows = _record_rows(distribution) + if verify_files: + for path, digest, size in rows: + if not digest: + raise VariantError(f"{path}: RECORD lists no hash") + algorithm, _, expected = digest.partition("=") + if algorithm != "sha256": + raise VariantError(f"{path}: RECORD hash is not sha256") + payload = Path(distribution.locate_file(path)).read_bytes() + actual = ( + base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + .rstrip(b"=") + .decode() + ) + if actual != expected or (size and int(size) != len(payload)): + raise VariantError( + f"installed file differs from its wheel RECORD: {path}" + ) + body = "\n".join(",".join(row) for row in rows).encode() + return hashlib.sha256(body).hexdigest() + + +def installed_identity(*, verify_files: bool = True) -> dict: + packages = {} + for name in PACKAGES: + try: + distribution = importlib.metadata.distribution(name) + except importlib.metadata.PackageNotFoundError as exc: + raise VariantError(f"{name} is not installed") from exc + packages[name] = { + "version": distribution.version, + "record_sha256": distribution_record_sha256( + distribution, verify_files=verify_files + ), + } + manifest = Path( + importlib.metadata.distribution("policyengine").locate_file(PACKAGED_MANIFEST) + ) + return { + "packages": packages, + "packaged_bundle_manifest_sha256": sha256_file(manifest), + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + } + + +def identity_mismatches(record: dict, installed: dict) -> list[str]: + identity = record["identity"] + out = [] + for name in PACKAGES: + want = identity["packages"][name] + have = installed["packages"].get(name, {}) + for field in ("version", "record_sha256"): + if want[field] != have.get(field): + out.append( + f"{name} {field}: certified {want[field]}, installed {have.get(field)}" + ) + if ( + identity["packaged_bundle_manifest_sha256"] + != installed["packaged_bundle_manifest_sha256"] + ): + out.append( + "packaged bundle manifest: certified " + f"{identity['packaged_bundle_manifest_sha256']}, installed " + f"{installed['packaged_bundle_manifest_sha256']}" + ) + if identity["python"] != installed["python"]: + out.append( + f"python: certified {identity['python']}, running {installed['python']}" + ) + return out + + +def verify_installed(record: dict, *, installed: dict | None = None) -> dict: + installed = installed if installed is not None else installed_identity() + mismatches = identity_mismatches(record, installed) + if mismatches: + raise VariantError( + f"{record['variant_id']}: installed environment is not the certified one: " + + "; ".join(mismatches) + ) + return installed + + +def isolate_certification_environment(environ=os.environ) -> list[str]: + """Use the wrapper's packaged certification on every host. + + With a Hugging Face token, policyengine.py derives the certification from + the remote release manifest instead, which changes release_bundle fields + (OFFLINE_BUNDLE_AUDIT.json records the 5.0.2 case; 6.2.6 behaves alike). + Call before importing policyengine; returns the variables removed.""" + removed = [name for name in TOKEN_VARIABLES if environ.pop(name, None) is not None] + environ.update(OFFLINE_VARIABLES) + return removed + + +def verify_release_bundle(record: dict, bundle: dict) -> None: + expected = record["identity"]["release_bundle"] + if dict(bundle) != expected: + differences = { + key: {"certified": expected.get(key), "runtime": bundle.get(key)} + for key in sorted(set(expected) | set(bundle)) + if expected.get(key) != bundle.get(key) + } + raise VariantError( + f"{record['variant_id']}: runtime release bundle differs: {differences}" + ) + + +# --- the dataset -------------------------------------------------------------- + + +def _cache_candidates(dataset: dict, hub_cache: Path) -> list[Path]: + owner, name = dataset["repo_id"].split("/", 1) + prefix = "models" if dataset["repo_type"] == "model" else "datasets" + snapshots = hub_cache / f"{prefix}--{owner}--{name}" / "snapshots" + return sorted(snapshots.glob(f"*/{dataset['path']}")) if snapshots.is_dir() else [] + + +def find_dataset_bytes(record: dict, *, source: Path | None = None) -> Path: + """A local file holding the certified bytes, found by digest, never by name.""" + dataset = record["identity"]["dataset"] + hub_cache = Path( + os.environ.get("HF_HUB_CACHE") + or Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) + / "hub" + ) + candidates = ( + [source] if source is not None else _cache_candidates(dataset, hub_cache) + ) + for candidate in candidates: + path = Path(candidate) + if ( + path.is_file() + and path.stat().st_size == dataset["size_bytes"] + and sha256_file(path) == dataset["sha256"] + ): + return path + raise VariantError( + f"{record['variant_id']}: no local file holds the certified dataset " + f"{dataset['path']} (sha256 {dataset['sha256']}); searched {[str(c) for c in candidates]}" + ) + + +def materialize_dataset( + record: dict, runtime_root: Path | None = None, *, source: Path | None = None +) -> Path: + """Place verified certified bytes where policyengine.py 6.x looks: the + runtime directory's ./data/ (provenance/dataset_materialization.py, + DEFAULT_DATA_DIR). Returns the absolute path.""" + if record["identity"]["materialization"] != "policyengine_data_dir": + raise VariantError( + f"{record['variant_id']} does not materialise into a data dir" + ) + dataset = record["identity"]["dataset"] + runtime_root = (runtime_root or RUNTIME_ROOT / record["variant_id"]).resolve() + destination = runtime_root / "data" / Path(dataset["path"]).name + if destination.is_symlink(): + raise VariantError(f"{destination} is a symlink; refusing it") + if destination.resolve().parent != (runtime_root / "data").resolve(): + raise VariantError("dataset destination escapes the runtime directory") + if ( + destination.is_file() + and destination.stat().st_size == dataset["size_bytes"] + and sha256_file(destination) == dataset["sha256"] + ): + return destination + origin = find_dataset_bytes(record, source=source) + destination.parent.mkdir(parents=True, exist_ok=True) + handle, temporary = tempfile.mkstemp(dir=destination.parent, prefix=".materialize-") + os.close(handle) + temporary = Path(temporary) + try: + shutil.copyfile(origin, temporary) + if sha256_file(temporary) != dataset["sha256"]: + raise VariantError("dataset copy failed SHA-256 verification") + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + return destination + + +# --- years and namespaces ----------------------------------------------------- + + +def supported_years(record: dict, registry_years) -> tuple[list[int], list[dict]]: + """Partition an event's registry years by the variant's data years.""" + data = record["data_years"] + supported = sorted( + year for year in set(registry_years) if data[0] <= year <= data[-1] + ) + unsupported = [ + { + "year": year, + "reason": "before_data_year" if year < data[0] else "after_extension_end", + } + for year in sorted(set(registry_years)) + if not data[0] <= year <= data[-1] + ] + return supported, unsupported + + +def namespace_root(record: dict, *, root: Path = ROOT) -> Path: + return root / record["result_root"] + + +def event_output_dir(record: dict, event: str, *, root: Path = ROOT) -> Path: + return namespace_root(record, root=root) / event + + +def sealing_receipt(path: Path, *, root: Path = ROOT) -> Path | None: + """The REPLAY_RECEIPT.json sealing a directory or one of its parents.""" + path = path.resolve() + root = root.resolve() + for directory in (path, *path.parents): + if (directory / RECEIPT_NAME).is_file(): + return directory / RECEIPT_NAME + if directory == root: + break + return None + + +def assert_not_sealed(path: Path, *, root: Path = ROOT) -> None: + receipt = sealing_receipt(path, root=root) + if receipt is not None: + raise VariantError( + f"{path} is inside a sealed replay namespace ({receipt}); write a new namespace" + ) + + +def manifest_fields( + record: dict, registry_years, full_pairs, supported_pairs, artifact_pairs +) -> dict: + """RUN_MANIFEST additions for a non-legacy variant (RELEASE_REPLAY.md).""" + supported, unsupported = supported_years(record, registry_years) + return { + "variant_id": record["variant_id"], + "variant_identity_sha256": record["identity_sha256"], + "data_years": record["data_years"], + "supported_years": supported, + "unsupported_years": unsupported, + "full_supported_grid": [ + {"measure_key": key, "year": year} for key, year in sorted(supported_pairs) + ], + "full_supported_grid_complete": set(supported_pairs) <= set(artifact_pairs), + } diff --git a/tests/test_obr_costings_registry.py b/tests/test_obr_costings_registry.py index 24ed0f1..3958d82 100644 --- a/tests/test_obr_costings_registry.py +++ b/tests/test_obr_costings_registry.py @@ -371,6 +371,10 @@ def test_dividend_lag_overrides_match_installed_engine_dates_per_path(): from policyengine_uk import CountryTaxBenefitSystem except ImportError as exc: pytest.skip(f"policyengine-uk cannot be imported: {exc}") + # The overrides record the dates of the registry's pinned engine; a + # replay environment for another certified variant has other dates. + if compute.package_version("policyengine-uk") != "2.89.2": + pytest.skip("the OBR-costing registry's dividend overrides pin 2.89.2") system = CountryTaxBenefitSystem() rates = system.parameters.gov.hmrc.income_tax.rates diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py new file mode 100644 index 0000000..7637a64 --- /dev/null +++ b/tests/test_uk_replay_variants.py @@ -0,0 +1,579 @@ +"""Certified replay variants: exact validation, support and namespaces. + +Engine-free and dataset-free: installed distributions are synthesised as +dist-info directories, simulations are stubbed, and the dataset is a few +bytes with a known digest. +""" + +import base64 +import copy +import hashlib +import importlib.metadata +import json +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from pipeline import compute_uk_event as compute +from pipeline import stage_uk_event as stage +from pipeline import uk_replay_variants as rv + +LEGACY = "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" +CURRENT = "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" + + +def record(variant_id=CURRENT): + return copy.deepcopy(rv.get_variant(variant_id)) + + +def reseal(r): + """Recompute a mutated record's own identity fields, so only the + consistency rule under test can refuse it.""" + r["identity_sha256"] = rv.identity_sha256(r["identity"]) + r["variant_id"] = rv.variant_slug(r["identity"]) + if r["layout"] == "replay_namespace": + r["result_root"] = f"{rv.REPLAY_RESULT_ROOT}/{r['variant_id']}" + return r + + +def test_committed_variants_validate_and_default_is_legacy(): + document = rv.load_document() + assert document["default_variant"] == LEGACY + assert {v["variant_id"] for v in document["variants"]} == {LEGACY, CURRENT} + for variant in document["variants"]: + rv.validate_record(variant) + assert rv.is_legacy(rv.get_variant()) + assert not rv.is_legacy(rv.get_variant(CURRENT)) + with pytest.raises(rv.VariantError, match="no certified replay variant"): + rv.get_variant("pe9.9.9_uk9.9.9_unknown") + + +def test_identity_bytes_equal_rfc8785_for_committed_identities(): + rfc8785 = pytest.importorskip("rfc8785") + for variant in rv.load_document()["variants"]: + assert rv.canonical_identity_bytes(variant["identity"]) == rfc8785.dumps( + variant["identity"] + ) + + +INCONSISTENT = [ + ( + "model version", + lambda r: r["identity"]["release_bundle"].update(model_version="2.126.8"), + ), + ( + "wrapper version", + lambda r: r["identity"]["release_bundle"].update(policyengine_version="6.2.7"), + ), + ( + "model package", + lambda r: r["identity"]["release_bundle"].update( + model_package="policyengine-us" + ), + ), + ( + "artifact digest", + lambda r: r["identity"]["release_bundle"].update( + certified_data_artifact_sha256="0" * 64 + ), + ), + ( + "dataset uri", + lambda r: r["identity"]["release_bundle"].update( + default_dataset_uri="hf://x/y.h5@1" + ), + ), + ("dataset bytes", lambda r: r["identity"]["dataset"].update(sha256="1" * 64)), + ("dataset size", lambda r: r["identity"]["dataset"].update(size_bytes=0)), + ( + "certified model", + lambda r: r["certification"].update(certified_for_model_version="2.125.0"), + ), + ( + "certified build", + lambda r: r["certification"].update( + data_build_id="policyengine-uk-data-1.57.0" + ), + ), + ( + "basis", + lambda r: r["certification"].update( + compatibility_basis="exact_build_model_version" + ), + ), + ("materialization", lambda r: r["identity"].update(materialization="download")), + ("python", lambda r: r["identity"].update(python="3")), + ("status", lambda r: r.update(status="promoted")), + ("executor", lambda r: r.update(executor=["kubernetes"])), + ("data years gap", lambda r: r.update(data_years=[2024, 2026])), + ("data years empty", lambda r: r.update(data_years=[])), + ("requirements bytes", lambda r: r["requirements"].update(sha256="2" * 64)), + ( + "requirements file", + lambda r: r["requirements"].update(path="docs/uk_replay/nope.txt"), + ), + ( + "installed hash", + lambda r: r["identity"]["packages"]["policyengine-uk"].update( + record_sha256="short" + ), + ), + ( + "extra package", + lambda r: r["identity"]["packages"].update( + {"policyengine-us": {"version": "1", "record_sha256": "3" * 64}} + ), + ), + ("result root", lambda r: r.update(result_root="results/uk/events")), +] + + +@pytest.mark.parametrize("label,mutate", INCONSISTENT, ids=[x[0] for x in INCONSISTENT]) +def test_each_inconsistent_record_is_refused(label, mutate): + r = record() + mutate(r) + if label != "result root": + reseal(r) + with pytest.raises(rv.VariantError): + rv.validate_record(r) + + +def test_identity_tampering_is_refused_without_resealing(): + r = record() + r["identity"]["packages"]["policyengine-uk"]["record_sha256"] = "4" * 64 + with pytest.raises(rv.VariantError, match="identity_sha256"): + rv.validate_record(r) + r = record() + r["variant_id"] = "pe6.2.6_uk2.125.1_other-data" + with pytest.raises(rv.VariantError, match="variant_id"): + rv.validate_record(r) + + +def test_legacy_record_must_agree_with_certified_bundle(tmp_path): + for relative in ( + "docs/uk_replay/requirements.txt", + "data/uk/certified_bundle.json", + ): + (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / relative).write_bytes((rv.ROOT / relative).read_bytes()) + rv.validate_record(record(LEGACY), root=tmp_path) + pinned = json.loads((tmp_path / "data/uk/certified_bundle.json").read_text()) + pinned["compatible_model_packages"][0]["specifier"] = "==2.90.0" + (tmp_path / "data/uk/certified_bundle.json").write_text(json.dumps(pinned)) + with pytest.raises(rv.VariantError, match="does not declare this model"): + rv.validate_record(record(LEGACY), root=tmp_path) + + +def installed_for(r): + identity = r["identity"] + return { + "packages": { + name: { + "version": identity["packages"][name]["version"], + "record_sha256": identity["packages"][name]["record_sha256"], + } + for name in rv.PACKAGES + }, + "packaged_bundle_manifest_sha256": identity["packaged_bundle_manifest_sha256"], + "python": identity["python"], + } + + +def test_installed_environment_must_match_every_field(): + r = record() + exact = installed_for(r) + assert rv.verify_installed(r, installed=exact) == exact + mutations = [] + for name in rv.PACKAGES: + for field in ("version", "record_sha256"): + mutations.append(("packages", name, field)) + mutations += [("packaged_bundle_manifest_sha256",), ("python",)] + for path in mutations: + changed = copy.deepcopy(exact) + node = changed + for part in path[:-1]: + node = node[part] + node[path[-1]] = "different" + with pytest.raises(rv.VariantError, match="not the certified one"): + rv.verify_installed(r, installed=changed) + + +def test_one_environment_never_validates_two_variants(): + current, legacy = record(CURRENT), record(LEGACY) + assert rv.identity_mismatches(legacy, installed_for(current)) + assert rv.identity_mismatches(current, installed_for(legacy)) + + +def write_distribution(site: Path, files: dict[str, bytes], installer: bytes): + info = site / "demo-1.0.dist-info" + info.mkdir(parents=True) + rows = [] + for relative, payload in { + **files, + "demo-1.0.dist-info/INSTALLER": installer, + }.items(): + path = site / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=") + rows.append(f"{relative},sha256={digest.decode()},{len(payload)}") + rows += ["demo-1.0.dist-info/RECORD,,", "demo/__pycache__/x.cpython-312.pyc,,"] + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: demo\nVersion: 1.0\n") + (info / "RECORD").write_text("\n".join(rows) + "\n") + return importlib.metadata.PathDistribution(info) + + +def test_record_digest_binds_wheel_files_not_the_installer(tmp_path): + files = {"demo/__init__.py": b"x = 1\n", "demo/data/manifest.json": b"{}"} + pip = write_distribution(tmp_path / "pip", files, b"pip\n") + uv = write_distribution(tmp_path / "uv", files, b"uv\n") + assert rv.distribution_record_sha256(pip) == rv.distribution_record_sha256(uv) + other = write_distribution( + tmp_path / "other", {**files, "demo/__init__.py": b"x = 2\n"}, b"uv\n" + ) + assert rv.distribution_record_sha256(other) != rv.distribution_record_sha256(uv) + (tmp_path / "uv" / "demo" / "__init__.py").write_bytes(b"x = 3\n") + with pytest.raises(rv.VariantError, match="differs from its wheel RECORD"): + rv.distribution_record_sha256(uv) + + +def test_certification_is_isolated_from_tokens(): + environ = {"HF_TOKEN": "secret", "HUGGING_FACE_TOKEN": "secret", "PATH": "/bin"} + removed = rv.isolate_certification_environment(environ) + assert sorted(removed) == ["HF_TOKEN", "HUGGING_FACE_TOKEN"] + assert not set(rv.TOKEN_VARIABLES) & set(environ) + assert environ["HF_HUB_OFFLINE"] == "1" and environ["PATH"] == "/bin" + + +def test_release_bundle_must_match_exactly(): + r = record() + bundle = dict(r["identity"]["release_bundle"]) + rv.verify_release_bundle(r, bundle) + for change in ( + {"compatibility_basis": "exact_build_model_version"}, + {"model_version": "2.126.8"}, + {"extra": "field"}, + ): + with pytest.raises(rv.VariantError, match="runtime release bundle differs"): + rv.verify_release_bundle(r, {**bundle, **change}) + missing = dict(bundle) + missing.pop("data_build_fingerprint") + with pytest.raises(rv.VariantError): + rv.verify_release_bundle(r, missing) + + +def tiny_variant(payload: bytes): + r = record() + r["identity"]["dataset"].update( + sha256=hashlib.sha256(payload).hexdigest(), size_bytes=len(payload) + ) + return r + + +def test_materialisation_copies_only_verified_bytes(tmp_path): + payload = b"certified bytes" + r = tiny_variant(payload) + source = tmp_path / "cache" / "enhanced_frs_2024_25.h5" + source.parent.mkdir() + source.write_bytes(payload) + runtime = tmp_path / "runtime" + placed = rv.materialize_dataset(r, runtime, source=source) + assert placed == (runtime / "data" / "enhanced_frs_2024_25.h5").resolve() + assert placed.read_bytes() == payload + assert rv.materialize_dataset(r, runtime, source=tmp_path / "absent") == placed + impostor = tmp_path / "impostor" / "enhanced_frs_2024_25.h5" + impostor.parent.mkdir() + impostor.write_bytes(b"other bytes same len") + placed.unlink() + with pytest.raises(rv.VariantError, match="no local file holds"): + rv.materialize_dataset(r, runtime, source=impostor) + impostor.write_bytes(b"certified bytez") + with pytest.raises(rv.VariantError, match="no local file holds"): + rv.materialize_dataset(r, runtime, source=impostor) + placed.symlink_to(source) + with pytest.raises(rv.VariantError, match="symlink"): + rv.materialize_dataset(r, runtime, source=source) + with pytest.raises(rv.VariantError, match="does not materialise"): + rv.materialize_dataset(record(LEGACY), runtime, source=source) + + +def test_materialisation_finds_the_cache_copy_by_digest(tmp_path, monkeypatch): + payload = b"certified bytes" + r = tiny_variant(payload) + hub = tmp_path / "hub" + snapshots = hub / "models--policyengine--policyengine-uk-data-private" / "snapshots" + (snapshots / "aaa").mkdir(parents=True) + (snapshots / "aaa" / "enhanced_frs_2024_25.h5").write_bytes(b"older release") + (snapshots / "bbb").mkdir() + (snapshots / "bbb" / "enhanced_frs_2024_25.h5").write_bytes(payload) + monkeypatch.setenv("HF_HUB_CACHE", str(hub)) + placed = rv.materialize_dataset(r, tmp_path / "runtime") + assert placed.read_bytes() == payload + + +@settings(max_examples=200, deadline=None) +@given( + st.integers(2015, 2035), + st.integers(0, 10), + st.sets(st.integers(2010, 2040), min_size=1, max_size=12), +) +def test_supported_year_partition_is_exact(first, span, registry_years): + r = {"data_years": list(range(first, first + span + 1))} + supported, unsupported = rv.supported_years(r, registry_years) + unsupported_years = [row["year"] for row in unsupported] + assert sorted(supported + unsupported_years) == sorted(registry_years) + assert not set(supported) & set(unsupported_years) + assert supported == sorted(supported) + assert all(first <= year <= first + span for year in supported) + for row in unsupported: + assert row["reason"] == ( + "before_data_year" if row["year"] < first else "after_extension_end" + ) + + +def test_sealed_namespace_refuses_writes(tmp_path): + namespace = tmp_path / "results" / "uk" / "replays" / CURRENT + (namespace / "spring_budget_2023").mkdir(parents=True) + rv.assert_not_sealed(namespace / "spring_budget_2023", root=tmp_path) + (namespace / rv.RECEIPT_NAME).write_text("{}") + with pytest.raises(rv.VariantError, match="sealed"): + rv.assert_not_sealed(namespace / "spring_budget_2023", root=tmp_path) + with pytest.raises(rv.VariantError, match="sealed"): + rv.assert_not_sealed(namespace / "spring_budget_2023" / "deeper", root=tmp_path) + + +def test_variant_years_refuse_an_explicit_unsupported_year(): + r = record() + registry = {"calendar_years": [2023, 2024, 2025]} + assert compute.validate_variant_years([], registry, r) == [2024, 2025] + with pytest.raises(ValueError, match="outside .*data years"): + compute.validate_variant_years([2023], registry, r) + with pytest.raises(ValueError, match="supports none"): + compute.validate_variant_years([], {"calendar_years": [2022, 2023]}, r) + with pytest.raises(ValueError, match="documented support"): + compute.validate_variant_years([2026], registry, r) + + +def measure(): + rows = [] + for fy, tax, benefit in (("2023-24", 90, -20), ("2024-25", 100, -30)): + rows += [ + { + "source_row_id": f"tax_{fy}", + "fy": fy, + "metric": "revenue_change", + "tax_head": "Income tax", + "value_gbp": tax, + }, + { + "source_row_id": f"benefit_{fy}", + "fy": fy, + "metric": "exchequer_impact", + "tax_head": "Welfare inside cap", + "value_gbp": benefit, + }, + ] + rows.append( + { + "source_row_id": "corporation_2023-24", + "fy": "2023-24", + "metric": "revenue_change", + "tax_head": "Corporation tax", + "value_gbp": 4, + "classification": "out_of_household_scope", + } + ) + return { + "measure_key": "test_event__tax_and_benefit", + "title": "An announcement", + "computability": "partial", + "construction": "reversal_on_certified_world", + "commences_fy": "2023-24", + "pe_baseline_modifier": {"gov.tax.rate": 0.2}, + "head_variables": ["income_tax", "universal_credit"], + "heads": [ + { + "obr_head": "Income tax", + "pe_variables": ["income_tax"], + "channel": "tax", + }, + { + "obr_head": "Welfare inside cap", + "pe_variables": ["universal_credit"], + "channel": "spending", + }, + ], + "source_rows": rows, + } + + +def simulation(tax, benefit): + return { + "aggregates_gbp": { + "income_tax": float(tax), + "universal_credit": float(benefit), + }, + "variable_metadata": { + "income_tax": {"entity": "person", "definition_period": "year"}, + "universal_credit": {"entity": "benunit", "definition_period": "year"}, + }, + "policyengine_bundle": { + "model_version": "2.125.1", + "certified_data_build_id": "policyengine-uk-data-1.58.0", + }, + "dataset_sha256_before": "a" * 64, + "dataset_sha256_after": "a" * 64, + "performance": {"wall_seconds": 1}, + } + + +def test_variant_run_skips_unsupported_years_and_stages_them_visibly( + tmp_path, monkeypatch +): + monkeypatch.setattr(compute, "ROOT", tmp_path) + registry_path = tmp_path / "registry.json" + registry_document = {"calendar_years": [2023, 2024], "measures": [measure()]} + registry_path.write_text(json.dumps(registry_document)) + monkeypatch.setattr(compute.registry_builder, "validate_registry", lambda d: None) + monkeypatch.setattr(compute.worlds, "engine_resolver", lambda: lambda p, d: None) + monkeypatch.setattr( + compute, + "preflight", + lambda: (_ for _ in ()).throw(AssertionError("legacy preflight for a variant")), + ) + monkeypatch.setattr( + compute, + "variant_preflight", + lambda variant, dataset_source=None: { + "sha256": "a" * 64, + "release_bundle": { + "certified_data_build_id": "policyengine-uk-data-1.58.0" + }, + }, + ) + years = [] + + def fake_run(year, variables, reform, pre): + years.append(year) + return simulation(900, 1000) if reform else simulation(1000, 1030) + + monkeypatch.setattr(compute, "run_sim", fake_run) + output = tmp_path / "results" / "uk" / "replays" / CURRENT / "test_event" + args = [ + "--event", + "test_event", + "--variant", + CURRENT, + "--registry", + str(registry_path), + "--output-dir", + str(output), + ] + assert compute.main(args) == 0 + assert set(years) == {2024} + manifest = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + assert manifest["variant_id"] == CURRENT + assert manifest["variant_identity_sha256"] == record()["identity_sha256"] + assert manifest["supported_years"] == [2024] + assert manifest["unsupported_years"] == [ + {"year": 2023, "reason": "before_data_year"} + ] + assert manifest["full_event_complete"] is False + assert manifest["full_supported_grid_complete"] is True + artifact = json.loads( + (output / "test_event__tax_and_benefit_2024.json").read_bytes() + ) + assert artifact["variant_id"] == CURRENT + rows, tally = stage.stage_event( + registry_document, + manifest, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + by_id = {row["source_row_id"]: row for row in rows} + assert by_id["tax_2024-25"]["pe_value"] == 100 + assert by_id["tax_2023-24"]["status"] == "unsupported_on_variant" + assert by_id["tax_2023-24"]["pe_value"] is None + assert "2024–2030" in by_id["tax_2023-24"]["reason"] + assert by_id["corporation_2023-24"]["computability"] == "out_of_household_scope" + assert by_id["corporation_2023-24"]["status"] == "not_computed" + assert tally["unsupported_source_rows"] == 2 + assert tally["unsupported_years"] == [2023] + assert tally["full_supported_grid_complete"] is True + assert tally["source_rows"] == tally["staged_rows"] == len(rows) == 5 + # Resume within the same variant reuses receipted artifacts. + years.clear() + assert compute.main(args + ["--resume"]) == 0 + assert years == [] + # A forged unsupported-year artifact is refused by staging. + forged = copy.deepcopy(manifest) + forged["unsupported_years"] = [{"year": 2024, "reason": "before_data_year"}] + with pytest.raises(ValueError, match="does not support|supported grid"): + stage.stage_event( + registry_document, + forged, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + other = copy.deepcopy(manifest) + other["variant_identity_sha256"] = "f" * 64 + with pytest.raises(ValueError, match="variant identity"): + stage.stage_event( + registry_document, + other, + artifact_dir=output, + registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), + ) + (output.parent / rv.RECEIPT_NAME).write_text("{}") + with pytest.raises(rv.VariantError, match="sealed"): + compute.main(args + ["--resume"]) + + +def test_dataset_year_guard_refuses_years_without_inputs(monkeypatch, tmp_path): + from pipeline import compute_uk_obr_costings as fiscal + + data_file = tmp_path / "data.h5" + data_file.write_bytes(b"x") + digest = hashlib.sha256(b"x").hexdigest() + + class Sim: + policyengine_bundle = { + "certified_data_build_id": "build", + "runtime_dataset_source": str(data_file), + } + tax_benefit_system = type("System", (), {"variables": {"income_tax": None}})() + dataset = type("Dataset", (), {"years": [2024, 2025]})() + + def calculate(self, name, period): + import numpy as np + + return np.array([1.0]) + + class Uk: + @staticmethod + def managed_microsimulation(**kwargs): + return Sim() + + import sys + import types + + monkeypatch.setitem(sys.modules, "policyengine", types.SimpleNamespace(uk=Uk)) + monkeypatch.setattr(fiscal, "variable_metadata", lambda *a, **k: {}) + ok = fiscal.run_managed_simulation( + year=2024, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=digest, + ) + assert ok["aggregates_gbp"] == {"income_tax": 1.0} + with pytest.raises(RuntimeError, match="outside the simulation dataset's years"): + fiscal.run_managed_simulation( + year=2023, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=digest, + ) From 62704beea0439921d77b3175d4d2b64bfdf447db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:49:05 -0400 Subject: [PATCH 4/9] Receipt: optional results.manifest_sha256 binding a sealed namespace NAMESPACE_MANIFEST.json lists the SHA-256 of every file in a sealed result namespace; the receipt names its digest. Additive and optional, so the schema stays v1. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/schemas/replay_receipt.v1.json | 539 +++++++++++++++--- 1 file changed, 452 insertions(+), 87 deletions(-) diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json index 2aac386..c5d8d9a 100644 --- a/docs/uk_replay/schemas/replay_receipt.v1.json +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -5,69 +5,178 @@ "description": "Returned by policyengine-scorecard for one qualification request. receipt_sha256 is the SHA-256 of the RFC 8785 (JCS) bytes of the receipt without that field. Flags describe movements against the base variant; they carry no verdict.", "type": "object", "additionalProperties": false, - "required": ["schema", "request_id", "status", "rejection", "variant_id", "variant_identity_sha256", "execution", "corpus", "integrity", "diff", "results", "receipt_sha256"], + "required": [ + "schema", + "request_id", + "status", + "rejection", + "variant_id", + "variant_identity_sha256", + "execution", + "corpus", + "integrity", + "diff", + "results", + "receipt_sha256" + ], "properties": { - "schema": {"const": "policyengine.uk-replay-receipt/v1"}, - "request_id": {"$ref": "#/$defs/sha256"}, - "status": {"enum": ["complete", "rejected", "failed"]}, + "schema": { + "const": "policyengine.uk-replay-receipt/v1" + }, + "request_id": { + "$ref": "#/$defs/sha256" + }, + "status": { + "enum": [ + "complete", + "rejected", + "failed" + ] + }, "rejection": { "oneOf": [ - {"type": "null"}, + { + "type": "null" + }, { "type": "object", "additionalProperties": false, - "required": ["code", "detail"], + "required": [ + "code", + "detail" + ], "properties": { - "code": {"enum": ["schema_invalid", "artifact_hash_mismatch", "release_bundle_mismatch", "incompatible_pair", "data_access_failed", "no_supported_years"]}, - "detail": {"type": "string", "minLength": 1} + "code": { + "enum": [ + "schema_invalid", + "artifact_hash_mismatch", + "release_bundle_mismatch", + "incompatible_pair", + "data_access_failed", + "no_supported_years" + ] + }, + "detail": { + "type": "string", + "minLength": 1 + } } } ] }, - "variant_id": {"type": ["string", "null"], "minLength": 1}, - "variant_identity_sha256": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/sha256"}]}, + "variant_id": { + "type": [ + "string", + "null" + ], + "minLength": 1 + }, + "variant_identity_sha256": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/sha256" + } + ] + }, "execution": { "oneOf": [ - {"type": "null"}, + { + "type": "null" + }, { "type": "object", "additionalProperties": false, - "required": ["packages", "packaged_bundle_manifest_sha256", "release_bundle", "dataset_sha256", "requirements_sha256", "python", "platform", "runner"], + "required": [ + "packages", + "packaged_bundle_manifest_sha256", + "release_bundle", + "dataset_sha256", + "requirements_sha256", + "python", + "platform", + "runner" + ], "properties": { "packages": { "type": "object", "additionalProperties": false, - "required": ["policyengine", "policyengine-uk", "policyengine-core"], + "required": [ + "policyengine", + "policyengine-uk", + "policyengine-core" + ], "properties": { - "policyengine": {"$ref": "#/$defs/installed_package"}, - "policyengine-uk": {"$ref": "#/$defs/installed_package"}, - "policyengine-core": {"$ref": "#/$defs/installed_package"} + "policyengine": { + "$ref": "#/$defs/installed_package" + }, + "policyengine-uk": { + "$ref": "#/$defs/installed_package" + }, + "policyengine-core": { + "$ref": "#/$defs/installed_package" + } + } + }, + "packaged_bundle_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "release_bundle": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "null" + ] } }, - "packaged_bundle_manifest_sha256": {"$ref": "#/$defs/sha256"}, - "release_bundle": {"type": "object", "additionalProperties": {"type": ["string", "null"]}}, - "dataset_sha256": {"$ref": "#/$defs/sha256"}, - "requirements_sha256": {"$ref": "#/$defs/sha256"}, - "python": {"type": "string", "pattern": "^3\\.[0-9]+\\.[0-9]+"}, - "platform": {"type": "string", "minLength": 1}, - "runner": {"type": "string", "minLength": 1} + "dataset_sha256": { + "$ref": "#/$defs/sha256" + }, + "requirements_sha256": { + "$ref": "#/$defs/sha256" + }, + "python": { + "type": "string", + "pattern": "^3\\.[0-9]+\\.[0-9]+" + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "runner": { + "type": "string", + "minLength": 1 + } } } ] }, "corpus": { "oneOf": [ - {"type": "null"}, + { + "type": "null" + }, { "type": "object", "additionalProperties": false, - "required": ["corpus_version", "events"], + "required": [ + "corpus_version", + "events" + ], "properties": { - "corpus_version": {"type": "string", "minLength": 1}, + "corpus_version": { + "type": "string", + "minLength": 1 + }, "events": { "type": "object", "minProperties": 1, - "additionalProperties": {"$ref": "#/$defs/event_result"} + "additionalProperties": { + "$ref": "#/$defs/event_result" + } } } } @@ -76,24 +185,47 @@ "integrity": { "type": "object", "additionalProperties": false, - "required": ["status", "failures"], + "required": [ + "status", + "failures" + ], "properties": { "status": { "description": "complete: every supported measure-year ran and every receipt verified. incomplete: some supported pairs are missing. failed: a crash, a non-finite value, an identity mismatch after the run started, or a broken accounting identity.", - "enum": ["complete", "incomplete", "failed", "not_run"] + "enum": [ + "complete", + "incomplete", + "failed", + "not_run" + ] }, "failures": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["code", "detail"], + "required": [ + "code", + "detail" + ], "properties": { - "code": {"type": "string", "minLength": 1}, - "detail": {"type": "string", "minLength": 1}, - "event": {"type": "string"}, - "measure_key": {"type": "string"}, - "year": {"type": "integer"} + "code": { + "type": "string", + "minLength": 1 + }, + "detail": { + "type": "string", + "minLength": 1 + }, + "event": { + "type": "string" + }, + "measure_key": { + "type": "string" + }, + "year": { + "type": "integer" + } } } } @@ -101,26 +233,67 @@ }, "diff": { "oneOf": [ - {"type": "null"}, + { + "type": "null" + }, { "type": "object", "additionalProperties": false, - "required": ["base_variant_id", "diff_sha256", "flag_counts", "flagged_rows"], + "required": [ + "base_variant_id", + "diff_sha256", + "flag_counts", + "flagged_rows" + ], "properties": { - "base_variant_id": {"type": "string", "minLength": 1}, - "diff_sha256": {"$ref": "#/$defs/sha256"}, + "base_variant_id": { + "type": "string", + "minLength": 1 + }, + "diff_sha256": { + "$ref": "#/$defs/sha256" + }, "flag_counts": { "type": "object", "additionalProperties": false, - "required": ["numerical", "structural", "coverage", "mapping", "support", "inertness", "construction"], + "required": [ + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction" + ], "properties": { - "numerical": {"type": "integer", "minimum": 0}, - "structural": {"type": "integer", "minimum": 0}, - "coverage": {"type": "integer", "minimum": 0}, - "mapping": {"type": "integer", "minimum": 0}, - "support": {"type": "integer", "minimum": 0}, - "inertness": {"type": "integer", "minimum": 0}, - "construction": {"type": "integer", "minimum": 0} + "numerical": { + "type": "integer", + "minimum": 0 + }, + "structural": { + "type": "integer", + "minimum": 0 + }, + "coverage": { + "type": "integer", + "minimum": 0 + }, + "mapping": { + "type": "integer", + "minimum": 0 + }, + "support": { + "type": "integer", + "minimum": 0 + }, + "inertness": { + "type": "integer", + "minimum": 0 + }, + "construction": { + "type": "integer", + "minimum": 0 + } } }, "flagged_rows": { @@ -128,21 +301,67 @@ "items": { "type": "object", "additionalProperties": false, - "required": ["event", "measure_key", "year", "level", "key", "flags", "base_gbp", "head_gbp"], + "required": [ + "event", + "measure_key", + "year", + "level", + "key", + "flags", + "base_gbp", + "head_gbp" + ], "properties": { - "event": {"type": "string", "minLength": 1}, - "measure_key": {"type": "string", "minLength": 1}, - "year": {"type": "integer"}, - "level": {"enum": ["source_row", "measure_year", "head"]}, - "key": {"type": "string", "minLength": 1}, + "event": { + "type": "string", + "minLength": 1 + }, + "measure_key": { + "type": "string", + "minLength": 1 + }, + "year": { + "type": "integer" + }, + "level": { + "enum": [ + "source_row", + "measure_year", + "head" + ] + }, + "key": { + "type": "string", + "minLength": 1 + }, "flags": { "type": "array", "minItems": 1, "uniqueItems": true, - "items": {"enum": ["numerical", "structural", "coverage", "mapping", "support", "inertness", "construction"]} + "items": { + "enum": [ + "numerical", + "structural", + "coverage", + "mapping", + "support", + "inertness", + "construction" + ] + } }, - "base_gbp": {"type": ["number", "null"]}, - "head_gbp": {"type": ["number", "null"]} + "base_gbp": { + "type": [ + "number", + "null" + ] + }, + "head_gbp": { + "type": [ + "number", + "null" + ] + } } } } @@ -153,74 +372,220 @@ "results": { "type": "object", "additionalProperties": false, - "required": ["repository", "ref", "path"], + "required": [ + "repository", + "ref", + "path" + ], "properties": { - "repository": {"const": "PolicyEngine/policyengine-scorecard"}, - "ref": {"type": "string", "minLength": 1}, - "commit": {"$ref": "#/$defs/sha1"}, - "path": {"type": "string", "minLength": 1} + "repository": { + "const": "PolicyEngine/policyengine-scorecard" + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "commit": { + "$ref": "#/$defs/sha1" + }, + "path": { + "type": "string", + "minLength": 1 + }, + "manifest_sha256": { + "description": "SHA-256 of NAMESPACE_MANIFEST.json, which lists the SHA-256 of every file in the sealed result namespace except itself and the receipt.", + "$ref": "#/$defs/sha256" + } } }, - "receipt_sha256": {"$ref": "#/$defs/sha256"} + "receipt_sha256": { + "$ref": "#/$defs/sha256" + } }, "allOf": [ { - "if": {"properties": {"status": {"const": "rejected"}}}, - "then": {"properties": {"rejection": {"type": "object"}, "corpus": {"type": "null"}, "diff": {"type": "null"}}} + "if": { + "properties": { + "status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "object" + }, + "corpus": { + "type": "null" + }, + "diff": { + "type": "null" + } + } + } }, { - "if": {"properties": {"status": {"enum": ["complete", "failed"]}}}, - "then": {"properties": {"rejection": {"type": "null"}, "variant_id": {"type": "string"}, "execution": {"type": "object"}}} + "if": { + "properties": { + "status": { + "enum": [ + "complete", + "failed" + ] + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "null" + }, + "variant_id": { + "type": "string" + }, + "execution": { + "type": "object" + } + } + } }, { - "if": {"properties": {"status": {"const": "complete"}}}, - "then": {"properties": {"integrity": {"properties": {"status": {"enum": ["complete", "incomplete"]}}}, "corpus": {"type": "object"}}} + "if": { + "properties": { + "status": { + "const": "complete" + } + } + }, + "then": { + "properties": { + "integrity": { + "properties": { + "status": { + "enum": [ + "complete", + "incomplete" + ] + } + } + }, + "corpus": { + "type": "object" + } + } + } } ], "$defs": { - "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, - "sha1": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, "installed_package": { "type": "object", "additionalProperties": false, - "required": ["version", "record_sha256"], + "required": [ + "version", + "record_sha256" + ], "properties": { - "version": {"type": "string", "minLength": 1}, - "record_sha256": {"$ref": "#/$defs/sha256"}, - "wheel_sha256": {"$ref": "#/$defs/sha256"} + "version": { + "type": "string", + "minLength": 1 + }, + "record_sha256": { + "$ref": "#/$defs/sha256" + }, + "wheel_sha256": { + "$ref": "#/$defs/sha256" + } } }, "event_result": { "type": "object", "additionalProperties": false, - "required": ["registry_sha256", "supported_years", "unsupported_years", "run_manifest_sha256", "staging_manifest_sha256", "comparison_provenance_sha256", "artifacts", "full_supported_grid_complete", "inert_measure_years"], + "required": [ + "registry_sha256", + "supported_years", + "unsupported_years", + "run_manifest_sha256", + "staging_manifest_sha256", + "comparison_provenance_sha256", + "artifacts", + "full_supported_grid_complete", + "inert_measure_years" + ], "properties": { - "registry_sha256": {"$ref": "#/$defs/sha256"}, - "supported_years": {"type": "array", "items": {"type": "integer"}, "uniqueItems": true}, + "registry_sha256": { + "$ref": "#/$defs/sha256" + }, + "supported_years": { + "type": "array", + "items": { + "type": "integer" + }, + "uniqueItems": true + }, "unsupported_years": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["year", "reason"], + "required": [ + "year", + "reason" + ], "properties": { - "year": {"type": "integer"}, - "reason": {"enum": ["before_data_year", "after_extension_end"]} + "year": { + "type": "integer" + }, + "reason": { + "enum": [ + "before_data_year", + "after_extension_end" + ] + } } } }, - "run_manifest_sha256": {"$ref": "#/$defs/sha256"}, - "staging_manifest_sha256": {"$ref": "#/$defs/sha256"}, - "comparison_provenance_sha256": {"$ref": "#/$defs/sha256"}, - "artifacts": {"type": "integer", "minimum": 0}, - "full_supported_grid_complete": {"type": "boolean"}, + "run_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "staging_manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "comparison_provenance_sha256": { + "$ref": "#/$defs/sha256" + }, + "artifacts": { + "type": "integer", + "minimum": 0 + }, + "full_supported_grid_complete": { + "type": "boolean" + }, "inert_measure_years": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["measure_key", "year"], - "properties": {"measure_key": {"type": "string"}, "year": {"type": "integer"}} + "required": [ + "measure_key", + "year" + ], + "properties": { + "measure_key": { + "type": "string" + }, + "year": { + "type": "integer" + } + } } } } From 91f46c15b22f9a07e323163ed15693ed4ac09b98 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:03:59 -0400 Subject: [PATCH 5/9] Bind the numerical environment into each replay variant's identity A rebuilt environment for the same three wheels resolved other pydantic, pydantic-core and filelock versions and left out pypdf, and a raw RECORD digest also bound the installer (uv_cache.json, console-script shebangs that name the venv). Now: - identity.requirements_sha256 binds the exact freeze; verify_installed checks every pinned distribution at its version (extras allowed); - record_sha256 excludes installer bookkeeping, bytecode and generated scripts outside site-packages, so pip and uv installs of one wheel agree; - a freeze with local-wheel or commit installs is normalised to exact pins; - a variant id takes an identity-digest suffix only when another identity already holds its version slug; - compute_uk_event --variants-file selects from another variants document (a qualification run adds its candidate there). Schemas: requested_by.workflow_run_id is optional (a catch-up run started outside Actions has none); a failed receipt may have no execution block (a step failed before the candidate's identity was known). Both relax v1. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e9838914f602a6298fc1c79c479e0975771c9f25) --- data/uk/replay_variants.json | 18 ++-- docs/uk_replay/QUALIFICATION_INTERFACE.md | 6 +- docs/uk_replay/RELEASE_REPLAY.md | 9 +- .../schemas/qualification_request.v1.json | 2 +- docs/uk_replay/schemas/replay_receipt.v1.json | 28 ++++++- pipeline/compute_uk_event.py | 8 +- pipeline/uk_replay_variants.py | 83 ++++++++++++++++++- tests/test_uk_replay_variants.py | 28 +++++++ 8 files changed, 159 insertions(+), 23 deletions(-) diff --git a/data/uk/replay_variants.json b/data/uk/replay_variants.json index ab0a90a..60743ba 100644 --- a/data/uk/replay_variants.json +++ b/data/uk/replay_variants.json @@ -53,11 +53,11 @@ "packaged_bundle_manifest_sha256": "255dbb893f455d078a6bcece87399f0a22478f56bc4c6e526b758c4fd5b4db24", "packages": { "policyengine": { - "record_sha256": "5d5de97a40dbc525bc41a6347fc793ba772bd78f9e0630f2f1d382bc92575e68", + "record_sha256": "8d5eec548a5ce206a8fe59ef018542c701e26c0f927351f92339a7a57591b990", "version": "5.0.2" }, "policyengine-core": { - "record_sha256": "dd6570fe988159d89e45df66ccc43355f70abddb8a3a58d246ef38ce711e5687", + "record_sha256": "103128a1c2a3397268b65e33b83edddfa8e406a4eb64fb6f4ff2310db421c770", "version": "3.27.1" }, "policyengine-uk": { @@ -83,9 +83,10 @@ "model_package": "policyengine-uk", "model_version": "2.89.2", "policyengine_version": "5.0.2" - } + }, + "requirements_sha256": "450e64b173b66333027ac36e056e19928d23df1cbe2690e462bab09b31ddff9d" }, - "identity_sha256": "5c642304a4fff8f30074899b2bb530197920703add1b92304a4c3c3a48c5d176", + "identity_sha256": "3f1a33f31ac70a8f0beac2373b6408c8be3366128ec0d0e3c1b716de3880003c", "layout": "legacy_events", "note": "The bundle the track 1 replays were computed against. Its 1.3GB populace dataset needs Modal's 32GiB containers for full events.", "requirements": { @@ -145,12 +146,12 @@ "packaged_bundle_manifest_sha256": "8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54", "packages": { "policyengine": { - "record_sha256": "072606d8f554123484caff5a1e0f12c3203b96d03f5990ebc4d80d6491441fd0", + "record_sha256": "f5990ac205b959621c56585c16ed963a5d1b07b1403adaa7242e0131fbf61f40", "version": "6.2.6", "wheel_sha256": "9ac20a2599cc1f4b50c128c2245042b105c40592e07fc25d39c99a529c9f410b" }, "policyengine-core": { - "record_sha256": "7a4e8a82c3467681971f5160428296b06e65357b8a70801f2cd631ea0943394a", + "record_sha256": "53a92f2e67982a16faeba61b8b3d145d3ab9a50d298e92c512d61166ffe5d5d4", "version": "3.33.0", "wheel_sha256": "2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789" }, @@ -178,9 +179,10 @@ "model_package": "policyengine-uk", "model_version": "2.125.1", "policyengine_version": "6.2.6" - } + }, + "requirements_sha256": "5c333faef091e6780b2c178ac7584249ac7db0159841921727b913505577bcdb" }, - "identity_sha256": "f241374dea3ca00aeba8849437481da34e6c6c9ddb0bb9b75d3e72bbbe22fd69", + "identity_sha256": "3f62a208a9bf522cccb391fe65c1411124abaf4488ab760c522b0568f5b417aa", "layout": "replay_namespace", "note": "The newest certified pair on 11 October 2026; data built with policyengine-uk 2.122.2 and certified for 2.125.1.", "requirements": { diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index d616748..15b712a 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -102,8 +102,10 @@ Two flows use the same receiver: * `request_id` is not a field: it is the SHA-256 of the request's JCS bytes. Both sides test against the golden fixture. -* `requested_by` accepts additional provenance fields (for example - `run_attempt`); every other object is closed. +* `requested_by` names the requesting repository and commit; + `workflow_run_id` is optional (a scorecard catch-up run started outside + Actions has none), and additional provenance fields such as `run_attempt` + are accepted. Every other object is closed. * The wrapper is identified by `version` plus either a PyPI wheel (`{"pypi": {"wheel_sha256": …}}`) or, for an unreleased candidate whose regenerated manifest is not on PyPI yet, a public git commit. Scorecard diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 769cdcb..3a4cc0c 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -81,14 +81,15 @@ A variant record: | Field | Meaning | |---|---| -| `variant_id` | `pe_uk_`; must equal the slug recomputed from `identity`. | +| `variant_id` | `pe_uk_`, recomputed from `identity`; when another identity already holds that slug (same versions, other wheels or numerical environment) the id adds `--`. | | `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | -| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`) and bytecode. Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | | `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | | `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | | `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`, `uri`. | | `identity.materialization` | `hf_cache_mirror` (policyengine 5.x: the existing HF-cache plus `data/uk/policyengine-local` mirror) or `policyengine_data_dir` (6.x: a verified copy at `/data/`). | | `identity.python` | Interpreter minor version, e.g. `3.12`. | +| `identity.requirements_sha256` | The SHA-256 of the requirements freeze below: the numerical environment (numpy, pandas, h5py and the rest) is part of the identity, because it can move numbers. | | `requirements` | `{path, sha256}`: the exact freeze installed for this variant. Legacy: `docs/uk_replay/requirements.txt`. Others: `docs/uk_replay/requirements/.txt`. | | `data_years` | Calendar years the population supports: from the dataset's first `time_period` through the engine's extension end, as observed in `simulation.dataset.years`. | | `data_years_evidence` | The observed `time_period`, `dataset.years` and the engine file/line that sets the extension end. | @@ -115,7 +116,9 @@ failure: and `compatible_model_packages` agree with the record. 2. **Installed environment.** Installed versions of the three packages equal the record; every installed file matches its `RECORD` hash and the - normalised `RECORD` hashes to `record_sha256`; the + normalised `RECORD` hashes to `record_sha256`; every distribution pinned in + the requirements freeze is installed at its pinned version (extra + distributions, such as the receipt tools, are allowed); the installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; the interpreter minor version matches. 3. **Certification path.** Before importing the engine the runner removes diff --git a/docs/uk_replay/schemas/qualification_request.v1.json b/docs/uk_replay/schemas/qualification_request.v1.json index 735b66c..16921af 100644 --- a/docs/uk_replay/schemas/qualification_request.v1.json +++ b/docs/uk_replay/schemas/qualification_request.v1.json @@ -105,7 +105,7 @@ "requested_by": { "type": "object", "additionalProperties": true, - "required": ["repository", "workflow_run_id", "commit"], + "required": ["repository", "commit"], "properties": { "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "workflow_run_id": {"type": "integer", "minimum": 1}, diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json index c5d8d9a..f127f28 100644 --- a/docs/uk_replay/schemas/replay_receipt.v1.json +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -429,10 +429,7 @@ "if": { "properties": { "status": { - "enum": [ - "complete", - "failed" - ] + "const": "complete" } } }, @@ -475,6 +472,29 @@ } } } + }, + { + "if": { + "properties": { + "status": { + "const": "failed" + } + } + }, + "then": { + "properties": { + "rejection": { + "type": "null" + }, + "integrity": { + "properties": { + "status": { + "const": "failed" + } + } + } + } + } } ], "$defs": { diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index aab3fca..e8e4b03 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -599,6 +599,12 @@ def main(argv: list[str] | None = None) -> int: "--variant", help="certified replay variant (data/uk/replay_variants.json); default: the file's default", ) + ap.add_argument( + "--variants-file", + type=Path, + default=variants.VARIANTS_PATH, + help="variant records to select from (a qualification run adds its candidate)", + ) ap.add_argument( "--dataset-file", type=Path, @@ -619,7 +625,7 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) - variant = variants.get_variant(args.variant) + variant = variants.get_variant(args.variant, path=args.variants_file.resolve()) legacy = variants.is_legacy(variant) variant_fields = variant_identity_fields(variant) default_registry, default_output = event_paths(args.event) diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py index a2f1ee4..2513961 100644 --- a/pipeline/uk_replay_variants.py +++ b/pipeline/uk_replay_variants.py @@ -50,8 +50,16 @@ VERSION = re.compile(r"^[0-9]+(?:\.[0-9]+)*[a-z0-9.+-]*$") PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9_.+!-]+)") # Installer bookkeeping differs between pip and uv for the same wheel; only -# the distribution's own files identify what was installed. -INSTALLER_METADATA = ("INSTALLER", "REQUESTED", "RECORD", "direct_url.json") +# the distribution's own files identify what was installed. Paths outside +# site-packages ("../../../bin/...") are console-script wrappers the +# installer generates, with the environment's interpreter in their shebang. +INSTALLER_METADATA = ( + "INSTALLER", + "REQUESTED", + "RECORD", + "direct_url.json", + "uv_cache.json", +) class VariantError(ValueError): @@ -91,6 +99,44 @@ def variant_slug(identity: dict) -> str: ) +def variant_id_for(identity: dict, taken: dict[str, str] | None = None) -> str: + """The version slug, suffixed with the identity digest when another + identity already holds that slug (same versions, other wheels or + numerical environment).""" + slug = variant_slug(identity) + digest = identity_sha256(identity) + holder = (taken or {}).get(slug) + return slug if holder in (None, digest) else f"{slug}--{digest[:10]}" + + +def normalized_freeze(text: str, versions: dict[str, str]) -> str: + """A freeze as exact pins. Engine packages installed from a verified local + wheel or a commit appear as ``name @ ``; they become ``name==version`` + (the wheel and installed-file digests bind them) with the source kept as a + comment. Any other non-pinned line refuses.""" + lines = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + name, at, source = stripped.partition(" @ ") + key = re.sub(r"[-_.]+", "-", name).lower() + if at and key in versions: + lines.append( + f"# {key} installed from {source.split('/')[-1] if source.startswith('file:') else source}" + ) + lines.append(f"{key}=={versions[key]}") + elif at: + raise VariantError(f"freeze line is not an exact pin: {stripped}") + else: + lines.append(stripped) + pins = requirements_pins("\n".join(lines)) + missing = sorted(set(versions) - set(pins)) + if missing: + raise VariantError(f"freeze lacks {missing}") + return "\n".join(lines) + "\n" + + def hf_uri(dataset: dict) -> str: return f"hf://{dataset['repo_id']}/{dataset['path']}@{dataset['revision']}" @@ -189,7 +235,11 @@ def need(condition: bool, message: str) -> None: uk = packages["policyengine-uk"]["version"] wrapper = packages["policyengine"]["version"] need( - record.get("variant_id") == variant_slug(identity), + record.get("variant_id") + in ( + variant_slug(identity), + f"{variant_slug(identity)}--{identity_sha256(identity)[:10]}", + ), "variant_id differs from its identity", ) need( @@ -280,6 +330,10 @@ def need(condition: bool, message: str) -> None: f"result_root must be {expected_root}", ) requirements = record.get("requirements") or {} + need( + identity.get("requirements_sha256") == requirements.get("sha256"), + "the identity binds another requirements file", + ) requirements_path = root / str(requirements.get("path", "")) if not requirements.get("path") or not requirements_path.is_file(): problems.append("requirements file is missing") @@ -342,6 +396,7 @@ def _record_rows(distribution) -> list[tuple[str, str, str]]: and name in INSTALLER_METADATA or "__pycache__/" in path or path.endswith(".pyc") + or path.startswith("../") ): continue rows.append((path, digest, size)) @@ -394,9 +449,25 @@ def installed_identity(*, verify_files: bool = True) -> dict: "packages": packages, "packaged_bundle_manifest_sha256": sha256_file(manifest), "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "distributions": { + re.sub(r"[-_.]+", "-", d.metadata["Name"]).lower(): d.version + for d in importlib.metadata.distributions() + }, } +def environment_mismatches(pins: dict[str, str], installed: dict) -> list[str]: + """Every pinned distribution installed at its pinned version. Extra + distributions (the receipt tools) are allowed; a missing or different + numerical dependency is not.""" + have = installed.get("distributions", {}) + return [ + f"{name}: pinned {version}, installed {have.get(name)}" + for name, version in sorted(pins.items()) + if have.get(name) != version + ] + + def identity_mismatches(record: dict, installed: dict) -> list[str]: identity = record["identity"] out = [] @@ -424,9 +495,13 @@ def identity_mismatches(record: dict, installed: dict) -> list[str]: return out -def verify_installed(record: dict, *, installed: dict | None = None) -> dict: +def verify_installed( + record: dict, *, installed: dict | None = None, root: Path = ROOT +) -> dict: installed = installed if installed is not None else installed_identity() mismatches = identity_mismatches(record, installed) + pins = requirements_pins((root / record["requirements"]["path"]).read_text()) + mismatches += environment_mismatches(pins, installed) if mismatches: raise VariantError( f"{record['variant_id']}: installed environment is not the certified one: " diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py index 7637a64..0e0ba24 100644 --- a/tests/test_uk_replay_variants.py +++ b/tests/test_uk_replay_variants.py @@ -178,9 +178,37 @@ def installed_for(r): }, "packaged_bundle_manifest_sha256": identity["packaged_bundle_manifest_sha256"], "python": identity["python"], + "distributions": rv.requirements_pins( + (rv.ROOT / r["requirements"]["path"]).read_text() + ), } +def test_every_pinned_distribution_is_part_of_the_installed_check(): + r = record() + exact = installed_for(r) + for name in ("numpy", "pandas", "h5py"): + changed = copy.deepcopy(exact) + changed["distributions"][name] = "0.0.0" + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=changed) + missing = copy.deepcopy(exact) + del missing["distributions"][name] + with pytest.raises(rv.VariantError, match=name): + rv.verify_installed(r, installed=missing) + extra = copy.deepcopy(exact) + extra["distributions"]["rfc8785"] = "0.1.4" + rv.verify_installed(r, installed=extra) + + +def test_the_identity_binds_the_requirements_freeze(): + r = record() + r["identity"]["requirements_sha256"] = "5" * 64 + reseal(r) + with pytest.raises(rv.VariantError, match="another requirements file"): + rv.validate_record(r) + + def test_installed_environment_must_match_every_field(): r = record() exact = installed_for(r) From 5bfd39705f0cf49563b5f757c4e7d874102e2e2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:10:44 -0400 Subject: [PATCH 6/9] Design doc: describe result namespaces as built The variants file is the index (no INDEX.json); a sealed namespace holds NAMESPACE_MANIFEST.json and REQUEST.json; reruns of a sealed variant use an explicit output directory rather than a rerun- suffix. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/RELEASE_REPLAY.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index 3a4cc0c..ea449ba 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -157,17 +157,21 @@ FY2023–24 counterparts; that loss is shown, never hidden. * Legacy results stay where they are, in `results/uk/events//`. * Every other variant writes to `results/uk/replays//`: - `VARIANT.json` (the validated record plus the runtime receipt), + `/` (the same files as a legacy event directory), `SUMMARY.md`, `SUPPORT.json`, `CONSTRUCTION_PREMISES.json`, `MODEL_DIAGNOSTICS.json`, - `/` (the same files as a legacy event directory), `SUMMARY.md`, and - finally `REPLAY_RECEIPT.json`. -* `results/uk/replays/INDEX.json` maps each variant to its root (legacy: - `results/uk/events`), status and receipt. -* A namespace with `REPLAY_RECEIPT.json` is **sealed**: the receipt binds the - SHA-256 of every file under it, the runner refuses to write there, and CI - fails if any sealed byte changes. A rerun of the same variant goes to a new - namespace suffix (`/rerun-`) and its determinism against the - sealed run is itself a check. + `REQUEST.json` (the qualification request the run answers), and finally + `NAMESPACE_MANIFEST.json` and `REPLAY_RECEIPT.json`. +* `data/uk/replay_variants.json` is the index: each record's `result_root` + names its namespace (legacy: `results/uk/events`) and its `status`. A + candidate's record and results live on the `replay-state` branch under + `requests//results/` until it is certified. +* A namespace with `REPLAY_RECEIPT.json` is **sealed**: + `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under it (and of + its release diff), the receipt binds that manifest's digest, the runner + refuses to write there, and CI fails if any sealed byte changes. A rerun of + a sealed variant writes to an explicit separate output directory, and its + bytes against the sealed run are a determinism check (CI replays one real + measure-year this way). * Release differences live in `results/uk/replays/diffs/__/`. ## Version-scoped diagnoses From 3001126d430d53ee8a7bb458a22de8d383026535 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:50:39 -0400 Subject: [PATCH 7/9] Replay variants: verify what staging trusts and what the engine loads Independent review of the variant runner found trust gaps; each is now closed and tested. - Staging took a manifest's variant_id and unsupported years on trust: a self-consistent manifest could stage a 2023 artifact from a population with no 2023 inputs, and dropping variant_id switched the artifact check off. Staging now loads the record from the variants document (--variants-file), requires its identity digest, data years and the supported/unsupported partition that record gives, and requires artifact variant fields to match the manifest (none for legacy). - --resume gathered receipts for every registry year; it now reads only supported years and refuses a receipt naming an unsupported one. - Staging, comparison and the summary now refuse a sealed namespace, as compute already did. - record_sha256 binds RECORD-listed files, but policyengine-core loads every file in a parameter directory and every module in a variable directory (parameter_node.py, tax_benefit_system.py). The installed check now also refuses files in the three packages' directories that no installed RECORD lists; engine bytecode is compiled from source into an empty sys.pycache_prefix with writing off (workers inherit it), and each imported package must be its verified distribution's own files. - Removing tokens forced the packaged certification only for a private data repository: policyengine.py 6.x fetches the release manifest with requests, which ignores HF_HUB_OFFLINE. The isolated environment now also points the HTTP(S) proxy variables at a closed local port and drops NO_PROXY, so the fetch fails for a public repository too. - Only certified variants run; a qualification run passes --allow-uncertified; the default must be certified. - The legacy variant's installed environment is now checked before its preflight, and --dataset-file is refused there instead of ignored. - A variant simulation whose dataset does not state its years fails instead of skipping the year check (legacy unchanged). - Variant rows name their own population instead of "a single certified 2023 population"; legacy text is unchanged. - Record fields of the wrong type raise VariantError; a symlinked runtime data directory is refused. Verified: the five legacy events restage and recompare byte-identically; both pinned venvs (.venv-candidate 6.2.6, .venv-replay 5.0.2) reproduce their committed records with no unlisted files; a real focused 6.2.6 replay (AB2024 CGT, 2025) through every new check is byte-identical to the committed artifact (sha256 8316a941...). Full suite: 1521 passed. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/RELEASE_REPLAY.md | 59 ++-- pipeline/compare_uk_event.py | 3 + pipeline/compute_uk_event.py | 79 +++++- pipeline/compute_uk_obr_costings.py | 11 +- pipeline/stage_uk_event.py | 94 +++++- pipeline/uk_replay_variants.py | 159 ++++++++++- tests/test_uk_event_compute.py | 19 +- tests/test_uk_replay_variants.py | 425 +++++++++++++++++++++++++++- 8 files changed, 789 insertions(+), 60 deletions(-) diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index ea449ba..c0835e2 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -83,10 +83,10 @@ A variant record: |---|---| | `variant_id` | `pe_uk_`, recomputed from `identity`; when another identity already holds that slug (same versions, other wheels or numerical environment) the id adds `--`. | | `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | -| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256` when the wheel itself was hashed. | +| `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256`: the wheel's digest as published on PyPI, recorded at certification; the receiver verifies each downloaded wheel against it, and the runner, which never sees a wheel, checks only its format. | | `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | | `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | -| `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`, `uri`. | +| `identity.dataset` | `repo_id`, `repo_type`, `revision`, `path`, `sha256`, `size_bytes`; `release_bundle.default_dataset_uri` must name the same file. | | `identity.materialization` | `hf_cache_mirror` (policyengine 5.x: the existing HF-cache plus `data/uk/policyengine-local` mirror) or `policyengine_data_dir` (6.x: a verified copy at `/data/`). | | `identity.python` | Interpreter minor version, e.g. `3.12`. | | `identity.requirements_sha256` | The SHA-256 of the requirements freeze below: the numerical environment (numpy, pandas, h5py and the rest) is part of the identity, because it can move numbers. | @@ -94,7 +94,7 @@ A variant record: | `data_years` | Calendar years the population supports: from the dataset's first `time_period` through the engine's extension end, as observed in `simulation.dataset.years`. | | `data_years_evidence` | The observed `time_period`, `dataset.years` and the engine file/line that sets the extension end. | | `certification` | Where the pairing is certified: `source` (e.g. `policyengine.py 6.2.6 packaged manifest`), `compatibility_basis`, `certified_for_model_version`, `data_build_id`. | -| `status` | `certified` (a released policyengine.py bundle), `candidate` (a W4 qualification request; never a default), or `retired`. | +| `status` | `certified` (a released policyengine.py bundle), `candidate` (a W4 qualification request), or `retired`. Only a certified variant runs, and the default must be certified; a qualification run, which replays its own candidate record, passes `--allow-uncertified`. | | `qualification_request_sha256` | For candidates, the request the variant was built from. | | `executor` | `local` (GitHub runner or workstation) and/or `modal`. The populace 1.3GB dataset needs Modal's 32GiB; the 128MB enhanced FRS runs on a 16GB runner. | @@ -116,23 +116,41 @@ failure: and `compatible_model_packages` agree with the record. 2. **Installed environment.** Installed versions of the three packages equal the record; every installed file matches its `RECORD` hash and the - normalised `RECORD` hashes to `record_sha256`; every distribution pinned in - the requirements freeze is installed at its pinned version (extra - distributions, such as the receipt tools, are allowed); the + normalised `RECORD` hashes to `record_sha256`; no file sits in the three + packages' directories without a `RECORD` entry in some installed + distribution (policyengine-core loads every file in a parameter directory + and every module in a variable directory, listed or not); every + distribution pinned in the requirements freeze is installed at its pinned + version (extra distributions, such as the receipt tools, are allowed); the installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; - the interpreter minor version matches. -3. **Certification path.** Before importing the engine the runner removes - `HUGGING_FACE_TOKEN`, `HF_TOKEN` and `HUGGINGFACE_HUB_TOKEN` from the - environment and forces Hugging Face offline mode, so the wrapper uses its - packaged certification on every host. Then + the interpreter minor version matches. The legacy variant is checked the + same way. +3. **What is imported.** Before the engine is imported, bytecode caching is + turned off and `sys.pycache_prefix` points at an empty directory, so + Python compiles every engine module from the verified sources and never + reads a `__pycache__` beside them; spawned year workers inherit both + settings. After import, each package's `__file__` must be its verified + distribution's own `__init__.py`, not a same-named tree earlier on + `sys.path`; each year worker checks this too. +4. **Certification path.** Before importing the engine the runner removes + `HUGGING_FACE_TOKEN`, `HF_TOKEN` and `HUGGINGFACE_HUB_TOKEN` and the + `NO_PROXY` exceptions from the environment, forces Hugging Face offline + mode, and points the HTTP(S) proxy variables at a closed local port. + policyengine.py 6.x fetches the data release manifest with `requests` + (`provenance/manifest.py`, `get_data_release_manifest`), which ignores + `HF_HUB_OFFLINE` but reads those proxy variables, so the fetch fails at + once for a private or a public data repository, and + `certify_data_release_compatibility` falls back to the wrapper's packaged + certification. Then `dict(pe.uk.uk_latest.release_bundle) == identity.release_bundle`, exactly. -4. **Dataset bytes.** The materialised file hashes to `dataset.sha256` and +5. **Dataset bytes.** The materialised file hashes to `dataset.sha256` and has `size_bytes`, before construction and after the last aggregate (existing `run_managed_simulation` checks), and the simulation's reported runtime dataset source is that file. -5. **Year support.** Requested years must lie in `data_years`, and inside - every simulation `year in simulation.dataset.years` (when the attribute - exists) or the run stops. Registry years outside `data_years` are not +6. **Year support.** Requested years must lie in `data_years`, and inside + every simulation `year in simulation.dataset.years` or the run stops; on a + non-legacy variant a dataset that does not state its years stops the run + too (the legacy check skips it, as it always has). Registry years outside `data_years` are not requested by default; they are reported as unsupported (below). A model/data combination is **incompatible** when no certified variant @@ -146,7 +164,13 @@ variant runs `calendar_years ∩ data_years`. The rest are **unsupported on the variant**, with a reason (`before_data_year` or `after_extension_end`). `RUN_MANIFEST.json` records `supported_years`, `unsupported_years` and `full_supported_grid_complete`; `full_event_complete` keeps its meaning -(every registry year). Staging marks a source row whose year is unsupported +(every registry year). Staging does not take those fields on trust: it loads +the manifest's `variant_id` from the variants document, requires the +manifest's identity digest, `data_years`, `supported_years` and +`unsupported_years` to be the ones that record gives, and requires every +artifact's variant fields to match (a legacy manifest and its artifacts carry +none). `--resume` reads receipts only for supported years and refuses any +receipt that names an unsupported one. Staging marks a source row whose year is unsupported as `status: "unsupported_on_variant"` with no numerical counterpart, kept in the row universe and counted separately from `not_computed`. On `pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0`, CY2023 is unsupported, so @@ -168,7 +192,8 @@ FY2023–24 counterparts; that loss is shown, never hidden. * A namespace with `REPLAY_RECEIPT.json` is **sealed**: `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under it (and of its release diff), the receipt binds that manifest's digest, the runner - refuses to write there, and CI fails if any sealed byte changes. A rerun of + refuses to write there (compute, staging, comparison and the summary all + check), and CI fails if any sealed byte changes. A rerun of a sealed variant writes to an explicit separate output directory, and its bytes against the sealed run are a determinism check (CI replays one real measure-year this way). diff --git a/pipeline/compare_uk_event.py b/pipeline/compare_uk_event.py index c5bf710..e746192 100644 --- a/pipeline/compare_uk_event.py +++ b/pipeline/compare_uk_event.py @@ -25,6 +25,7 @@ sys.path.insert(0, str(ROOT)) from pipeline.build_uk_event_registry import _sum as source_gbp_sum +from pipeline import uk_replay_variants as variants from pipeline.compare_uk_obr_costings import atomic_write_bytes, ratio_and_bin AXES_PATH = ROOT / "data" / "uk" / "obr_divergence_axes.json" @@ -1053,6 +1054,7 @@ def write_comparison( ) if registry.get("event_slug") != event: raise EventComparisonError("registry event differs from --event") + variants.assert_not_sealed(output_dir) output_dir.mkdir(parents=True, exist_ok=True) staging_manifest = _json(staging_manifest_path) replay_grid = { @@ -1387,6 +1389,7 @@ def write_summary(events_root: Path, *, artifact_root: Path = ROOT) -> None: (artifact_root / "data" / "uk" / "events").glob("*_measures.json") ) } + variants.assert_not_sealed(events_root) events_root.mkdir(parents=True, exist_ok=True) atomic_write_bytes( events_root / "SUMMARY.md", diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index e8e4b03..1d69fdd 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -282,10 +282,15 @@ def build_artifact( return artifact -def preflight() -> dict: - """The committed pin and the managed release must agree before any run.""" +def preflight(variant: dict | None = None) -> dict: + """The committed pin and the managed release must agree before any run. + + Given the legacy variant record, the engine is also compiled from its + verified sources and imported from its own distributions.""" fiscal.configure_offline() worlds.configure_offline() + if variant is not None: + variants.isolate_bytecode(variants.RUNTIME_ROOT / variant["variant_id"]) committed = worlds.preflight() managed = fiscal.preflight_certified_dataset() if ( @@ -293,9 +298,21 @@ def preflight() -> dict: or fiscal.data_bundle_id(managed["release_bundle"]) != committed["revision"] ): raise RuntimeError("managed release differs from data/uk/certified_bundle.json") + if variant is not None: + import_engine() + return {**managed, "check_import_origins": True} return managed +def import_engine() -> None: + """Import all three engine packages and check where each came from.""" + import policyengine # noqa: F401 + import policyengine_core # noqa: F401 + import policyengine_uk # noqa: F401 + + variants.verify_import_origins() + + def variant_identity_fields(variant: dict) -> dict: """Artifact and receipt identity of a non-legacy variant; legacy: none.""" if variants.is_legacy(variant): @@ -327,8 +344,10 @@ def variant_preflight(variant: dict, *, dataset_source: Path | None = None) -> d runtime_root = (variants.RUNTIME_ROOT / variant["variant_id"]).resolve() dataset = variants.materialize_dataset(variant, runtime_root, source=dataset_source) os.chdir(runtime_root) + variants.isolate_bytecode(runtime_root) import policyengine as pe + import_engine() release_bundle = dict(pe.uk.uk_latest.release_bundle) variants.verify_release_bundle(variant, release_bundle) return { @@ -336,6 +355,10 @@ def variant_preflight(variant: dict, *, dataset_source: Path | None = None) -> d "sha256": variant["identity"]["dataset"]["sha256"], "runtime_dataset_source": str(dataset), "installed": installed, + # A year outside the population's data years must fail, not + # calculate defaults (compute_uk_obr_costings.run_managed_simulation). + "require_dataset_years": True, + "check_import_origins": True, } @@ -354,6 +377,7 @@ def run_sim(year: int, variables: list[str], reform: dict | None, pre: dict) -> runtime_dataset_source=Path(pre["runtime_dataset_source"]), expected_dataset_sha256=pre["sha256"], include_engine_provenance=True, + require_dataset_years=pre.get("require_dataset_years", False), **scenario_kwargs, ) fiscal.assert_managed_bundle(result["policyengine_bundle"], pre["release_bundle"]) @@ -370,6 +394,9 @@ def run_year_job(job: dict) -> dict: flush=True, ) baseline = run_sim(year, job["variables"], None, pre) + # This spawned worker imported the engine itself: check its origin too. + if pre.get("check_import_origins"): + variants.verify_import_origins() timings = [ { "world": "certified_baseline", @@ -610,6 +637,11 @@ def main(argv: list[str] | None = None) -> int: type=Path, help="a local file holding the variant's certified dataset bytes (verified by SHA-256)", ) + ap.add_argument( + "--allow-uncertified", + action="store_true", + help="replay a candidate variant (qualification runs only)", + ) ap.add_argument("--registry", type=Path) ap.add_argument("--output-dir", type=Path) ap.add_argument("--measures", nargs="*") @@ -625,8 +657,17 @@ def main(argv: list[str] | None = None) -> int: help="spawn at most two year workers; each holds one live managed simulation", ) args = ap.parse_args(argv) - variant = variants.get_variant(args.variant, path=args.variants_file.resolve()) + variant = variants.get_variant( + args.variant, + path=args.variants_file.resolve(), + allow_uncertified=args.allow_uncertified, + ) legacy = variants.is_legacy(variant) + if legacy and args.dataset_file is not None: + raise ValueError( + "--dataset-file applies to a variant that materialises its dataset; " + "the legacy variant reads its pinned Hugging Face cache" + ) variant_fields = variant_identity_fields(variant) default_registry, default_output = event_paths(args.event) if not legacy: @@ -671,7 +712,7 @@ def main(argv: list[str] | None = None) -> int: if coverage: raise ValueError("\n".join(coverage)) if args.dry_run or args.guards: - if not legacy and args.guards: + if args.guards: check_variant_environment(variant) try: resolve = worlds.engine_resolver() @@ -714,8 +755,10 @@ def main(argv: list[str] | None = None) -> int: variants.assert_not_sealed(output_dir) verify_registry_identity(registry_path, registry_sha256) print(f"[pid {os.getpid()}] starting offline bundle preflight", flush=True) + if legacy: + check_variant_environment(variant) pre = ( - preflight() + preflight(variant) if legacy else variant_preflight(variant, dataset_source=args.dataset_file) ) @@ -732,10 +775,15 @@ def main(argv: list[str] | None = None) -> int: print(f"[pid {os.getpid()}] processed-world reversal guard complete", flush=True) verify_registry_identity(registry_path, registry_sha256) output_dir.mkdir(parents=True, exist_ok=True) + supported_calendar_years = ( + registry["calendar_years"] + if legacy + else variants.supported_years(variant, registry["calendar_years"])[0] + ) previous_digests = retained_artifact_digests( output_dir, event=args.event, - years=registry["calendar_years"], + years=supported_calendar_years, registry_sha256=registry_sha256, pre=pre, variant_fields=variant_fields, @@ -744,15 +792,9 @@ def main(argv: list[str] | None = None) -> int: full_pairs = { (key, year) for key in runnable for year in registry["calendar_years"] } - supported_pairs = ( - full_pairs - if legacy - else { - (key, year) - for key, year in full_pairs - if year in variants.supported_years(variant, registry["calendar_years"])[0] - } - ) + supported_pairs = { + (key, year) for key, year in full_pairs if year in supported_calendar_years + } expected_paths = { str((output_dir / f"{key}_{year}.json").relative_to(ROOT)): (key, year) for key, year in full_pairs @@ -764,6 +806,13 @@ def main(argv: list[str] | None = None) -> int: "retained receipt names a measure/year outside the event grid" ) key, year = expected_paths[relative] + if (key, year) not in supported_pairs: + # No artifact exists for a year the population cannot supply, + # whatever an older receipt says (RELEASE_REPLAY.md). + raise ValueError( + f"retained receipt names {key} {year}, outside the variant's " + "supported years" + ) if not args.resume and (key, year) in requested_pairs: continue path = ROOT / relative diff --git a/pipeline/compute_uk_obr_costings.py b/pipeline/compute_uk_obr_costings.py index eaef9c6..04f2a85 100644 --- a/pipeline/compute_uk_obr_costings.py +++ b/pipeline/compute_uk_obr_costings.py @@ -1025,8 +1025,12 @@ def run_managed_simulation( expected_dataset_sha256: str, include_engine_provenance: bool = False, scenario: Any = None, + require_dataset_years: bool = False, ) -> dict[str, Any]: - """Run, aggregate, delete, and collect exactly one managed sim.""" + """Run, aggregate, delete, and collect exactly one managed sim. + + With ``require_dataset_years`` a simulation whose dataset does not state + its years fails instead of skipping the year check.""" if scenario is not None and reform is not None: raise ValueError("pass either reform or scenario, not both") @@ -1074,6 +1078,11 @@ def run_managed_simulation( # calculates defaults without raising (a 2023 income tax of zero # on a 2024 dataset). Both pinned engines expose dataset.years. dataset_years = getattr(getattr(sim, "dataset", None), "years", None) + if dataset_years is None and require_dataset_years: + raise RuntimeError( + "the simulation dataset does not state its years; " + f"cannot confirm {year} has inputs" + ) if dataset_years is not None and int(year) not in { int(value) for value in dataset_years }: diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index b0b0d62..d290e81 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -17,6 +17,7 @@ from pipeline import compute_uk_event as compute from pipeline import compute_uk_obr_costings as fiscal from pipeline import stage_uk_ab2025 as ab_stage +from pipeline import uk_replay_variants as variants AXES = [ "behavioural_adjustment", @@ -99,15 +100,75 @@ def validate_artifact( raise ValueError("per-measure head effects do not sum to the measure total") +LEGACY_POPULATION_NOTE = ( + "OBR uses the forecast available at announcement; PE uses a single " + "certified 2023 population with later calibration and uprating targets." +) + + +def population_note(record: dict | None) -> str: + """The first row annotation: which population PE used.""" + if record is None: + return LEGACY_POPULATION_NOTE + dataset = record["identity"]["dataset"] + return ( + "OBR uses the forecast available at announcement; PE uses a single " + f"certified population ({dataset['path']}, {dataset['repo_id']} " + f"{dataset['revision']}) with later calibration and uprating targets." + ) + + +def manifest_variant( + manifest: dict | None, registry: dict, *, variants_path: Path +) -> dict | None: + """The variant record a compute manifest claims, checked against the + variants document rather than trusted: its identity digest must match, + and its unsupported years must be the ones the record's data years give. + A manifest naming no variant is the legacy layout and lists none.""" + if manifest is None: + return None + variant_id = manifest.get("variant_id") + if variant_id is None: + if manifest.get("unsupported_years") or "variant_identity_sha256" in manifest: + raise ValueError("a legacy compute manifest lists variant fields") + return None + record = variants.get_variant( + variant_id, path=variants_path, allow_uncertified=True + ) + if variants.is_legacy(record): + raise ValueError("the legacy variant's manifest names no variant_id") + if manifest.get("variant_identity_sha256") != record["identity_sha256"]: + raise ValueError("compute manifest variant identity differs from its record") + supported, unsupported = variants.supported_years( + record, registry["calendar_years"] + ) + if ( + manifest.get("unsupported_years") != unsupported + or manifest.get("supported_years") != supported + or manifest.get("data_years") != record["data_years"] + ): + raise ValueError( + "compute manifest supported years differ from the variant's data years" + ) + return record + + def stage_event( - registry: dict, manifest: dict | None, *, artifact_dir: Path, registry_sha256: str + registry: dict, + manifest: dict | None, + *, + artifact_dir: Path, + registry_sha256: str, + variants_path: Path = variants.VARIANTS_PATH, ) -> tuple[list[dict], dict]: """Engine-free staging; missing measures and unsupported FYs stay visible.""" artifacts = {} paths = {} # A non-legacy variant's manifest names its identity and the registry - # years its population cannot supply (RELEASE_REPLAY.md). - variant_id = (manifest or {}).get("variant_id") + # years its population cannot supply (RELEASE_REPLAY.md); both are + # checked against the variants document. + record = manifest_variant(manifest, registry, variants_path=variants_path) + variant_id = record["variant_id"] if record is not None else None unsupported = {row["year"] for row in (manifest or {}).get("unsupported_years", [])} if manifest is not None: event = registry.get("event_slug", registry.get("event")) @@ -146,11 +207,9 @@ def stage_event( registry_sha256, manifest["certified_dataset_sha256"], ) - if variant_id is not None and ( - artifact.get("variant_id") != variant_id - or artifact.get("variant_identity_sha256") - != manifest["variant_identity_sha256"] - ): + if artifact.get("variant_id") != variant_id or artifact.get( + "variant_identity_sha256" + ) != (manifest.get("variant_identity_sha256") if variant_id else None): raise ValueError( "artifact variant identity differs from the compute manifest" ) @@ -309,7 +368,7 @@ def declared_pairs(field): "source_table": source.get("source_table"), "source_column": source.get("source_column"), "annotations": [ - "OBR uses the forecast available at announcement; PE uses a single certified 2023 population with later calibration and uprating targets.", + population_note(record), "PE calendar-year totals proxy fiscal-year totals; PE is static and its certified policy world differs from the announcement baseline.", ], } @@ -461,6 +520,12 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--registry", type=Path) ap.add_argument("--artifact-dir", type=Path) ap.add_argument("--output", type=Path) + ap.add_argument( + "--variants-file", + type=Path, + default=variants.VARIANTS_PATH, + help="variant records a manifest's variant_id is checked against", + ) args = ap.parse_args(argv) default_registry, default_artifacts = compute.event_paths(args.event) registry_path = args.registry or default_registry @@ -474,11 +539,18 @@ def main(argv: list[str] | None = None) -> int: registry = json.loads(payload) compute.validate_event_identity(registry, args.event) compute.registry_builder.validate_registry(registry) + output = args.output or artifact_dir / "STAGED.jsonl" + # A sealed namespace's staged rows are part of what its receipt binds. + variants.assert_not_sealed(artifact_dir) + variants.assert_not_sealed(output.parent) rows, tally = stage_event( - registry, manifest, artifact_dir=artifact_dir, registry_sha256=registry_sha256 + registry, + manifest, + artifact_dir=artifact_dir, + registry_sha256=registry_sha256, + variants_path=args.variants_file.resolve(), ) compute.verify_registry_identity(registry_path, registry_sha256) - output = args.output or artifact_dir / "STAGED.jsonl" output.parent.mkdir(parents=True, exist_ok=True) staged_payload = b"".join( (json.dumps(row, sort_keys=True, allow_nan=False) + "\n").encode() diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py index 2513961..49bfda3 100644 --- a/pipeline/uk_replay_variants.py +++ b/pipeline/uk_replay_variants.py @@ -42,6 +42,28 @@ "HF_HUB_DISABLE_TELEMETRY": "1", "HF_HUB_DISABLE_IMPLICIT_TOKEN": "1", } +# policyengine.py 6.x fetches the data release manifest with requests +# (provenance/manifest.py, get_data_release_manifest), which ignores +# HF_HUB_OFFLINE. requests reads these proxy variables, so with them the +# fetch fails at once and certify_data_release_compatibility falls back to +# the wrapper's packaged certification, the one each record certifies. +NO_NETWORK = { + name: "http://127.0.0.1:9" + for name in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ) +} +PROXY_EXCEPTIONS = ("NO_PROXY", "no_proxy") +ENGINE_MODULES = { + "policyengine": "policyengine", + "policyengine-uk": "policyengine_uk", + "policyengine-core": "policyengine_core", +} MATERIALIZATIONS = ("hf_cache_mirror", "policyengine_data_dir") LAYOUTS = ("legacy_events", "replay_namespace") STATUSES = ("certified", "candidate", "retired") @@ -173,20 +195,35 @@ def load_document(path: Path = VARIANTS_PATH) -> dict: raise VariantError("replay_variants.json repeats a variant_id") if document.get("default_variant") not in ids: raise VariantError("default_variant is not a listed variant") + default = variants[ids.index(document["default_variant"])] + if default.get("status") != "certified": + raise VariantError("default_variant must be a certified variant") return document def get_variant( - variant_id: str | None = None, *, path: Path = VARIANTS_PATH, root: Path = ROOT + variant_id: str | None = None, + *, + path: Path = VARIANTS_PATH, + root: Path = ROOT, + allow_uncertified: bool = False, ) -> dict: - """The validated record; None selects the file's default variant.""" + """The validated record; None selects the file's default variant. + + Only a certified variant runs, unless a qualification run, which replays + its own candidate record, asks with ``allow_uncertified``.""" document = load_document(path) wanted = variant_id or document["default_variant"] for record in document["variants"]: if record["variant_id"] == wanted: validate_record(record, root=root) + if record["status"] != "certified" and not allow_uncertified: + raise VariantError( + f"{wanted} is {record['status']}, not certified; only a " + "qualification run replays an uncertified variant" + ) return record - raise VariantError(f"no certified replay variant {wanted!r}") + raise VariantError(f"no replay variant {wanted!r}") def is_legacy(record: dict) -> bool: @@ -210,6 +247,14 @@ def need(condition: bool, message: str) -> None: certification = record["certification"] except (KeyError, TypeError) as exc: raise VariantError(f"variant record is missing {exc}") from exc + if not all( + isinstance(part, dict) + for part in (identity, packages, bundle, dataset, certification) + ): + raise VariantError( + "identity, its packages, release_bundle and dataset, and " + "certification must be objects" + ) need( set(packages) == set(PACKAGES), "identity.packages must name exactly the three packages", @@ -453,9 +498,95 @@ def installed_identity(*, verify_files: bool = True) -> dict: re.sub(r"[-_.]+", "-", d.metadata["Name"]).lower(): d.version for d in importlib.metadata.distributions() }, + "unlisted_files": unlisted_package_files() if verify_files else {}, } +def unlisted_package_files(names=PACKAGES) -> dict[str, list[str]]: + """Files in each package's top-level directories that no installed + distribution's RECORD lists. + + A listed file is checked against its hash, but policyengine-core also + loads files nobody listed: every file in a parameter directory + (parameters/parameter_node.py, sorted os.listdir) and every module in a + variable directory (taxbenefitsystems/tax_benefit_system.py, + add_variables_from_directory). Bytecode caches are skipped here because + runs never read them (isolate_bytecode).""" + listed = set() + for distribution in importlib.metadata.distributions(): + for line in (distribution.read_text("RECORD") or "").splitlines(): + relative = line.rsplit(",", 2)[0] + if relative: + listed.add(Path(distribution.locate_file(relative)).resolve()) + found = {} + for name in names: + distribution = importlib.metadata.distribution(name) + tops = { + row[0].split("/", 1)[0] + for row in _record_rows(distribution) + if "/" in row[0] and ".dist-info/" not in row[0] + } + extra = [] + for top in sorted(tops): + base = Path(distribution.locate_file(top)) + for directory, subdirectories, files in os.walk(base): + subdirectories[:] = sorted( + d for d in subdirectories if d != "__pycache__" + ) + for file in sorted(files): + path = Path(directory) / file + if path.resolve() not in listed: + extra.append( + str(path.relative_to(distribution.locate_file(""))) + ) + if extra: + found[name] = extra + return found + + +def isolate_bytecode(runtime_root: Path, environ=os.environ) -> Path: + """Compile the engine from its verified sources, never from a cache. + + Bytecode is read only from sys.pycache_prefix when one is set, so an + empty prefix with writing turned off ignores every __pycache__ beside + the sources (including one whose header matches its source). Spawned + year workers inherit both settings. Call before importing the engine.""" + loaded = sorted(m for m in ENGINE_MODULES.values() if m in sys.modules) + if loaded: + raise VariantError(f"engine modules imported before isolation: {loaded}") + prefix = (runtime_root / "no-bytecode").resolve() + if prefix.is_symlink(): + raise VariantError(f"{prefix} is a symlink; refusing it") + if prefix.exists(): + shutil.rmtree(prefix) + prefix.mkdir(parents=True) + sys.pycache_prefix = str(prefix) + sys.dont_write_bytecode = True + environ["PYTHONPYCACHEPREFIX"] = str(prefix) + environ["PYTHONDONTWRITEBYTECODE"] = "1" + return prefix + + +def verify_import_origins(names=PACKAGES) -> None: + """Each imported engine package is the verified distribution's own + files, not a same-named tree earlier on sys.path.""" + for name in names: + module_name = ENGINE_MODULES[name] + module = sys.modules.get(module_name) + if module is None: + raise VariantError(f"{module_name} is not imported yet") + expected = Path( + importlib.metadata.distribution(name).locate_file( + f"{module_name}/__init__.py" + ) + ).resolve() + if Path(module.__file__).resolve() != expected: + raise VariantError( + f"{module_name} imported from {module.__file__}, not its " + f"verified distribution ({expected})" + ) + + def environment_mismatches(pins: dict[str, str], installed: dict) -> list[str]: """Every pinned distribution installed at its pinned version. Extra distributions (the receipt tools) are allowed; a missing or different @@ -502,6 +633,11 @@ def verify_installed( mismatches = identity_mismatches(record, installed) pins = requirements_pins((root / record["requirements"]["path"]).read_text()) mismatches += environment_mismatches(pins, installed) + mismatches += [ + f"{name}: files outside its RECORD: {', '.join(paths[:5])}" + + (f" and {len(paths) - 5} more" if len(paths) > 5 else "") + for name, paths in sorted(installed.get("unlisted_files", {}).items()) + ] if mismatches: raise VariantError( f"{record['variant_id']}: installed environment is not the certified one: " @@ -516,9 +652,17 @@ def isolate_certification_environment(environ=os.environ) -> list[str]: With a Hugging Face token, policyengine.py derives the certification from the remote release manifest instead, which changes release_bundle fields (OFFLINE_BUNDLE_AUDIT.json records the 5.0.2 case; 6.2.6 behaves alike). - Call before importing policyengine; returns the variables removed.""" - removed = [name for name in TOKEN_VARIABLES if environ.pop(name, None) is not None] + Without a token a private data repository answers 401; a public one + would still answer, so the proxy variables (NO_NETWORK) make the fetch + fail for both. Call before importing policyengine; returns the + variables removed.""" + removed = [ + name + for name in (*TOKEN_VARIABLES, *PROXY_EXCEPTIONS) + if environ.pop(name, None) is not None + ] environ.update(OFFLINE_VARIABLES) + environ.update(NO_NETWORK) return removed @@ -583,8 +727,9 @@ def materialize_dataset( dataset = record["identity"]["dataset"] runtime_root = (runtime_root or RUNTIME_ROOT / record["variant_id"]).resolve() destination = runtime_root / "data" / Path(dataset["path"]).name - if destination.is_symlink(): - raise VariantError(f"{destination} is a symlink; refusing it") + for path in (runtime_root / "data", destination): + if path.is_symlink(): + raise VariantError(f"{path} is a symlink; refusing it") if destination.resolve().parent != (runtime_root / "data").resolve(): raise VariantError("dataset destination escapes the runtime directory") if ( diff --git a/tests/test_uk_event_compute.py b/tests/test_uk_event_compute.py index 9ece78b..ed70ac1 100644 --- a/tests/test_uk_event_compute.py +++ b/tests/test_uk_event_compute.py @@ -16,6 +16,13 @@ from pipeline import stage_uk_event as stage +@pytest.fixture(autouse=True) +def engine_free_environment(monkeypatch): + """These tests stub the engine; the installed-environment check that + precedes every real run is tested in test_uk_replay_variants.py.""" + monkeypatch.setattr(compute, "check_variant_environment", lambda variant: {}) + + def test_optional_engine_provenance_preserves_legacy_metadata(monkeypatch): variable = SimpleNamespace( label="Income tax", @@ -557,7 +564,7 @@ def test_no_digest_no_simulation(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(RuntimeError("missing digest")), + lambda *args: (_ for _ in ()).throw(RuntimeError("missing digest")), ) called = [] monkeypatch.setattr(compute, "run_sim", lambda *args: called.append(args)) @@ -681,7 +688,7 @@ def test_changed_registry_refused_before_preflight(tmp_path, monkeypatch): lambda document: registry.write_text("{}"), ) called = [] - monkeypatch.setattr(compute, "preflight", lambda: called.append("preflight")) + monkeypatch.setattr(compute, "preflight", lambda *args: called.append("preflight")) with pytest.raises(ValueError, match="registry bytes changed"): compute.main(["--event", "test_event", "--registry", str(registry)]) assert not called @@ -707,7 +714,7 @@ def test_compute_reuses_baselines_and_reruns_produce_identical_bytes( monkeypatch.setattr( compute, "preflight", - lambda: { + lambda *args: { "sha256": "a" * 64, "release_bundle": {"certified_data_build_id": "certified-test"}, }, @@ -773,7 +780,7 @@ def test_cli_event_identity_checked_before_preflight(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(AssertionError("preflight must not run")), + lambda *args: (_ for _ in ()).throw(AssertionError("preflight must not run")), ) with pytest.raises(ValueError, match="event identity"): compute.main(["--event", "test_event", "--registry", str(path)]) @@ -794,7 +801,7 @@ def test_focused_default_rerun_guard_precedes_preflight(tmp_path, monkeypatch): monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(AssertionError("preflight reached")), + lambda *args: (_ for _ in ()).throw(AssertionError("preflight reached")), ) args = ["--event", "test_event", "--measures", measure()["measure_key"]] with pytest.raises(ValueError, match="focused reruns"): @@ -823,7 +830,7 @@ def test_interrupted_compute_resumes_only_receipted_artifacts(tmp_path, monkeypa monkeypatch.setattr( compute, "preflight", - lambda: { + lambda *args: { "sha256": "a" * 64, "release_bundle": {"certified_data_build_id": "certified-test"}, }, diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py index 0e0ba24..821d81b 100644 --- a/tests/test_uk_replay_variants.py +++ b/tests/test_uk_replay_variants.py @@ -8,8 +8,11 @@ import base64 import copy import hashlib +import importlib import importlib.metadata import json +import os +import sys from pathlib import Path import pytest @@ -46,7 +49,7 @@ def test_committed_variants_validate_and_default_is_legacy(): rv.validate_record(variant) assert rv.is_legacy(rv.get_variant()) assert not rv.is_legacy(rv.get_variant(CURRENT)) - with pytest.raises(rv.VariantError, match="no certified replay variant"): + with pytest.raises(rv.VariantError, match="no replay variant"): rv.get_variant("pe9.9.9_uk9.9.9_unknown") @@ -469,7 +472,9 @@ def test_variant_run_skips_unsupported_years_and_stages_them_visibly( monkeypatch.setattr( compute, "preflight", - lambda: (_ for _ in ()).throw(AssertionError("legacy preflight for a variant")), + lambda *args: (_ for _ in ()).throw( + AssertionError("legacy preflight for a variant") + ), ) monkeypatch.setattr( compute, @@ -538,7 +543,9 @@ def fake_run(year, variables, reform, pre): # A forged unsupported-year artifact is refused by staging. forged = copy.deepcopy(manifest) forged["unsupported_years"] = [{"year": 2024, "reason": "before_data_year"}] - with pytest.raises(ValueError, match="does not support|supported grid"): + with pytest.raises( + ValueError, match="does not support|supported grid|supported years differ" + ): stage.stage_event( registry_document, forged, @@ -605,3 +612,415 @@ def managed_microsimulation(**kwargs): runtime_dataset_source=data_file, expected_dataset_sha256=digest, ) + + +# --- review of #160 ------------------------------------------------------------ + + +def staged_variant_run(tmp_path, monkeypatch): + """A one-measure variant run over CY2023–2024 (2023 unsupported).""" + monkeypatch.setattr(compute, "ROOT", tmp_path) + registry_path = tmp_path / "registry.json" + registry = {"calendar_years": [2023, 2024], "measures": [measure()]} + registry_path.write_text(json.dumps(registry)) + monkeypatch.setattr(compute.registry_builder, "validate_registry", lambda d: None) + monkeypatch.setattr(compute.worlds, "engine_resolver", lambda: lambda p, d: None) + monkeypatch.setattr( + compute, + "variant_preflight", + lambda variant, dataset_source=None: { + "sha256": "a" * 64, + "release_bundle": { + "certified_data_build_id": "policyengine-uk-data-1.58.0" + }, + }, + ) + monkeypatch.setattr( + compute, + "run_sim", + lambda year, variables, reform, pre: ( + simulation(900, 1000) if reform else simulation(1000, 1030) + ), + ) + output = tmp_path / "results" / "uk" / "replays" / CURRENT / "test_event" + args = [ + "--event", + "test_event", + "--variant", + CURRENT, + "--registry", + str(registry_path), + "--output-dir", + str(output), + ] + assert compute.main(args) == 0 + manifest = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + digest = hashlib.sha256(registry_path.read_bytes()).hexdigest() + return registry, manifest, output, digest, args + + +@pytest.mark.parametrize( + "label,forge", + [ + # B1: a self-consistent manifest that claims 2023 is supported. + ( + "no unsupported years", + lambda m: m.update(unsupported_years=[], supported_years=[2023, 2024]), + ), + ("other data years", lambda m: m.update(data_years=[2023, 2030])), + ("other identity", lambda m: m.update(variant_identity_sha256="f" * 64)), + ("unknown variant", lambda m: m.update(variant_id="pe9_uk9_x")), + # Deleting variant_id must not turn the identity check off. + ("variant fields dropped", lambda m: m.pop("variant_id")), + ], +) +def test_staging_checks_a_manifests_variant_claims_against_the_record( + tmp_path, monkeypatch, label, forge +): + registry, manifest, output, digest, _ = staged_variant_run(tmp_path, monkeypatch) + stage.stage_event(registry, manifest, artifact_dir=output, registry_sha256=digest) + forged = copy.deepcopy(manifest) + forge(forged) + with pytest.raises((ValueError, rv.VariantError)): + stage.stage_event(registry, forged, artifact_dir=output, registry_sha256=digest) + + +def test_a_legacy_manifest_lists_no_variant_fields(): + registry = {"calendar_years": [2023], "measures": []} + for fields in ( + {"unsupported_years": [{"year": 2023, "reason": "before_data_year"}]}, + {"variant_identity_sha256": "a" * 64}, + ): + with pytest.raises(ValueError, match="legacy compute manifest"): + stage.manifest_variant(fields, registry, variants_path=rv.VARIANTS_PATH) + assert stage.manifest_variant({}, registry, variants_path=rv.VARIANTS_PATH) is None + + +def test_resume_refuses_a_receipt_for_an_unsupported_year(tmp_path, monkeypatch): + # B2: a receipt (from an older revision, or edited) naming a 2023 + # artifact must not put that year into a variant's manifest. + registry, manifest, output, digest, args = staged_variant_run(tmp_path, monkeypatch) + forged = output / "test_event__tax_and_benefit_2023.json" + forged.write_bytes( + (output / "test_event__tax_and_benefit_2024.json") + .read_bytes() + .replace(b'"year": 2024', b'"year": 2023') + ) + manifest["artifacts"][str(forged.relative_to(tmp_path))] = hashlib.sha256( + forged.read_bytes() + ).hexdigest() + (output / "RUN_MANIFEST.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="outside the variant's supported years"): + compute.main(args + ["--resume"]) + # A progress receipt for 2023 is never read at all. + (output / "RUN_MANIFEST.json").unlink() + (output / "RUN_PROGRESS_2023.json").write_text( + json.dumps({"artifacts": {str(forged.relative_to(tmp_path)): "0" * 64}}) + ) + assert compute.main(args + ["--resume"]) == 0 + rerun = json.loads((output / "RUN_MANIFEST.json").read_bytes()) + assert all("_2023.json" not in path for path in rerun["artifacts"]) + + +def test_staging_and_comparison_refuse_a_sealed_namespace(tmp_path, monkeypatch): + # B3: the staged rows and comparison are part of what the receipt binds. + from pipeline import compare_uk_event as compare + + registry, manifest, output, digest, _ = staged_variant_run(tmp_path, monkeypatch) + (output.parent / rv.RECEIPT_NAME).write_text("{}") + monkeypatch.setattr(rv, "ROOT", tmp_path) + monkeypatch.setattr( + stage.compute, "event_paths", lambda event: (tmp_path / "registry.json", output) + ) + with pytest.raises(rv.VariantError, match="sealed"): + stage.main(["--event", "test_event", "--artifact-dir", str(output)]) + assert not (output / "STAGED.jsonl").exists() + with pytest.raises(rv.VariantError, match="sealed"): + compare.write_summary(output.parent, artifact_root=tmp_path) + + +def test_population_note_names_the_variant_population(): + assert stage.population_note(None) == stage.LEGACY_POPULATION_NOTE + assert "single certified 2023 population" in stage.LEGACY_POPULATION_NOTE + note = stage.population_note(record()) + assert "enhanced_frs_2024_25.h5" in note and "1.58.0" in note + assert "2023" not in note + + +def test_only_certified_variants_run_unless_qualifying(tmp_path): + document = rv.load_document() + candidate = copy.deepcopy(document) + for r in candidate["variants"]: + if r["variant_id"] == CURRENT: + r["status"] = "candidate" + path = tmp_path / "variants.json" + path.write_text(json.dumps(candidate)) + with pytest.raises(rv.VariantError, match="not certified"): + rv.get_variant(CURRENT, path=path) + assert rv.get_variant(CURRENT, path=path, allow_uncertified=True)["status"] == ( + "candidate" + ) + candidate["default_variant"] = CURRENT + path.write_text(json.dumps(candidate)) + with pytest.raises(rv.VariantError, match="default_variant must be"): + rv.load_document(path) + + +def test_the_cli_refuses_a_candidate_without_the_qualification_flag( + tmp_path, monkeypatch +): + document = rv.load_document() + for r in document["variants"]: + if r["variant_id"] == CURRENT: + r["status"] = "candidate" + path = tmp_path / "variants.json" + path.write_text(json.dumps(document)) + args = ["--event", "autumn_budget_2024", "--variant", CURRENT, "--dry-run"] + with pytest.raises(rv.VariantError, match="not certified"): + compute.main(args + ["--variants-file", str(path)]) + + +def test_the_legacy_variant_refuses_a_dataset_file(): + # M5: the legacy variant reads its Hugging Face cache; a file would be + # silently ignored. + with pytest.raises(ValueError, match="--dataset-file"): + compute.main( + ["--event", "autumn_budget_2024", "--dataset-file", "x.h5", "--dry-run"] + ) + + +def test_a_legacy_run_checks_its_installed_environment_first(monkeypatch): + # M4: the legacy record's digests are checked before its preflight. + calls = [] + monkeypatch.setattr( + compute, + "check_variant_environment", + lambda variant: ( + calls.append(variant["variant_id"]) + or (_ for _ in ()).throw(rv.VariantError("not the certified environment")) + ), + ) + monkeypatch.setattr( + compute, + "preflight", + lambda *a: (_ for _ in ()).throw(AssertionError("preflight before check")), + ) + monkeypatch.setattr(rv, "assert_not_sealed", lambda path: None) + with pytest.raises(rv.VariantError, match="not the certified environment"): + compute.main(["--event", "autumn_budget_2024"]) + assert calls == [LEGACY] + + +@pytest.mark.parametrize( + "label,mutate", + [ + ("dataset repo", lambda r: r["identity"]["dataset"].update(repo_id="a/b")), + ("dataset revision", lambda r: r["identity"]["dataset"].update(revision="9")), + ("dataset path", lambda r: r["identity"]["dataset"].update(path="other.h5")), + ( + "dataset repo type", + lambda r: r["identity"]["dataset"].update(repo_type="dataset"), + ), + ("layout", lambda r: r.update(layout="flat")), + ("bool data year", lambda r: r.update(data_years=[True, 2])), + ("certification", lambda r: r.update(certification="certified")), + ("identity", lambda r: r.update(identity=[])), + ], +) +def test_single_field_changes_are_refused(label, mutate): + r = record() + mutate(r) + with pytest.raises(rv.VariantError): + rv.validate_record(r) + + +def test_a_wrong_legacy_result_root_is_refused(): + r = record(LEGACY) + r["result_root"] = f"{rv.REPLAY_RESULT_ROOT}/{LEGACY}" + with pytest.raises(rv.VariantError, match="result_root"): + rv.validate_record(r) + + +def test_a_requirements_pin_that_disagrees_is_refused(tmp_path): + r = record() + text = (rv.ROOT / r["requirements"]["path"]).read_text() + changed = text.replace("policyengine-uk==2.125.1", "policyengine-uk==2.125.2") + assert changed != text + path = tmp_path / "docs" / "requirements.txt" + path.parent.mkdir(parents=True) + path.write_text(changed) + r["requirements"] = { + "path": "docs/requirements.txt", + "sha256": hashlib.sha256(changed.encode()).hexdigest(), + } + r["identity"]["requirements_sha256"] = r["requirements"]["sha256"] + reseal(r) + with pytest.raises(rv.VariantError, match="requirements pin policyengine-uk"): + rv.validate_record(r, root=tmp_path) + + +def demo_site(tmp_path, monkeypatch): + files = { + "demo/__init__.py": b"x = 1\n", + "demo/parameters/rate.yaml": b"values: {2024-01-01: 0.2}\n", + } + site = tmp_path / "site" + write_distribution(site, files, b"uv\n") + monkeypatch.syspath_prepend(str(site)) + importlib.invalidate_caches() + return site + + +def test_an_unlisted_file_in_a_package_directory_is_found(tmp_path, monkeypatch): + # B4: core loads every file in a parameter directory, so a file no RECORD + # lists can change results while every listed file still verifies. + site = demo_site(tmp_path, monkeypatch) + assert rv.unlisted_package_files(("demo",)) == {} + (site / "demo" / "__pycache__").mkdir() + (site / "demo" / "__pycache__" / "y.cpython-312.pyc").write_bytes(b"\0") + assert rv.unlisted_package_files(("demo",)) == {} # bytecode is never read + (site / "demo" / "parameters" / "extra.yaml").write_text("values: {}\n") + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/extra.yaml"] + } + + +def test_unlisted_files_fail_the_installed_check(): + r = record() + installed = installed_for(r) + rv.verify_installed(r, installed=installed) + installed["unlisted_files"] = {"policyengine-uk": ["policyengine_uk/x.yaml"]} + with pytest.raises(rv.VariantError, match="files outside its RECORD"): + rv.verify_installed(r, installed=installed) + + +def test_bytecode_isolation_precedes_engine_import(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + monkeypatch.setattr(sys, "dont_write_bytecode", sys.dont_write_bytecode) + environ = {} + for module in rv.ENGINE_MODULES.values(): + monkeypatch.delitem(sys.modules, module, raising=False) + stale = tmp_path / "no-bytecode" / "policyengine_uk" + stale.mkdir(parents=True) + (stale / "x.pyc").write_bytes(b"\0") + prefix = rv.isolate_bytecode(tmp_path, environ) + assert sys.pycache_prefix == str(prefix) == environ["PYTHONPYCACHEPREFIX"] + assert sys.dont_write_bytecode and environ["PYTHONDONTWRITEBYTECODE"] == "1" + assert list(prefix.iterdir()) == [] # a stale cache is cleared + monkeypatch.setitem(sys.modules, "policyengine_uk", object()) + with pytest.raises(rv.VariantError, match="imported before isolation"): + rv.isolate_bytecode(tmp_path, environ) + + +def test_an_engine_imported_from_another_tree_is_refused(tmp_path, monkeypatch): + site = demo_site(tmp_path, monkeypatch) + monkeypatch.setitem(rv.ENGINE_MODULES, "demo", "demo") + module = type(sys)("demo") + module.__file__ = str(site / "demo" / "__init__.py") + monkeypatch.setitem(sys.modules, "demo", module) + rv.verify_import_origins(("demo",)) + shadow = tmp_path / "shadow" / "demo" / "__init__.py" + shadow.parent.mkdir(parents=True) + shadow.write_text("x = 2\n") + module.__file__ = str(shadow) + with pytest.raises(rv.VariantError, match="not its verified distribution"): + rv.verify_import_origins(("demo",)) + + +def test_isolation_routes_every_http_request_to_a_closed_port(): + # M2: requests ignores HF_HUB_OFFLINE but reads these variables + # (urllib.request.getproxies), so the release-manifest fetch fails and + # the wrapper uses its packaged certification. + import urllib.request + + environ = {"NO_PROXY": "*", "no_proxy": "huggingface.co", "PATH": "/bin"} + removed = rv.isolate_certification_environment(environ) + assert {"NO_PROXY", "no_proxy"} <= set(removed) + original = dict(os.environ) + try: + os.environ.clear() + os.environ.update(environ) + proxies = urllib.request.getproxies() + finally: + os.environ.clear() + os.environ.update(original) + assert proxies["https"] == proxies["http"] == "http://127.0.0.1:9" + assert "no" not in proxies + + +def test_a_symlinked_data_directory_is_refused(tmp_path): + payload = b"certified bytes" + r = tiny_variant(payload) + source = tmp_path / "source.h5" + source.write_bytes(payload) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "data").symlink_to(elsewhere, target_is_directory=True) + with pytest.raises(rv.VariantError, match="symlink"): + rv.materialize_dataset(r, runtime, source=source) + assert list(elsewhere.iterdir()) == [] + + +def test_a_dataset_that_does_not_state_its_years_fails_when_required( + monkeypatch, tmp_path +): + # M3: a variant's simulation fails closed when its dataset hides its years. + from pipeline import compute_uk_obr_costings as fiscal + + data_file = tmp_path / "data.h5" + data_file.write_bytes(b"x") + + class Sim: + policyengine_bundle = { + "certified_data_build_id": "build", + "runtime_dataset_source": str(data_file), + } + tax_benefit_system = type("System", (), {"variables": {"income_tax": None}})() + dataset = object() + + def calculate(self, name, period): + import numpy as np + + return np.array([1.0]) + + class Uk: + @staticmethod + def managed_microsimulation(**kwargs): + return Sim() + + import types + + monkeypatch.setitem(sys.modules, "policyengine", types.SimpleNamespace(uk=Uk)) + monkeypatch.setattr(fiscal, "variable_metadata", lambda *a, **k: {}) + kwargs = dict( + year=2024, + variables=["income_tax"], + reform=None, + runtime_dataset_source=data_file, + expected_dataset_sha256=hashlib.sha256(b"x").hexdigest(), + ) + assert fiscal.run_managed_simulation(**kwargs)["aggregates_gbp"] # legacy + with pytest.raises(RuntimeError, match="does not state its years"): + fiscal.run_managed_simulation(**kwargs, require_dataset_years=True) + + +def test_the_installed_engine_is_a_committed_variant_when_present(): + """In a pinned venv (the CI replay-engine job, .venv-candidate, + .venv-replay), the real installed environment reproduces a committed + record's digests, with no unlisted files.""" + try: + version = importlib.metadata.version("policyengine-uk") + except importlib.metadata.PackageNotFoundError: + pytest.skip("no engine installed") + matches = [ + r + for r in rv.load_document()["variants"] + if r["identity"]["packages"]["policyengine-uk"]["version"] == version + ] + if not matches: + pytest.skip(f"policyengine-uk {version} is no committed variant") + installed = rv.installed_identity() + assert installed["unlisted_files"] == {} + rv.verify_installed(matches[0], installed=installed) From dc040dedd921bb61bcb5a16c3dd9dc35c7fa3de4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:50:39 -0400 Subject: [PATCH 8/9] Receipt schema: close the status loopholes; pin the certification recipe - rejected: variant_id, variant_identity_sha256 and execution must be null and integrity not_run (nothing ran). - complete: variant_identity_sha256 must be a digest and results.manifest_sha256 must be present (every complete receipt is sealed; results.commit stays optional because the commit binding lives in STATUS.json). - execution.python is anchored; a request's data.path refuses ".." segments. - The golden receipt gains a placeholder results.manifest_sha256; its JCS bytes and receipt_sha256 are regenerated (new digest 5cbc44c1...). The golden request and its request_id are unchanged. - expected_release_bundle's description and agreed point 4 now name the full isolation (tokens and NO_PROXY removed, proxies at a closed port), pending W4's confirmation. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/QUALIFICATION_INTERFACE.md | 8 +- .../schemas/qualification_request.v1.json | 4 +- docs/uk_replay/schemas/replay_receipt.v1.json | 26 +++++- tests/fixtures/uk_replay/DIGESTS.json | 2 +- .../uk_replay/replay_receipt_golden.jcs | 2 +- .../uk_replay/replay_receipt_golden.json | 5 +- tests/test_uk_replay_interface.py | 80 +++++++++++++++++++ 7 files changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index 15b712a..d99fac7 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -221,7 +221,13 @@ How flags arise, so a gate built on them sees the intended events: `receipt_commit`; nothing of this receipt nests inside W4's execution receipt. 4. W4 computes `expected_release_bundle` in a subprocess with the three - Hugging Face token variables removed. + Hugging Face token variables and `NO_PROXY`/`no_proxy` removed, and the + HTTP(S) proxy variables pointed at a closed local port, exactly as + `uk_replay_variants.isolate_certification_environment` does. (Amended 11 + October, pending W4's confirmation: removing the tokens alone forces the + packaged certification only for a private data repository; policyengine.py + fetches the release manifest with `requests`, which ignores + `HF_HUB_OFFLINE`, so a public one would still answer.) 5. W4's proposed gate on these fields (pending d1344): `rejected`, `failed`, integrity `failed` or `incomplete`, and structural, coverage, mapping, inertness or construction flags fail the replay gate; numerical and diff --git a/docs/uk_replay/schemas/qualification_request.v1.json b/docs/uk_replay/schemas/qualification_request.v1.json index 16921af..5865ca3 100644 --- a/docs/uk_replay/schemas/qualification_request.v1.json +++ b/docs/uk_replay/schemas/qualification_request.v1.json @@ -76,14 +76,14 @@ "repo_id": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "repo_type": {"enum": ["model", "dataset"]}, "revision": {"type": "string", "minLength": 1}, - "path": {"type": "string", "minLength": 1, "pattern": "^[^/][^\\\\]*$"}, + "path": {"type": "string", "minLength": 1, "pattern": "^(?!.*(^|/)\\.\\.(/|$))[^/][^\\\\]*$"}, "sha256": {"$ref": "#/$defs/sha256"}, "size_bytes": {"type": "integer", "minimum": 1}, "release_manifest_sha256": {"$ref": "#/$defs/sha256"} } }, "expected_release_bundle": { - "description": "The exact dict(pe.uk.uk_latest.release_bundle) under packaged certification, computed with HF_TOKEN, HUGGING_FACE_TOKEN and HUGGINGFACE_HUB_TOKEN removed from the environment.", + "description": "The exact dict(pe.uk.uk_latest.release_bundle) under packaged certification: computed in a subprocess with HF_TOKEN, HUGGING_FACE_TOKEN, HUGGINGFACE_HUB_TOKEN, NO_PROXY and no_proxy removed and HTTP_PROXY, HTTPS_PROXY, ALL_PROXY (and their lowercase forms) set to http://127.0.0.1:9, so the wrapper's release-manifest request cannot reach the network and the packaged certification is used for a private or public data repository (pipeline/uk_replay_variants.py, isolate_certification_environment).", "type": "object", "required": ["policyengine_version", "model_version", "certified_data_build_id", "certified_data_artifact_sha256", "default_dataset_uri"], "additionalProperties": {"type": ["string", "null"]}, diff --git a/docs/uk_replay/schemas/replay_receipt.v1.json b/docs/uk_replay/schemas/replay_receipt.v1.json index f127f28..a9d5261 100644 --- a/docs/uk_replay/schemas/replay_receipt.v1.json +++ b/docs/uk_replay/schemas/replay_receipt.v1.json @@ -140,7 +140,7 @@ }, "python": { "type": "string", - "pattern": "^3\\.[0-9]+\\.[0-9]+" + "pattern": "^3\\.[0-9]+\\.[0-9]+$" }, "platform": { "type": "string", @@ -421,6 +421,22 @@ }, "diff": { "type": "null" + }, + "variant_id": { + "type": "null" + }, + "variant_identity_sha256": { + "type": "null" + }, + "execution": { + "type": "null" + }, + "integrity": { + "properties": { + "status": { + "const": "not_run" + } + } } } } @@ -443,6 +459,14 @@ }, "execution": { "type": "object" + }, + "variant_identity_sha256": { + "$ref": "#/$defs/sha256" + }, + "results": { + "required": [ + "manifest_sha256" + ] } } } diff --git a/tests/fixtures/uk_replay/DIGESTS.json b/tests/fixtures/uk_replay/DIGESTS.json index b7e61f6..c7f000d 100644 --- a/tests/fixtures/uk_replay/DIGESTS.json +++ b/tests/fixtures/uk_replay/DIGESTS.json @@ -3,6 +3,6 @@ "request_id": "2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf" }, "replay_receipt_golden.json": { - "receipt_sha256": "a070ac40cb4e6d7daef520cfc1be4ed4958cdd4fe275975dc6d55e856d4ff515" + "receipt_sha256": "5cbc44c1b79506a6410c12b437a7451a0e1387ab6a133b9cb8dc0bf71b396650" } } diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.jcs b/tests/fixtures/uk_replay/replay_receipt_golden.jcs index 871f1d3..d542e5b 100644 --- a/tests/fixtures/uk_replay/replay_receipt_golden.jcs +++ b/tests/fixtures/uk_replay/replay_receipt_golden.jcs @@ -1 +1 @@ -{"corpus":{"corpus_version":"uk-fiscal-event-replay/1","events":{"spring_budget_2023":{"artifacts":8,"comparison_provenance_sha256":"8888888888888888888888888888888888888888888888888888888888888888","full_supported_grid_complete":true,"inert_measure_years":[],"registry_sha256":"5555555555555555555555555555555555555555555555555555555555555555","run_manifest_sha256":"6666666666666666666666666666666666666666666666666666666666666666","staging_manifest_sha256":"7777777777777777777777777777777777777777777777777777777777777777","supported_years":[2024,2025,2026,2027],"unsupported_years":[{"reason":"before_data_year","year":2023}]}}},"diff":{"base_variant_id":"pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z","diff_sha256":"9999999999999999999999999999999999999999999999999999999999999999","flag_counts":{"construction":0,"coverage":0,"inertness":0,"mapping":0,"numerical":2,"structural":1,"support":1},"flagged_rows":[{"base_gbp":-15000000000000000,"event":"spring_budget_2023","flags":["numerical"],"head_gbp":0.30000000000000004,"key":"universal_credit","level":"head","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2024},{"base_gbp":1e-7,"event":"spring_budget_2023","flags":["numerical","structural"],"head_gbp":-123456789.125,"key":"total","level":"measure_year","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2025},{"base_gbp":-2500000,"event":"spring_budget_2023","flags":["support"],"head_gbp":null,"key":"obr-pmd-row-123","level":"source_row","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2023}]},"execution":{"dataset_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","packages":{"policyengine":{"record_sha256":"1111111111111111111111111111111111111111111111111111111111111111","version":"6.2.6"},"policyengine-core":{"record_sha256":"3333333333333333333333333333333333333333333333333333333333333333","version":"3.33.0"},"policyengine-uk":{"record_sha256":"2222222222222222222222222222222222222222222222222222222222222222","version":"2.125.1"}},"platform":"Golden fixture (synthetic values; café £)","python":"3.12.14","release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"requirements_sha256":"4444444444444444444444444444444444444444444444444444444444444444","runner":"github-actions ubuntu-latest"},"integrity":{"failures":[],"status":"complete"},"rejection":null,"request_id":"2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf","results":{"commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","path":"requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/","ref":"replay-state","repository":"PolicyEngine/policyengine-scorecard"},"schema":"policyengine.uk-replay-receipt/v1","status":"complete","variant_id":"pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0","variant_identity_sha256":"0000000000000000000000000000000000000000000000000000000000000000"} \ No newline at end of file +{"corpus":{"corpus_version":"uk-fiscal-event-replay/1","events":{"spring_budget_2023":{"artifacts":8,"comparison_provenance_sha256":"8888888888888888888888888888888888888888888888888888888888888888","full_supported_grid_complete":true,"inert_measure_years":[],"registry_sha256":"5555555555555555555555555555555555555555555555555555555555555555","run_manifest_sha256":"6666666666666666666666666666666666666666666666666666666666666666","staging_manifest_sha256":"7777777777777777777777777777777777777777777777777777777777777777","supported_years":[2024,2025,2026,2027],"unsupported_years":[{"reason":"before_data_year","year":2023}]}}},"diff":{"base_variant_id":"pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z","diff_sha256":"9999999999999999999999999999999999999999999999999999999999999999","flag_counts":{"construction":0,"coverage":0,"inertness":0,"mapping":0,"numerical":2,"structural":1,"support":1},"flagged_rows":[{"base_gbp":-15000000000000000,"event":"spring_budget_2023","flags":["numerical"],"head_gbp":0.30000000000000004,"key":"universal_credit","level":"head","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2024},{"base_gbp":1e-7,"event":"spring_budget_2023","flags":["numerical","structural"],"head_gbp":-123456789.125,"key":"total","level":"measure_year","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2025},{"base_gbp":-2500000,"event":"spring_budget_2023","flags":["support"],"head_gbp":null,"key":"obr-pmd-row-123","level":"source_row","measure_key":"spring_budget_2023__uc_childcare_cap_increase","year":2023}]},"execution":{"dataset_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","packaged_bundle_manifest_sha256":"8633ad814c9c85e53c7c513f69c4c7b1500a03cd2f6ef133d6e2ea8346b2db54","packages":{"policyengine":{"record_sha256":"1111111111111111111111111111111111111111111111111111111111111111","version":"6.2.6"},"policyengine-core":{"record_sha256":"3333333333333333333333333333333333333333333333333333333333333333","version":"3.33.0"},"policyengine-uk":{"record_sha256":"2222222222222222222222222222222222222222222222222222222222222222","version":"2.125.1"}},"platform":"Golden fixture (synthetic values; café £)","python":"3.12.14","release_bundle":{"bundle_id":"uk-6.2.6","certified_by":"policyengine.py bundle certification","certified_data_artifact_sha256":"5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6","certified_data_build_id":"policyengine-uk-data-1.58.0","compatibility_basis":"legacy_compatible_model_package","country_id":"uk","data_build_fingerprint":"sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8","data_build_model_git_sha":"4cbedbecca352f52dffe47e849553c2759a00711","data_build_model_version":"2.122.2","data_package":"policyengine-uk-data","data_version":"1.58.0","default_dataset":"enhanced_frs_2024_25","default_dataset_uri":"hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0","model_package":"policyengine-uk","model_version":"2.125.1","policyengine_version":"6.2.6"},"requirements_sha256":"4444444444444444444444444444444444444444444444444444444444444444","runner":"github-actions ubuntu-latest"},"integrity":{"failures":[],"status":"complete"},"rejection":null,"request_id":"2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf","results":{"commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","manifest_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","path":"requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/","ref":"replay-state","repository":"PolicyEngine/policyengine-scorecard"},"schema":"policyengine.uk-replay-receipt/v1","status":"complete","variant_id":"pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0","variant_identity_sha256":"0000000000000000000000000000000000000000000000000000000000000000"} \ No newline at end of file diff --git a/tests/fixtures/uk_replay/replay_receipt_golden.json b/tests/fixtures/uk_replay/replay_receipt_golden.json index 433f095..8789893 100644 --- a/tests/fixtures/uk_replay/replay_receipt_golden.json +++ b/tests/fixtures/uk_replay/replay_receipt_golden.json @@ -131,7 +131,8 @@ "repository": "PolicyEngine/policyengine-scorecard", "ref": "replay-state", "commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "path": "requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/" + "path": "requests/2e4dd8350bfdc5a39780e002c14ab49c02cbf28b77b222af420e38c68bb945cf/results/", + "manifest_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, - "receipt_sha256": "a070ac40cb4e6d7daef520cfc1be4ed4958cdd4fe275975dc6d55e856d4ff515" + "receipt_sha256": "5cbc44c1b79506a6410c12b437a7451a0e1387ab6a133b9cb8dc0bf71b396650" } diff --git a/tests/test_uk_replay_interface.py b/tests/test_uk_replay_interface.py index 7cd4c78..aa30afa 100644 --- a/tests/test_uk_replay_interface.py +++ b/tests/test_uk_replay_interface.py @@ -113,6 +113,9 @@ def test_rejected_receipt_carries_a_reason_and_no_results(): receipt.update( status="rejected", rejection={"code": "release_bundle_mismatch", "detail": "model_version"}, + variant_id=None, + variant_identity_sha256=None, + execution=None, corpus=None, diff=None, integrity={"status": "not_run", "failures": []}, @@ -131,6 +134,83 @@ def test_rejected_receipt_carries_a_reason_and_no_results(): ) +def _rejected(): + receipt = golden("replay_receipt_golden.json") + receipt.update( + status="rejected", + rejection={"code": "incompatible_pair", "detail": "x"}, + variant_id=None, + variant_identity_sha256=None, + execution=None, + corpus=None, + diff=None, + integrity={"status": "not_run", "failures": []}, + ) + return receipt + + +@pytest.mark.parametrize( + "change", + [ + # A rejected request ran nothing and has no identity (review of #160, M6). + {"variant_id": "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0"}, + {"variant_identity_sha256": "0" * 64}, + {"execution": golden("replay_receipt_golden.json")["execution"]}, + {"integrity": {"status": "complete", "failures": []}}, + {"integrity": {"status": "failed", "failures": []}}, + ], +) +def test_a_rejected_receipt_cannot_claim_a_run(change): + ui.validate(ui.seal_receipt(_rejected()), ui.RECEIPT_SCHEMA_ID) + with pytest.raises(jsonschema.ValidationError): + ui.validate(ui.seal_receipt({**_rejected(), **change}), ui.RECEIPT_SCHEMA_ID) + + +def test_a_complete_receipt_names_its_identity_and_sealed_namespace(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "variant_identity_sha256": None}), + ui.RECEIPT_SCHEMA_ID, + ) + results = {k: v for k, v in receipt["results"].items() if k != "manifest_sha256"} + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt({**receipt, "results": results}), ui.RECEIPT_SCHEMA_ID + ) + + +def test_a_failed_receipt_cannot_report_complete_integrity(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + receipt["status"] = "failed" + for status in ("complete", "incomplete", "not_run"): + with pytest.raises(jsonschema.ValidationError): + ui.validate( + ui.seal_receipt( + {**receipt, "integrity": {"status": status, "failures": []}} + ), + ui.RECEIPT_SCHEMA_ID, + ) + + +def test_execution_python_is_exactly_a_version(): + receipt = golden("replay_receipt_golden.json") + del receipt["receipt_sha256"] + receipt["execution"] = {**receipt["execution"], "python": "3.12.13; rm -rf"} + with pytest.raises(jsonschema.ValidationError): + ui.validate(ui.seal_receipt(receipt), ui.RECEIPT_SCHEMA_ID) + + +@pytest.mark.parametrize("path", ["../x.h5", "a/../x.h5", "a/..", ".."]) +def test_request_dataset_path_has_no_parent_segments(path): + request = golden("qualification_request_golden.json") + request["candidate"]["data"]["path"] = path + with pytest.raises(jsonschema.ValidationError): + ui.validate_request(request) + + def test_tampered_receipt_fails_its_digest(): receipt = golden("replay_receipt_golden.json") receipt["diff"]["flagged_rows"][0]["head_gbp"] = 0.3 From eec87f42719cc73f41774ac940f67f5e55a67dba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:12:05 -0400 Subject: [PATCH 9/9] Replay variants: second review round, verified on both executors Review of dc040de: - The Modal image allowlist lacked pipeline/uk_replay_variants.py and data/uk/replay_variants.json, which compute_uk_event has imported and read since this branch began, so every remote run would have failed at import. Both are added; a test walks the worker's pipeline imports and the files a legacy run reads and requires each in the allowlist. - The unlisted-file scan could be bypassed. policyengine-core's loaders recurse into every subdirectory, __pycache__ included (tax_benefit_system.py: glob "*/"), and follow symlinked ones (parameter_node.py: os.path.isdir). The scan now skips only *.pyc inside a __pycache__, reports every symlink, and counts a file as listed only in its own package's RECORD, by unresolved path. A *.pth startup file beside the packages must belong to a pinned distribution or be the virtual environment's own shim (site.py runs their import lines). - The symlink guard in isolate_bytecode resolved the path first and never fired; it now checks before anything resolves or removes it. - The legacy variant now runs under the same certification isolation and the same exact release-bundle comparison as any other variant; --guards compiles the engine from source and checks import origins too. - Staging also checks the manifest's dataset digest and data bundle against the record. - installed_distributions keeps the first copy of a name on sys.path, as Python's import does. Found by running this on Modal: its containers keep their own client packages behind the pinned environment, and counting the later copy refused a correct environment. - Legacy preservation is now tested: staging, comparison and the summary for all five events regenerate from committed artifacts and match the committed bytes; no legacy artifact, manifest or receipt carries a variant field. - Docs: the checks' real order; what the identity leaves out and why; what the installed check does not bind; the seal manifest and the CI job are marked as arriving with the receiver change; interface point 4 is marked confirmed by W4 (policyengine.py#574). Verified: a focused legacy replay on Modal (AB2024 CGT, 2025) through every check is byte-identical to the committed artifact (sha256 f8dcfdd9...); the 6.2.6 --guards run and focused replay pass locally with an artifact byte-identical to the committed one (8316a941...); both pinned venvs reproduce their records. Full suite: 1536 passed before the last fix, then the changed files' 169 tests. Co-Authored-By: Claude Opus 5.5 --- docs/uk_replay/QUALIFICATION_INTERFACE.md | 9 +- docs/uk_replay/RELEASE_REPLAY.md | 68 +++++--- pipeline/compute_uk_event.py | 13 +- pipeline/modal_uk_event.py | 2 + pipeline/stage_uk_event.py | 7 + pipeline/uk_replay_variants.py | 134 +++++++++++---- tests/test_modal_uk_event.py | 63 +++++++ tests/test_uk_replay_legacy.py | 94 +++++++++++ tests/test_uk_replay_variants.py | 191 +++++++++++++++++++++- 9 files changed, 514 insertions(+), 67 deletions(-) diff --git a/docs/uk_replay/QUALIFICATION_INTERFACE.md b/docs/uk_replay/QUALIFICATION_INTERFACE.md index d99fac7..1303577 100644 --- a/docs/uk_replay/QUALIFICATION_INTERFACE.md +++ b/docs/uk_replay/QUALIFICATION_INTERFACE.md @@ -224,10 +224,11 @@ How flags arise, so a gate built on them sees the intended events: Hugging Face token variables and `NO_PROXY`/`no_proxy` removed, and the HTTP(S) proxy variables pointed at a closed local port, exactly as `uk_replay_variants.isolate_certification_environment` does. (Amended 11 - October, pending W4's confirmation: removing the tokens alone forces the - packaged certification only for a private data repository; policyengine.py - fetches the release manifest with `requests`, which ignores - `HF_HUB_OFFLINE`, so a public one would still answer.) + October and confirmed by W4 the same day, implemented in + policyengine.py#574: removing the tokens alone forces the packaged + certification only for a private data repository; policyengine.py fetches + the release manifest with `requests`, which ignores `HF_HUB_OFFLINE`, so a + public one would still answer.) 5. W4's proposed gate on these fields (pending d1344): `rejected`, `failed`, integrity `failed` or `incomplete`, and structural, coverage, mapping, inertness or construction flags fail the replay gate; numerical and diff --git a/docs/uk_replay/RELEASE_REPLAY.md b/docs/uk_replay/RELEASE_REPLAY.md index c0835e2..4c8c1f8 100644 --- a/docs/uk_replay/RELEASE_REPLAY.md +++ b/docs/uk_replay/RELEASE_REPLAY.md @@ -82,7 +82,7 @@ A variant record: | Field | Meaning | |---|---| | `variant_id` | `pe_uk_`, recomputed from `identity`; when another identity already holds that slug (same versions, other wheels or numerical environment) the id adds `--`. | -| `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. | +| `identity` | Everything that determines execution. Its canonical-JSON SHA-256 is `identity_sha256`. `data_years`, `layout`, `status` and `executor` are outside it on purpose: they describe or file a run and do not change a computed byte. `data_years` is checked instead against the simulation's own `dataset.years` in every run, and staging recomputes the supported years from the record. | | `identity.packages` | For `policyengine`, `policyengine-uk`, `policyengine-core`: exact `version`, and `record_sha256`: the SHA-256 of the distribution's own `RECORD` entries, sorted, excluding installer bookkeeping (`INSTALLER`, `REQUESTED`, `RECORD`, `direct_url.json`, `uv_cache.json`), bytecode, and the console-script wrappers the installer generates outside site-packages (their shebang names the environment's interpreter). Every listed file is checked on disk against its recorded hash when the identity is read, so the digest binds the wheel's installed contents, and pip and uv installs of one wheel agree. Optional `wheel_sha256`: the wheel's digest as published on PyPI, recorded at certification; the receiver verifies each downloaded wheel against it, and the runner, which never sees a wheel, checks only its format. | | `identity.packaged_bundle_manifest_sha256` | SHA-256 of `policyengine/data/bundle/manifest.json` as installed. | | `identity.release_bundle` | The exact `pe.uk.uk_latest.release_bundle` dict expected at runtime under **packaged (offline) certification**. | @@ -100,8 +100,13 @@ A variant record: ### What "validated exactly" means -Before any simulation the runner checks, in order, and refuses on the first -failure: +Before any simulation the runner checks all of the following and refuses on +the first failure. The numbering groups them by subject; the code runs record +consistency, the installed environment, certification isolation, dataset +materialisation, bytecode isolation, the engine import and its origins, then +the release-bundle comparison (`compute_uk_event.variant_preflight`; the +legacy variant's `preflight` runs the same checks around its original +managed-release preflight). 1. **Record consistency (engine-free).** `variant_id` and `identity_sha256` recompute; `release_bundle.model_version == packages["policyengine-uk"].version`; @@ -116,15 +121,26 @@ failure: and `compatible_model_packages` agree with the record. 2. **Installed environment.** Installed versions of the three packages equal the record; every installed file matches its `RECORD` hash and the - normalised `RECORD` hashes to `record_sha256`; no file sits in the three - packages' directories without a `RECORD` entry in some installed - distribution (policyengine-core loads every file in a parameter directory - and every module in a variable directory, listed or not); every - distribution pinned in the requirements freeze is installed at its pinned - version (extra distributions, such as the receipt tools, are allowed); the - installed packaged manifest hashes to `packaged_bundle_manifest_sha256`; - the interpreter minor version matches. The legacy variant is checked the - same way. + normalised `RECORD` hashes to `record_sha256`; nothing sits in the three + packages' directories that the package's own `RECORD` does not list: + policyengine-core loads every file in a parameter directory and every + module in a variable directory, recursing into subdirectories + (`__pycache__` included) and following symlinked ones, so an unlisted + file, any symlink, and anything but `*.pyc` inside a `__pycache__` are + refused, and another distribution's `RECORD` does not count; every + `*.pth` startup file beside the packages belongs to a distribution pinned + in the requirements freeze or is the virtual environment's own + `_virtualenv.pth`; every distribution pinned in the requirements freeze + is installed at its pinned version (extra distributions, such as the + receipt tools, are allowed); the installed packaged manifest hashes to + `packaged_bundle_manifest_sha256`; the interpreter minor version matches. + The legacy variant is checked the same way. + + What this does not bind: the files of pinned distributions other than the + three packages are checked by version, not by hash, and an extra + distribution's importable modules are not inspected. The check catches a + drifted or mixed environment; it is not a defence against someone who + can edit the environment at will. 3. **What is imported.** Before the engine is imported, bytecode caching is turned off and `sys.pycache_prefix` points at an empty directory, so Python compiles every engine module from the verified sources and never @@ -143,6 +159,9 @@ failure: `certify_data_release_compatibility` falls back to the wrapper's packaged certification. Then `dict(pe.uk.uk_latest.release_bundle) == identity.release_bundle`, exactly. + The legacy variant runs under the same isolation and the same exact + comparison, in addition to its original check against + `data/uk/certified_bundle.json`. 5. **Dataset bytes.** The materialised file hashes to `dataset.sha256` and has `size_bytes`, before construction and after the last aggregate (existing `run_managed_simulation` checks), and the simulation's reported @@ -189,14 +208,16 @@ FY2023–24 counterparts; that loss is shown, never hidden. names its namespace (legacy: `results/uk/events`) and its `status`. A candidate's record and results live on the `replay-state` branch under `requests//results/` until it is certified. -* A namespace with `REPLAY_RECEIPT.json` is **sealed**: - `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under it (and of - its release diff), the receipt binds that manifest's digest, the runner - refuses to write there (compute, staging, comparison and the summary all - check), and CI fails if any sealed byte changes. A rerun of - a sealed variant writes to an explicit separate output directory, and its - bytes against the sealed run are a determinism check (CI replays one real - measure-year this way). +* A namespace with `REPLAY_RECEIPT.json` is **sealed**: the runner refuses + to write there (compute, staging, comparison and the summary all check). + A rerun of a sealed variant writes to an explicit separate output + directory, and its bytes against the sealed run are a determinism check. + The receiver (`pipeline/uk_replay_qualify.py`, which arrives with the + receiver change, PR #163) writes the seal itself: + `NAMESPACE_MANIFEST.json` lists the SHA-256 of every file under the + namespace (and of its release diff) and the receipt binds that manifest's + digest. The same change adds the CI job that fails when a sealed byte + changes and replays one real measure-year; until it lands, neither exists. * Release differences live in `results/uk/replays/diffs/__/`. ## Version-scoped diagnoses @@ -305,8 +326,11 @@ Each has deterministic boundary cases and Hypothesis properties in * **Replay determinism.** For the same complete execution fingerprint (variant identity, registry bytes, measure, year), numerical artifact bytes are identical; timings stay in `RUN_LOG.json`. -* **Legacy preservation.** Legacy event registries, staging and comparison - outputs regenerate byte-identically after these changes. +* **Legacy preservation.** Legacy staging, comparison and summary outputs + regenerate byte-identically from the committed artifacts + (`tests/test_uk_replay_legacy.py` reruns all five events and compares + bytes), and no committed legacy artifact, manifest or receipt carries a + variant field. * **Diff flags.** `diff(a, a)` has no row flags. Flag sets are symmetric under swapping sides (with signs reversed). No numerical flag at or below the threshold; one strictly above it. Row order and file order do not change diff --git a/pipeline/compute_uk_event.py b/pipeline/compute_uk_event.py index 1d69fdd..b7ecf66 100644 --- a/pipeline/compute_uk_event.py +++ b/pipeline/compute_uk_event.py @@ -285,11 +285,14 @@ def build_artifact( def preflight(variant: dict | None = None) -> dict: """The committed pin and the managed release must agree before any run. - Given the legacy variant record, the engine is also compiled from its - verified sources and imported from its own distributions.""" + Given the legacy variant record, the run is also isolated like any other + variant's (packaged certification, engine compiled from its verified + sources and imported from its own distributions) and the release bundle + must equal the record's exactly.""" fiscal.configure_offline() worlds.configure_offline() if variant is not None: + variants.isolate_certification_environment() variants.isolate_bytecode(variants.RUNTIME_ROOT / variant["variant_id"]) committed = worlds.preflight() managed = fiscal.preflight_certified_dataset() @@ -300,6 +303,7 @@ def preflight(variant: dict | None = None) -> dict: raise RuntimeError("managed release differs from data/uk/certified_bundle.json") if variant is not None: import_engine() + variants.verify_release_bundle(variant, managed["release_bundle"]) return {**managed, "check_import_origins": True} return managed @@ -713,7 +717,10 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("\n".join(coverage)) if args.dry_run or args.guards: if args.guards: + # --guards reads the installed engine: check it is the certified + # one and compile it from its verified sources. check_variant_environment(variant) + variants.isolate_bytecode(variants.RUNTIME_ROOT / variant["variant_id"]) try: resolve = worlds.engine_resolver() except ImportError: @@ -724,6 +731,8 @@ def main(argv: list[str] | None = None) -> int: gaps = worlds.reversal_delta_mismatches(index, resolve, years=years) if gaps: raise ValueError("\n".join(gaps)) + if args.guards: + import_engine() guard = "every reversal delta restates certified current law" plan = { "event": args.event, diff --git a/pipeline/modal_uk_event.py b/pipeline/modal_uk_event.py index d5f4361..998c640 100644 --- a/pipeline/modal_uk_event.py +++ b/pipeline/modal_uk_event.py @@ -71,7 +71,9 @@ "pipeline/compute_uk_obr_costings.py", "pipeline/build_uk_event_registry.py", "pipeline/uk_engine_registry.py", + "pipeline/uk_replay_variants.py", "data/uk/certified_bundle.json", + "data/uk/replay_variants.json", "docs/uk_replay/requirements.txt", "docs/uk_replay/OFFLINE_BUNDLE_AUDIT.json", "sources/uk_replay/source.json", diff --git a/pipeline/stage_uk_event.py b/pipeline/stage_uk_event.py index d290e81..d1f5155 100644 --- a/pipeline/stage_uk_event.py +++ b/pipeline/stage_uk_event.py @@ -139,6 +139,13 @@ def manifest_variant( raise ValueError("the legacy variant's manifest names no variant_id") if manifest.get("variant_identity_sha256") != record["identity_sha256"]: raise ValueError("compute manifest variant identity differs from its record") + identity = record["identity"] + if manifest.get("certified_dataset_sha256") != identity["dataset"][ + "sha256" + ] or manifest.get("data_bundle") != fiscal.data_bundle_id( + identity["release_bundle"] + ): + raise ValueError("compute manifest dataset differs from the variant's record") supported, unsupported = variants.supported_years( record, registry["calendar_years"] ) diff --git a/pipeline/uk_replay_variants.py b/pipeline/uk_replay_variants.py index 49bfda3..a9d54cd 100644 --- a/pipeline/uk_replay_variants.py +++ b/pipeline/uk_replay_variants.py @@ -474,6 +474,19 @@ def distribution_record_sha256(distribution, *, verify_files: bool = True) -> st return hashlib.sha256(body).hexdigest() +def installed_distributions() -> dict[str, str]: + """Version of each importable distribution. When a name appears twice on + sys.path the first one is what Python imports (and what + importlib.metadata.version reports), so the first wins here too: a + host's own packages behind the pinned environment, such as Modal's + client dependencies in its containers, do not count.""" + versions = {} + for distribution in importlib.metadata.distributions(): + name = re.sub(r"[-_.]+", "-", distribution.metadata["Name"]).lower() + versions.setdefault(name, distribution.version) + return versions + + def installed_identity(*, verify_files: bool = True) -> dict: packages = {} for name in PACKAGES: @@ -494,56 +507,116 @@ def installed_identity(*, verify_files: bool = True) -> dict: "packages": packages, "packaged_bundle_manifest_sha256": sha256_file(manifest), "python": f"{sys.version_info.major}.{sys.version_info.minor}", - "distributions": { - re.sub(r"[-_.]+", "-", d.metadata["Name"]).lower(): d.version - for d in importlib.metadata.distributions() - }, + "distributions": installed_distributions(), "unlisted_files": unlisted_package_files() if verify_files else {}, + "startup_files": startup_files() if verify_files else [], } def unlisted_package_files(names=PACKAGES) -> dict[str, list[str]]: - """Files in each package's top-level directories that no installed - distribution's RECORD lists. + """Files and links in each package's top-level directories that the + package's own RECORD does not list. A listed file is checked against its hash, but policyengine-core also - loads files nobody listed: every file in a parameter directory - (parameters/parameter_node.py, sorted os.listdir) and every module in a - variable directory (taxbenefitsystems/tax_benefit_system.py, - add_variables_from_directory). Bytecode caches are skipped here because - runs never read them (isolate_bytecode).""" - listed = set() - for distribution in importlib.metadata.distributions(): - for line in (distribution.read_text("RECORD") or "").splitlines(): - relative = line.rsplit(",", 2)[0] - if relative: - listed.add(Path(distribution.locate_file(relative)).resolve()) + loads files nobody listed: every file in a parameter directory and its + subdirectories (parameters/parameter_node.py: sorted os.listdir, then + os.path.isdir) and every module in a variable directory and its + subdirectories (taxbenefitsystems/tax_benefit_system.py, + add_variables_from_directory: glob "*.py", then glob "*/"). Both recurse + into a ``__pycache__`` and follow a symlinked directory, so: + + * only ``*.pyc`` files inside a ``__pycache__`` are skipped (runs never + read them: isolate_bytecode); anything else there is reported; + * every symlink, to a file or a directory, is reported; + * a file counts as listed only in its own package's RECORD, by its + unresolved path: another distribution's RECORD is never hash-checked. + """ found = {} for name in names: distribution = importlib.metadata.distribution(name) + rows = _record_rows(distribution) + listed = { + os.path.normpath(str(distribution.locate_file(row[0]))) for row in rows + } tops = { row[0].split("/", 1)[0] - for row in _record_rows(distribution) + for row in rows if "/" in row[0] and ".dist-info/" not in row[0] } + root = str(distribution.locate_file("")) extra = [] for top in sorted(tops): - base = Path(distribution.locate_file(top)) + base = str(distribution.locate_file(top)) + if os.path.islink(base): + extra.append(f"{top} (symlink)") + continue for directory, subdirectories, files in os.walk(base): - subdirectories[:] = sorted( - d for d in subdirectories if d != "__pycache__" - ) - for file in sorted(files): - path = Path(directory) / file - if path.resolve() not in listed: - extra.append( - str(path.relative_to(distribution.locate_file(""))) - ) + subdirectories.sort() + cache = "__pycache__" in Path(directory).parts + for entry in sorted([*subdirectories, *files]): + path = os.path.normpath(os.path.join(directory, entry)) + relative = os.path.relpath(path, root) + if os.path.islink(path): + extra.append(f"{relative} (symlink)") + elif entry in subdirectories: + continue + elif cache and entry.endswith(".pyc"): + continue + elif path not in listed: + extra.append(relative) if extra: found[name] = extra return found +VIRTUALENV_SHIM = ("_virtualenv.pth", "import _virtualenv") + + +def startup_files(names=PACKAGES) -> list[dict]: + """The ``*.pth`` files beside the engine packages. Python runs their + ``import`` lines at startup (site.py), before any check here, so each + must belong to a pinned distribution or be the virtual environment's + own shim; verify_installed decides.""" + owners = {} + for distribution in importlib.metadata.distributions(): + owner = re.sub(r"[-_.]+", "-", distribution.metadata["Name"]).lower() + for line in (distribution.read_text("RECORD") or "").splitlines(): + relative = line.rsplit(",", 2)[0] + if relative.endswith(".pth"): + owners[os.path.normpath(str(distribution.locate_file(relative)))] = ( + owner + ) + sites = sorted( + {str(importlib.metadata.distribution(name).locate_file("")) for name in names} + ) + out = [] + for site in sites: + for entry in sorted(os.listdir(site)): + if not entry.endswith(".pth"): + continue + path = os.path.normpath(os.path.join(site, entry)) + out.append( + { + "name": entry, + "owner": owners.get(path), + "text": Path(path).read_text(errors="replace").strip()[:200], + "symlink": os.path.islink(path), + } + ) + return out + + +def startup_file_problems(files: list[dict], pins: dict[str, str]) -> list[str]: + problems = [] + for file in files: + shim = (file["name"], file["text"]) == VIRTUALENV_SHIM + if file.get("symlink") or not (shim or file["owner"] in pins): + problems.append( + f"startup file {file['name']} belongs to no pinned distribution" + ) + return problems + + def isolate_bytecode(runtime_root: Path, environ=os.environ) -> Path: """Compile the engine from its verified sources, never from a cache. @@ -554,8 +627,8 @@ def isolate_bytecode(runtime_root: Path, environ=os.environ) -> Path: loaded = sorted(m for m in ENGINE_MODULES.values() if m in sys.modules) if loaded: raise VariantError(f"engine modules imported before isolation: {loaded}") - prefix = (runtime_root / "no-bytecode").resolve() - if prefix.is_symlink(): + prefix = runtime_root.resolve() / "no-bytecode" + if prefix.is_symlink(): # before anything resolves or removes it raise VariantError(f"{prefix} is a symlink; refusing it") if prefix.exists(): shutil.rmtree(prefix) @@ -638,6 +711,7 @@ def verify_installed( + (f" and {len(paths) - 5} more" if len(paths) > 5 else "") for name, paths in sorted(installed.get("unlisted_files", {}).items()) ] + mismatches += startup_file_problems(installed.get("startup_files", []), pins) if mismatches: raise VariantError( f"{record['variant_id']}: installed environment is not the certified one: " diff --git a/tests/test_modal_uk_event.py b/tests/test_modal_uk_event.py index 4a75e47..ca03260 100644 --- a/tests/test_modal_uk_event.py +++ b/tests/test_modal_uk_event.py @@ -359,3 +359,66 @@ def concurrent(**kwargs): for key in recorded["env"] ) assert len(recorded["mounts"]) == len(mounts) + 1 # build-time frozen requirements + + +# Imported inside functions the replay worker never calls: the fiscal +# costings CLI's own restage and comparison steps (restage_existing_run and +# main in compute_uk_obr_costings.py). +NOT_ON_THE_WORKER_PATH = {"pipeline/compare_uk_obr_costings.py"} + + +def pipeline_imports(relative: str) -> set[str]: + """Every pipeline module a file imports, at module level or in a function.""" + import ast + + found = set() + for node in ast.walk(ast.parse((runner.ROOT / relative).read_text())): + names = [] + if isinstance(node, ast.ImportFrom) and node.module == "pipeline": + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "pipeline." + ): + names = [node.module.split(".", 1)[1]] + elif isinstance(node, ast.Import): + names = [ + alias.name.split(".", 1)[1] + for alias in node.names + if alias.name.startswith("pipeline.") + ] + for name in names: + candidate = f"pipeline/{name.replace('.', '/')}.py" + if (runner.ROOT / candidate).is_file(): + found.add(candidate) + return found + + +def pipeline_import_closure(start: str) -> set[str]: + seen, queue = set(), [start] + while queue: + relative = queue.pop() + if relative in seen or relative in NOT_ON_THE_WORKER_PATH: + continue + seen.add(relative) + queue.extend(pipeline_imports(relative)) + return seen + + +def test_the_remote_image_holds_everything_the_runner_imports_and_reads(): + # Review of #160: the worker imports compute_uk_event, which imports the + # variants module and reads the variants document; a file missing from + # the allowlist fails every remote run at import. + from pipeline import uk_replay_variants as variants + + allowed = set(runner.repository_files("autumn_budget_2024")) + needed = pipeline_import_closure("pipeline/modal_uk_event.py") + assert "pipeline/uk_replay_variants.py" in needed + assert needed <= allowed, sorted(needed - allowed) + legacy = variants.get_variant() + assert variants.is_legacy(legacy) + read = { + str(variants.VARIANTS_PATH.relative_to(runner.ROOT)), + str(variants.CERTIFIED_BUNDLE.relative_to(runner.ROOT)), + legacy["requirements"]["path"], + } + assert read <= allowed, sorted(read - allowed) diff --git a/tests/test_uk_replay_legacy.py b/tests/test_uk_replay_legacy.py index 19f8276..1ef998c 100644 --- a/tests/test_uk_replay_legacy.py +++ b/tests/test_uk_replay_legacy.py @@ -4,6 +4,8 @@ import json from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parent.parent @@ -30,3 +32,95 @@ def unavailable(): capsys.readouterr().out == (ROOT / "tests/fixtures/uk_replay_ab2025_dry_run.txt").read_text() ) + + +LEGACY_EVENTS = ( + "autumn_budget_2024", + "autumn_statement_2023", + "spring_budget_2024", + "spring_statement_2025", + "spring_budget_2023", +) + + +@pytest.mark.parametrize("event", LEGACY_EVENTS) +def test_committed_legacy_staging_and_comparison_regenerate_byte_for_byte( + event, tmp_path +): + """The variant work must not move a legacy byte: staging and comparison, + rerun from the committed artifacts, reproduce the committed files.""" + from pipeline import compare_uk_event as compare + from pipeline import compute_uk_event as compute + from pipeline import compute_uk_obr_costings as fiscal + from pipeline import stage_uk_event as stage + + registry_path, artifact_dir = compute.event_paths(event) + payload = registry_path.read_bytes() + manifest = json.loads((artifact_dir / "RUN_MANIFEST.json").read_bytes()) + assert "variant_id" not in manifest and "unsupported_years" not in manifest + rows, tally = stage.stage_event( + json.loads(payload), + manifest, + artifact_dir=artifact_dir, + registry_sha256=hashlib.sha256(payload).hexdigest(), + ) + staged = b"".join( + (json.dumps(row, sort_keys=True, allow_nan=False) + "\n").encode() + for row in rows + ) + assert staged == (artifact_dir / "STAGED.jsonl").read_bytes() + staging_manifest = compute.canonical_bytes( + { + **tally, + "event": event, + "staged_path": fiscal.relative_to_root(artifact_dir / "STAGED.jsonl"), + "staged_sha256": hashlib.sha256(staged).hexdigest(), + } + ) + assert staging_manifest == (artifact_dir / "STAGING_MANIFEST.json").read_bytes() + compare.write_comparison( + event, staged_path=artifact_dir / "STAGED.jsonl", output_dir=tmp_path + ) + written = sorted(path.name for path in tmp_path.iterdir()) + assert written == [ + "COMPARISON.csv", + "COMPARISON.json", + "COMPARISON.md", + "COMPARISON_PROVENANCE.json", + ] + for name in written: + assert (tmp_path / name).read_bytes() == (artifact_dir / name).read_bytes() + + +def test_committed_legacy_summary_regenerates_byte_for_byte(monkeypatch): + from pipeline import compare_uk_event as compare + + written = {} + monkeypatch.setattr( + compare, "atomic_write_bytes", lambda path, data: written.update({path: data}) + ) + root = ROOT / "results" / "uk" / "events" + compare.write_summary(root) + assert list(written) == [root / "SUMMARY.md"] + assert written[root / "SUMMARY.md"] == (root / "SUMMARY.md").read_bytes() + + +def test_legacy_artifacts_and_manifests_carry_no_variant_fields(): + """Every committed legacy artifact, manifest and progress receipt keeps + its pre-variant shape.""" + root = ROOT / "results" / "uk" / "events" + checked = 0 + for event in LEGACY_EVENTS: + for path in sorted((root / event).glob("*.json")): + if path.name.startswith("COMPARISON"): + continue + document = json.loads(path.read_bytes()) + assert isinstance(document, dict) + leaked = [ + key + for key in document + if "variant" in key or "supported" in key or key == "data_years" + ] + assert not leaked, (path.name, leaked) + checked += 1 + assert checked > 100 diff --git a/tests/test_uk_replay_variants.py b/tests/test_uk_replay_variants.py index 821d81b..679e8c3 100644 --- a/tests/test_uk_replay_variants.py +++ b/tests/test_uk_replay_variants.py @@ -25,6 +25,7 @@ LEGACY = "pe5.0.2_uk2.89.2_populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" CURRENT = "pe6.2.6_uk2.125.1_policyengine-uk-data-1.58.0" +DATASET_SHA256 = rv.get_variant(CURRENT)["identity"]["dataset"]["sha256"] def record(variant_id=CURRENT): @@ -454,8 +455,8 @@ def simulation(tax, benefit): "model_version": "2.125.1", "certified_data_build_id": "policyengine-uk-data-1.58.0", }, - "dataset_sha256_before": "a" * 64, - "dataset_sha256_after": "a" * 64, + "dataset_sha256_before": DATASET_SHA256, + "dataset_sha256_after": DATASET_SHA256, "performance": {"wall_seconds": 1}, } @@ -480,7 +481,7 @@ def test_variant_run_skips_unsupported_years_and_stages_them_visibly( compute, "variant_preflight", lambda variant, dataset_source=None: { - "sha256": "a" * 64, + "sha256": variant["identity"]["dataset"]["sha256"], "release_bundle": { "certified_data_build_id": "policyengine-uk-data-1.58.0" }, @@ -629,7 +630,7 @@ def staged_variant_run(tmp_path, monkeypatch): compute, "variant_preflight", lambda variant, dataset_source=None: { - "sha256": "a" * 64, + "sha256": variant["identity"]["dataset"]["sha256"], "release_bundle": { "certified_data_build_id": "policyengine-uk-data-1.58.0" }, @@ -670,6 +671,8 @@ def staged_variant_run(tmp_path, monkeypatch): ("other data years", lambda m: m.update(data_years=[2023, 2030])), ("other identity", lambda m: m.update(variant_identity_sha256="f" * 64)), ("unknown variant", lambda m: m.update(variant_id="pe9_uk9_x")), + ("other dataset", lambda m: m.update(certified_dataset_sha256="b" * 64)), + ("other data bundle", lambda m: m.update(data_bundle="another-build")), # Deleting variant_id must not turn the identity check off. ("variant fields dropped", lambda m: m.pop("variant_id")), ], @@ -712,10 +715,22 @@ def test_resume_refuses_a_receipt_for_an_unsupported_year(tmp_path, monkeypatch) (output / "RUN_MANIFEST.json").write_text(json.dumps(manifest)) with pytest.raises(ValueError, match="outside the variant's supported years"): compute.main(args + ["--resume"]) - # A progress receipt for 2023 is never read at all. + # A progress receipt for 2023 is never read, even one whose identity + # fields match this run exactly (it would be accepted for 2024). (output / "RUN_MANIFEST.json").unlink() + genuine = json.loads((output / "RUN_PROGRESS_2024.json").read_bytes()) (output / "RUN_PROGRESS_2023.json").write_text( - json.dumps({"artifacts": {str(forged.relative_to(tmp_path)): "0" * 64}}) + json.dumps( + { + **genuine, + "year": 2023, + "artifacts": { + str(forged.relative_to(tmp_path)): hashlib.sha256( + forged.read_bytes() + ).hexdigest() + }, + } + ) ) assert compute.main(args + ["--resume"]) == 0 rerun = json.loads((output / "RUN_MANIFEST.json").read_bytes()) @@ -885,6 +900,69 @@ def test_an_unlisted_file_in_a_package_directory_is_found(tmp_path, monkeypatch) } +def test_the_scan_sees_what_core_would_load(tmp_path, monkeypatch): + # Review of dc040de: core's loaders recurse into __pycache__ and follow + # symlinked directories, and only the package's own RECORD is hash-checked. + site = demo_site(tmp_path, monkeypatch) + cache = site / "demo" / "parameters" / "__pycache__" + cache.mkdir() + (cache / "ok.cpython-312.pyc").write_bytes(b"\0") + (cache / "x.py").write_text("raise SystemExit\n") + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/__pycache__/x.py"] + } + (cache / "x.py").unlink() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "rate.yaml").write_text("values: {}\n") + (site / "demo" / "parameters" / "linked").symlink_to( + outside, target_is_directory=True + ) + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/linked (symlink)"] + } + (site / "demo" / "parameters" / "linked").unlink() + # A symlink to a listed file is still a symlink, not that file. + (site / "demo" / "alias.py").symlink_to(site / "demo" / "__init__.py") + assert rv.unlisted_package_files(("demo",)) == {"demo": ["demo/alias.py (symlink)"]} + (site / "demo" / "alias.py").unlink() + # Another distribution listing a file inside this package does not count. + extra = site / "demo" / "parameters" / "from_other.yaml" + extra.write_text("values: {}\n") + info = site / "other-1.0.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.1\nName: other\nVersion: 1.0\n") + (info / "RECORD").write_text("demo/parameters/from_other.yaml,,\n") + importlib.invalidate_caches() + assert rv.unlisted_package_files(("demo",)) == { + "demo": ["demo/parameters/from_other.yaml"] + } + + +def test_startup_files_must_belong_to_a_pinned_distribution(tmp_path, monkeypatch): + # site.py runs a .pth file's import lines before any check here. + site = demo_site(tmp_path, monkeypatch) + (site / "_virtualenv.pth").write_text("import _virtualenv\n") + (site / "hook.pth").write_text("import os; os.environ['X'] = '1'\n") + files = rv.startup_files(("demo",)) + assert [f["name"] for f in files] == ["_virtualenv.pth", "hook.pth"] + assert rv.startup_file_problems(files, {"demo": "1.0"}) == [ + "startup file hook.pth belongs to no pinned distribution" + ] + # The shim's name with other content is not the shim. + (site / "_virtualenv.pth").write_text("import evil\n") + assert len(rv.startup_file_problems(rv.startup_files(("demo",)), {})) == 2 + # A .pth a pinned distribution lists is that distribution's own. + owned = [{"name": "x.pth", "owner": "demo", "text": "import x", "symlink": False}] + assert rv.startup_file_problems(owned, {"demo": "1.0"}) == [] + assert rv.startup_file_problems(owned, {}) != [] + r = record() + installed = installed_for(r) + installed["startup_files"] = files + with pytest.raises(rv.VariantError, match="startup file hook.pth"): + rv.verify_installed(r, installed=installed) + + def test_unlisted_files_fail_the_installed_check(): r = record() installed = installed_for(r) @@ -912,6 +990,24 @@ def test_bytecode_isolation_precedes_engine_import(tmp_path, monkeypatch): rv.isolate_bytecode(tmp_path, environ) +def test_bytecode_isolation_refuses_a_symlinked_directory(tmp_path, monkeypatch): + # The guard runs before anything resolves or removes the path: a + # symlinked no-bytecode directory must not get its target deleted. + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + monkeypatch.setattr(sys, "dont_write_bytecode", sys.dont_write_bytecode) + for module in rv.ENGINE_MODULES.values(): + monkeypatch.delitem(sys.modules, module, raising=False) + precious = tmp_path / "precious" + precious.mkdir() + (precious / "keep.txt").write_text("keep") + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "no-bytecode").symlink_to(precious, target_is_directory=True) + with pytest.raises(rv.VariantError, match="symlink"): + rv.isolate_bytecode(runtime, {}) + assert (precious / "keep.txt").read_text() == "keep" + + def test_an_engine_imported_from_another_tree_is_refused(tmp_path, monkeypatch): site = demo_site(tmp_path, monkeypatch) monkeypatch.setitem(rv.ENGINE_MODULES, "demo", "demo") @@ -1007,9 +1103,10 @@ def managed_microsimulation(**kwargs): def test_the_installed_engine_is_a_committed_variant_when_present(): - """In a pinned venv (the CI replay-engine job, .venv-candidate, - .venv-replay), the real installed environment reproduces a committed - record's digests, with no unlisted files.""" + """In a pinned venv (.venv-candidate, .venv-replay, and the CI + replay-engine job once the receiver change adds it), the real installed + environment reproduces a committed record's digests, with no unlisted + files. Without an engine installed this test skips.""" try: version = importlib.metadata.version("policyengine-uk") except importlib.metadata.PackageNotFoundError: @@ -1024,3 +1121,79 @@ def test_the_installed_engine_is_a_committed_variant_when_present(): installed = rv.installed_identity() assert installed["unlisted_files"] == {} rv.verify_installed(matches[0], installed=installed) + + +def test_the_legacy_preflight_is_isolated_and_checks_its_release_bundle(monkeypatch): + # Review of dc040de, M4: the legacy run uses the packaged certification + # and must match its record's release bundle exactly, like any variant. + calls = [] + legacy = record(LEGACY) + bundle = dict(legacy["identity"]["release_bundle"]) + monkeypatch.setattr(compute.fiscal, "configure_offline", lambda: None) + monkeypatch.setattr(compute.worlds, "configure_offline", lambda: None) + monkeypatch.setattr( + rv, "isolate_certification_environment", lambda: calls.append("certification") + ) + monkeypatch.setattr(rv, "isolate_bytecode", lambda root: calls.append("bytecode")) + monkeypatch.setattr( + compute.worlds, + "preflight", + lambda: { + "sha256": legacy["identity"]["dataset"]["sha256"], + "revision": bundle["certified_data_build_id"], + }, + ) + managed = { + "sha256": legacy["identity"]["dataset"]["sha256"], + "release_bundle": bundle, + } + monkeypatch.setattr( + compute.fiscal, + "preflight_certified_dataset", + lambda: calls.append("engine") or dict(managed), + ) + monkeypatch.setattr(compute, "import_engine", lambda: calls.append("origins")) + pre = compute.preflight(legacy) + assert calls == ["certification", "bytecode", "engine", "origins"] + assert pre["check_import_origins"] is True + managed["release_bundle"] = {**bundle, "compatibility_basis": "other"} + with pytest.raises(rv.VariantError, match="runtime release bundle differs"): + compute.preflight(legacy) + # Without a record (the Modal preflight-only path) it is the original check. + calls.clear() + managed["release_bundle"] = bundle + assert "check_import_origins" not in compute.preflight() + assert calls == ["engine"] + + +def test_guards_check_the_environment_and_compile_from_source(monkeypatch): + calls = [] + monkeypatch.setattr( + compute, "check_variant_environment", lambda variant: calls.append("installed") + ) + monkeypatch.setattr(rv, "isolate_bytecode", lambda root: calls.append("bytecode")) + monkeypatch.setattr( + compute.worlds, + "engine_resolver", + lambda: calls.append("engine") or (lambda path, date: None), + ) + monkeypatch.setattr(compute.worlds, "reversal_delta_mismatches", lambda *a, **k: []) + monkeypatch.setattr(compute, "import_engine", lambda: calls.append("origins")) + assert compute.main(["--event", "autumn_budget_2024", "--guards"]) == 0 + assert calls == ["installed", "bytecode", "engine", "origins"] + + +def test_a_shadowed_distribution_copy_does_not_count(tmp_path, monkeypatch): + # Found on Modal: the container's own client packages sit behind the + # pinned environment on sys.path; Python imports the first copy. + for index, version in enumerate(("2.0", "1.0")): + info = tmp_path / f"site{index}" / f"shadowed-{version}.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: shadowed\nVersion: {version}\n" + ) + monkeypatch.syspath_prepend(str(tmp_path / "site1")) + monkeypatch.syspath_prepend(str(tmp_path / "site0")) # first on the path + importlib.invalidate_caches() + assert importlib.metadata.version("shadowed") == "2.0" + assert rv.installed_distributions()["shadowed"] == "2.0"