From 1f0809f918599a812db4828a6745bb0dcb6c0d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:11:34 +0200 Subject: [PATCH 01/26] Retarget the UK build to FRS 2024-25: release object, re-pins, period re-map, ingest scale ladder The frozen release object uk/frs_release.json (survey/base 2024, calibration 2025, SN 9563, DOI, UKDS zip sha, HF acquisition pins) drives lockstep asserts over the re-pinned raw-tab manifest: all 21 frs_table artifacts re-pinned to the 2024-25 tabs with the SPI-convention keys, six stages' SN 9252 prose defect fixed, and the typed spec moved in lockstep. TIME_PERIOD and the HMRC SPI/CGT build periods move to "2024"; the 2023-24 HMRC published surface is re-mapped as a signed nearest-available-vintage declaration (period_mapping: latest_published_tax_year) with the frozen original byte-untouched and the source-contract validator reconstructing the live canonical payload. take_up_contract build_year 2024 flips the two 2024 date-keyed rates. The ingest driver joins the #627 scale ladder (--sample-fraction post-frs_spine via the generic frame-sampling helpers) with receipt-postures on three full-scale fences at sampled rungs, and the WAS bridge-donor locator defect that refused every full-roster licensed run is fixed with a hermetic pin-coherence regression test. Part of #723. Co-Authored-By: Claude Fable 5 --- .../723-uk-frs-2024-25-retarget.changed.md | 1 + .../723-uk-ingest-scale-ladder.added.md | 1 + .../723-uk-was-bridge-donor-pin.fixed.md | 1 + .../spec_engine/schema/sources.schema.json | 9 + .../microcosm/build/uk/cgt_source_stages.json | 8 +- .../microcosm/build/uk/country_package.json | 5 + .../src/microcosm/build/uk/frs_release.json | 25 + .../src/microcosm/build/uk/source_stages.json | 319 ++++++---- .../src/microcosm/build/uk/spec/sources.yaml | 566 ++++++++---------- .../microcosm/build/uk/take_up_contract.json | 4 +- .../microcosm/build/uk_runtime/frs_release.py | 112 ++++ .../microcosm/build/uk_runtime/frs_spine.py | 2 +- .../build/uk_runtime/hmrc_capital_gains.py | 6 +- .../microcosm/build/uk_runtime/hmrc_income.py | 12 +- .../build/uk_runtime/hmrc_source_contract.py | 84 ++- .../microcosm/build/uk_runtime/spi_spine.py | 40 +- .../tests/test_country_spec.py | 2 + .../tests/test_spec_engine_country_bundles.py | 5 +- .../tests/test_uk_cgt_source_manifest.py | 6 +- .../tests/test_uk_frs_release.py | 87 +++ .../tests/test_uk_frs_spine.py | 161 ++++- .../tests/test_uk_hmrc_income.py | 4 +- .../test_uk_hmrc_income_source_manifest.py | 35 +- .../tests/test_uk_hmrc_replay.py | 4 +- .../tests/test_uk_national_build.py | 4 +- .../tests/test_uk_source_stages.py | 27 +- .../tests/test_uk_spi_spine.py | 22 + .../tests/test_uk_take_up_contract.py | 6 +- .../microcosm-build/tests/test_us_plan.py | 1 + tools/build_uk_frs_spine.py | 214 ++++++- tools/build_uk_national_dataset.py | 7 +- tools/verify_uk_identity_stability.py | 86 ++- 32 files changed, 1345 insertions(+), 521 deletions(-) create mode 100644 changelog.d/723-uk-frs-2024-25-retarget.changed.md create mode 100644 changelog.d/723-uk-ingest-scale-ladder.added.md create mode 100644 changelog.d/723-uk-was-bridge-donor-pin.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk/frs_release.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py create mode 100644 packages/microcosm-build/tests/test_uk_frs_release.py diff --git a/changelog.d/723-uk-frs-2024-25-retarget.changed.md b/changelog.d/723-uk-frs-2024-25-retarget.changed.md new file mode 100644 index 0000000000..c7b06615fc --- /dev/null +++ b/changelog.d/723-uk-frs-2024-25-retarget.changed.md @@ -0,0 +1 @@ +Retarget the UK build to FRS 2024-25 (#723): a new frozen release object (`uk/frs_release.json` — survey/base 2024, calibration year 2025, SN 9563, DOI, UKDS zip sha, HF acquisition pins) drives lockstep asserts over the re-pinned raw-tab manifest (all 21 `frs_table` artifacts at sha+size for the 2024-25 tabs, SPI-convention `tax_year_start`/`ukds_study_number`/`doi` keys), the build period moves to `"2024"`, the take-up contract's `build_year` follows (tax-free-childcare and TV-evasion rates flip to their 2024 date keys), and the HMRC 2023-24 published surface is re-mapped as a signed nearest-available-vintage declaration (`period_mapping: "latest_published_tax_year"`, values and vintage labels unchanged — no uprating). The parity bar re-freezes on `enhanced_frs_2024_25.h5` @ the v1.56.14 tag: the eFRS parity reference (145 columns, entity counts (16,288 raw + 10,000 SPI) × 2 + 270 CGT band donors = 52,846 — now test-pinned), the release-input coverage manifest, and the input-mass reference registry identity all move to the 2024-25 pins; gate thresholds do not move (#686 arms the re-measured baselines). Six stages' SN 9252 (FRS 2022-23) source citations were a standing prose defect, fixed and signed. diff --git a/changelog.d/723-uk-ingest-scale-ladder.added.md b/changelog.d/723-uk-ingest-scale-ladder.added.md new file mode 100644 index 0000000000..2045631b8f --- /dev/null +++ b/changelog.d/723-uk-ingest-scale-ladder.added.md @@ -0,0 +1 @@ +The UK ingest spine driver joins the #627 scale ladder (#723): `--sample-fraction` (0.01/0.10/1.0) subsamples households right after the raw `frs_spine` stage through the generic frame-sampling helpers, records a sampling receipt and rung token in the build sidecar, refuses `--checkpoint-dir` on sampled rungs, and adopts the named-edge abort receipt. Sampled rungs get an explicit receipt posture on three full-scale fences — raw-tab person coverage restricts to survivors, the SPI synthetic stack scales by the fraction, and the ultra-sparse effective-mass restoration check records instead of raising — while f100 keeps every fence strict. diff --git a/changelog.d/723-uk-was-bridge-donor-pin.fixed.md b/changelog.d/723-uk-was-bridge-donor-pin.fixed.md new file mode 100644 index 0000000000..11fb9c0668 --- /dev/null +++ b/changelog.d/723-uk-was-bridge-donor-pin.fixed.md @@ -0,0 +1 @@ +The WAS round-8 bridge-donor artifact in `lcfs_consumption` declared a different `locator` than `was_wealth`'s pin for the same tab, so the full-roster spine driver refused every licensed run at the pin-coherence check — a break invisible to secrets-free PR CI (#723). The locator is aligned to the caller-supplied convention and a hermetic regression test now collects `_artifact_pins()` over the complete manifest. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 31175588f5..1c7d9d1d5d 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -5309,6 +5309,15 @@ }, "runtime_sha256_required": { "type": "boolean" + }, + "tax_year_start": { + "type": "integer" + }, + "ukds_study_number": { + "type": "integer" + }, + "doi": { + "type": "string" } } }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json index 914b021d6d..9e237c095d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/cgt_source_stages.json @@ -35,8 +35,8 @@ "3_3_2021-22", "3_4_2020-21" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "basis": "Individuals with a CGT liability only; gains after losses and attributed gains but before the annual exempt amount; taxable income after reliefs and the Personal Allowance; trusts excluded; counts publish in thousands and amounts in GBP millions.", "runtime_sha256_required": true }, @@ -93,7 +93,7 @@ "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", "column_reconciliation": "every income column rescales onto its published All-row taxpayer total", "shortfall_policy": "proportional scale-down when the population holds less gainer mass than published taxpayers", - "minimum_allocation_people": 1.0, + "minimum_allocation_people": 1, "weights": "household_weight mapped to persons; no person splits across bands" }, { @@ -116,7 +116,7 @@ "kind": "record_mass_conservation_receipt", "entity": "household", "reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", - "declared_factor": 1.0, + "declared_factor": 1, "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this reason." }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 5a1613375e..f21b003025 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -52,6 +52,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "frs_release.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "gates.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/frs_release.json b/packages/microcosm-build/src/microcosm/build/uk/frs_release.json new file mode 100644 index 0000000000..b983251347 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/frs_release.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "country": "uk", + "policy": "The UK FRS release pin freezes the raw Family Resources Survey vintage that seeds the UK build; runtime loaders and source manifests must agree on its survey, base, calibration, UKDS, DOI, acquisition, and hash identity before licensed tabs are consumed.", + "name": "frs_2024_25", + "survey_year": 2024, + "base_year": 2024, + "calibration_year": 2025, + "time_period": "2024", + "vintage": "2024_25", + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1", + "ukds_tab_zip": { + "filename": "9563tab_05DD0069587DBD25E5719D355CE05FC0827D5EDD58C24ECE9AB85ACD954A9AEB_V1.zip", + "sha256": "05dd0069587dbd25e5719d355ce05fc0827d5edd58c24ece9ab85acd954a9aeb" + }, + "acquisition": { + "huggingface_repo": "policyengine/policyengine-uk-data-private", + "huggingface_revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "zip_filename": "frs_2024_25.zip", + "zip_sha256": "05dd0069587dbd25e5719d355ce05fc0827d5edd58c24ece9ab85acd954a9aeb", + "zip_size_bytes": 46637202 + }, + "notes": "Tape years labeled by tax year (2024_25) per the SPI-stage convention. Donor vintages (SPI 2022-23, WAS r8, LCFS 2023-24, ETB) unchanged. Adopted from the old E11 scope (microcosm#723)." +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 20a473d747..9e052b4c0e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -5,8 +5,8 @@ "stages": [ { "stage": "frs_spine", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; local licensed 2023_24 tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "household", "artifacts": [ { @@ -14,154 +14,196 @@ "table": "accounts", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "accounts.tab", - "sha256": "c5e31932bfd06087f835d2c83c0984c85a93409bf5ef85b699cb0958abcba1ea", - "size_bytes": 1807921, - "runtime_sha256_required": true + "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", + "size_bytes": 1812923, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benefits", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benefits.tab", - "sha256": "ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b", - "size_bytes": 4460292, - "runtime_sha256_required": true + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benunit", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benunit.tab", - "sha256": "88946815eace8561516d5cbb442c27e319c1e90abc381fb2338f0126e3b9e05b", - "size_bytes": 21213867, - "runtime_sha256_required": true + "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", + "size_bytes": 13986782, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "child", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "child.tab", - "sha256": "b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e", - "size_bytes": 2913156, - "runtime_sha256_required": true + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "chldcare", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "chldcare.tab", - "sha256": "566e0ebca1d5e2f3e424e556c91f4cb583d17dadfdfa59feb3841eda7e5976a3", - "size_bytes": 273837, - "runtime_sha256_required": true + "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", + "size_bytes": 275878, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "extchild", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "extchild.tab", - "sha256": "8d358d7ee66ee4a7ceab87b4f24fbbf21ac86dc038dc7831e51fb271f96a57ec", - "size_bytes": 18677, - "runtime_sha256_required": true + "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", + "size_bytes": 15150, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "househol", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "househol.tab", - "sha256": "5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30", - "size_bytes": 12387117, - "runtime_sha256_required": true + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "job", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "job.tab", - "sha256": "88b77ffe06865f029f713bb1d55ff12bdea8a1234de5bc293e72458fe64f3a74", - "size_bytes": 10934873, - "runtime_sha256_required": true + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "maint", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "maint.tab", - "sha256": "f2dc924eb5a51b0c357791693d15b431327dc39c6421011efb313d88bf839695", - "size_bytes": 15440, - "runtime_sha256_required": true + "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", + "size_bytes": 13993, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "mortgage", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "mortgage.tab", - "sha256": "ce36b477d67837c469608a0d68f7ef269ac04758974235f1157d2f6b92cdbfdc", - "size_bytes": 631783, - "runtime_sha256_required": true + "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", + "size_bytes": 600552, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "oddjob", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "oddjob.tab", - "sha256": "b4ba3dd3151f73a01422983c60514a3e38458ddfa4fb33ae4ed0326873406305", - "size_bytes": 5165, - "runtime_sha256_required": true + "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", + "size_bytes": 5339, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "penprov", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "penprov.tab", - "sha256": "ee001461c40306ec24b38b2881e1774121114266a2ee449d606cd0a811c37731", - "size_bytes": 522313, - "runtime_sha256_required": true + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "size_bytes": 513614, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "pension", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "pension.tab", - "sha256": "150d6fad1fce81254fb7aea1526fbb00b63d4027d6e2ac4c26bb90aea3127eb7", - "size_bytes": 1225838, - "runtime_sha256_required": true + "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", + "size_bytes": 1232411, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -306,12 +348,12 @@ "housing_service_charges", "external_child_payments" ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage." + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity." }, { "stage": "frs_employment", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "person", "artifacts": [ { @@ -319,11 +361,14 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -352,8 +397,8 @@ }, { "stage": "frs_council_tax", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "household", "artifacts": [ { @@ -361,11 +406,14 @@ "table": "househol", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "househol.tab", - "sha256": "5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30", - "size_bytes": 12387117, - "runtime_sha256_required": true + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -399,8 +447,8 @@ }, { "stage": "frs_disability", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", "grain": "person", "artifacts": [], "operations": [ @@ -427,8 +475,8 @@ }, { "stage": "frs_education", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs.", "grain": "person", "artifacts": [ { @@ -436,22 +484,28 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "child", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "child.tab", - "sha256": "b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e", - "size_bytes": 2913156, - "runtime_sha256_required": true + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "size_bytes": 2753961, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -493,12 +547,12 @@ "child_ema", "age_started_or_accepted_current_education_or_training" ], - "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary and falls to the default. EMA uses the shared weeks-in-year constant." + "notes": "Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant." }, { "stage": "frs_legacy_proxies", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters.", "grain": "person", "artifacts": [ { @@ -506,11 +560,14 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ @@ -543,8 +600,8 @@ }, { "stage": "frs_education_grant_split", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DfE grant parameters.", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters.", "grain": "person", "artifacts": [], "operations": [ @@ -571,12 +628,12 @@ "nonnegative_outputs": [ "disabled_students_allowance_eligible_expenses" ], - "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read." + "notes": "Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." }, { "stage": "frs_take_up", - "survey": "Family Resources Survey 2023-24", - "source": "Family Resources Survey 2023-24 reported receipt anchors plus sourced UK take-up contract rates.", + "survey": "Family Resources Survey 2024-25", + "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", "grain": "benunit", "artifacts": [], "operations": [ @@ -675,7 +732,7 @@ }, { "stage": "frs_person_draws", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", "grain": "person", "artifacts": [], @@ -721,7 +778,7 @@ }, { "stage": "frs_household_draws", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", "grain": "household", "artifacts": [], @@ -765,7 +822,7 @@ }, { "stage": "frs_brma", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor.", "grain": "household", "artifacts": [ @@ -1048,7 +1105,7 @@ "kind": "private_microdata", "format": "tab", "vintage": "2018_20", - "locator": "was_round_8_hhold_eul_may_2025_230525.tab", + "locator": "caller-supplied local input", "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "size_bytes": 39073613, "runtime_sha256_required": true, @@ -1457,8 +1514,8 @@ }, { "stage": "frs_hmrc_spine_leaves", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", "grain": "person", "artifacts": [ { @@ -1466,30 +1523,36 @@ "table": "adult", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "adult.tab", - "sha256": "e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06", - "size_bytes": 35323384, - "runtime_sha256_required": true + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { "role": "frs_table", "table": "benefits", "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": "benefits.tab", - "sha256": "ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b", - "size_bytes": 4460292, - "runtime_sha256_required": true + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" } ], "operations": [ { "kind": "retain_adjudicated_frs_hmrc_leaves", "population": "uk_frs_raw_spine", - "source_vintage": "2023-24", - "mapped_build_period": 2023, + "source_vintage": "2024-25", + "mapped_build_period": 2024, "annualization": "weekly raw FRS amounts * (365.25 / 7)", "status": "adjudicated_partial_replay", "retained_full_constituents": { @@ -1589,7 +1652,7 @@ }, { "stage": "spi_support_channel", - "survey": "Family Resources Survey 2023-24", + "survey": "Family Resources Survey 2024-25", "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", "grain": "household", "artifacts": [], @@ -1695,8 +1758,8 @@ "Table_3_6", "Table_3_7" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "runtime_sha256_required": true } ], @@ -1902,8 +1965,8 @@ "require_all_predictors": true, "require_all_outputs": true, "initialize_frs_channel_columns": { - "gift_aid": 0.0, - "charitable_investment_gifts": 0.0 + "gift_aid": 0, + "charitable_investment_gifts": 0 }, "n_estimators": 100, "seed": 42 @@ -2000,8 +2063,8 @@ { "kind": "materialize_hmrc_income_bands_fail_closed", "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "column_index_base": 0, "data_row_start_index": 5, "stop_label": "All ranges", @@ -2258,7 +2321,7 @@ "support_channel_column": "person_support_channel", "required_support_channel": "spi", "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, + "minimum_nondefault_mass_share": 0.000001, "fail_below_floor": true } ], @@ -2337,8 +2400,8 @@ }, { "stage": "frs_hmrc_retained_leaves", - "survey": "Family Resources Survey 2023-24", - "source": "Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", "grain": "person", "artifacts": [], "operations": [ @@ -2351,8 +2414,8 @@ { "kind": "retain_adjudicated_frs_hmrc_leaves", "population": "certified_microcosm_uk_candidate_base_channel", - "source_vintage": "2023-24", - "mapped_build_period": 2023, + "source_vintage": "2024-25", + "mapped_build_period": 2024, "annualization": "weekly raw FRS amounts * (365.25 / 7)", "status": "adjudicated_partial_replay", "retained_full_constituents": { @@ -2474,8 +2537,8 @@ "Table_3_6", "Table_3_7" ], - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "runtime_sha256_required": true } ], @@ -2778,8 +2841,8 @@ { "kind": "materialize_hmrc_income_bands_fail_closed", "artifact_role": "published_fact_surface", - "mapped_build_period": 2023, - "period_mapping": "tax_year_start", + "mapped_build_period": 2024, + "period_mapping": "latest_published_tax_year", "column_index_base": 0, "data_row_start_index": 5, "stop_label": "All ranges", @@ -3036,7 +3099,7 @@ "support_channel_column": "person_support_channel", "required_support_channel": "spi", "mass_share_denominator": "all_person_effective_mass", - "minimum_nondefault_mass_share": 1e-06, + "minimum_nondefault_mass_share": 0.000001, "fail_below_floor": true } ], diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 99f44cc1bf..e419475073 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -13,142 +13,181 @@ sources: stage_manifest: version: 1 country: uk - policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact - must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay - keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. + policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. stages: - stage: frs_spine - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; - local licensed 2023_24 tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: household artifacts: - role: frs_table table: accounts kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: accounts.tab - sha256: c5e31932bfd06087f835d2c83c0984c85a93409bf5ef85b699cb0958abcba1ea - size_bytes: 1807921 + sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + size_bytes: 1812923 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benefits kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benefits.tab - sha256: ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b - size_bytes: 4460292 + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + size_bytes: 2362329 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benunit kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benunit.tab - sha256: 88946815eace8561516d5cbb442c27e319c1e90abc381fb2338f0126e3b9e05b - size_bytes: 21213867 + sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + size_bytes: 13986782 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: child kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: child.tab - sha256: b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e - size_bytes: 2913156 + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + size_bytes: 2753961 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: chldcare kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: chldcare.tab - sha256: 566e0ebca1d5e2f3e424e556c91f4cb583d17dadfdfa59feb3841eda7e5976a3 - size_bytes: 273837 + sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + size_bytes: 275878 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: extchild kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: extchild.tab - sha256: 8d358d7ee66ee4a7ceab87b4f24fbbf21ac86dc038dc7831e51fb271f96a57ec - size_bytes: 18677 + sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + size_bytes: 15150 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: househol kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: househol.tab - sha256: 5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30 - size_bytes: 12387117 + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + size_bytes: 12108606 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: job kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: job.tab - sha256: 88b77ffe06865f029f713bb1d55ff12bdea8a1234de5bc293e72458fe64f3a74 - size_bytes: 10934873 + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + size_bytes: 10518760 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: maint kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: maint.tab - sha256: f2dc924eb5a51b0c357791693d15b431327dc39c6421011efb313d88bf839695 - size_bytes: 15440 + sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + size_bytes: 13993 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: mortgage kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: mortgage.tab - sha256: ce36b477d67837c469608a0d68f7ef269ac04758974235f1157d2f6b92cdbfdc - size_bytes: 631783 + sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + size_bytes: 600552 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: oddjob kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: oddjob.tab - sha256: b4ba3dd3151f73a01422983c60514a3e38458ddfa4fb33ae4ed0326873406305 - size_bytes: 5165 + sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + size_bytes: 5339 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: penprov kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: penprov.tab - sha256: ee001461c40306ec24b38b2881e1774121114266a2ee449d606cd0a811c37731 - size_bytes: 522313 + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + size_bytes: 513614 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: pension kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: pension.tab - sha256: 150d6fad1fce81254fb7aea1526fbb00b63d4027d6e2ac4c26bb90aea3127eb7 - size_bytes: 1225838 + sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + size_bytes: 1232411 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -157,9 +196,7 @@ stages: numeric_errors: coerce runtime_sha256_required: true - kind: replace_sentinels - sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled - to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays - unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' + sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' - kind: assemble_group_entities household_id: sernum benunit_id: sernum * 100 + benunit @@ -278,23 +315,24 @@ stages: - structural_insurance_payments - housing_service_charges - external_child_payments - notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields - remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally - absent from this stage. + notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. - stage: frs_employment - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -310,22 +348,24 @@ stages: - sic_industry_division nonnegative_outputs: - sic_industry_division - notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, - so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. + notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. - stage: frs_council_tax - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: household artifacts: - role: frs_table table: househol kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: househol.tab - sha256: 5fd26b8b675f33b3b30c9ac789a18da17de734790f77e00ded287d1c3a187b30 - size_bytes: 12387117 + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + size_bytes: 12108606 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -345,12 +385,10 @@ stages: - council_tax nonnegative_outputs: - council_tax - notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted - before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. + notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. - stage: frs_disability - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DWP parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters. grain: person artifacts: [] operations: @@ -367,31 +405,36 @@ stages: - is_disabled_for_benefits - is_enhanced_disabled_for_benefits - is_severely_disabled_for_benefits - notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped - at E10; export allowlists stay fail-closed. + notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped at E10; export allowlists stay fail-closed. - stage: frs_education - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: child kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: child.tab - sha256: b5dc84fe8b002ee925e61fae23fed27b11537af9fb174f1d07d9cc1748b9702e - size_bytes: 2913156 + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + size_bytes: 2753961 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -403,8 +446,7 @@ stages: scope: current/highest education, QYP inputs, EMA cell-mean degenerate fills, and benefits-in-own-right flag - kind: impute_cell_means cells: - - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt - for children)' + - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt for children)' donor_filter: participants with non-negative reported amounts missing: participants with sentinel negative reported amounts value: donor-mean fill, floored at zero, annualized with 365.25 / 7 @@ -422,26 +464,24 @@ stages: - adult_ema - child_ema - age_started_or_accepted_current_education_or_training - notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries - the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data - Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed - difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary - and falls to the default. EMA uses the shared weeks-in-year constant.' + notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2024-25 data dictionary (UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Codes 1-87 are empirically present at 2024-25; code 87 remains undocumented in the SN 9563 dictionary and falls to the default. EMA uses the shared weeks-in-year constant.' - stage: frs_legacy_proxies - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DWP parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters. grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: read_tables format: tab @@ -459,12 +499,10 @@ stages: - legacy_jobseeker_proxy - esa_health_condition_proxy - esa_support_group_proxy - notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine - hours scale; state_pension_age is consumed but not persisted. + notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine hours scale; state_pension_age is consumed but not persisted. - stage: frs_education_grant_split - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus - policyengine-uk DfE grant parameters. + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DfE grant parameters. grain: person artifacts: [] operations: @@ -482,11 +520,10 @@ stages: - education_grants nonnegative_outputs: - disabled_students_allowance_eligible_expenses - notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned - zero vector rather than an engine shape-sizing read. + notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted. - stage: frs_take_up - survey: Family Resources Survey 2023-24 - source: Family Resources Survey 2023-24 reported receipt anchors plus sourced UK take-up contract rates. + survey: Family Resources Survey 2024-25 + source: Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates. grain: benunit artifacts: [] operations: @@ -557,10 +594,9 @@ stages: - maximum_extended_childcare_hours_usage nonnegative_outputs: - maximum_extended_childcare_hours_usage - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. - Reported positive receipt anchors are transient and consumed only. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. - stage: frs_person_draws - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Sourced UK take-up contract rates and identity-keyed deterministic draws. grain: person artifacts: [] @@ -588,10 +624,9 @@ stages: - attends_private_school_random_draw nonnegative_outputs: - attends_private_school_random_draw - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally - not produced. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. - stage: frs_household_draws - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Sourced UK stochastic contract rates and identity-keyed deterministic draws. grain: household artifacts: [] @@ -621,10 +656,9 @@ stages: - would_evade_tv_licence_fee - main_residential_property_purchased_is_first_home - property_purchased - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent - draw, matching the incumbent reference share. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. - stage: frs_brma - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor. grain: household artifacts: @@ -649,12 +683,10 @@ stages: seed: 0 outputs: - brma - notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt - brma:household_pick. + notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. - stage: was_wealth survey: Wealth and Assets Survey round 8 - source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local - licensed 2006-22 household tab. + source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -790,18 +822,8 @@ stages: - cash_isa - stocks_and_shares_isa - student_loan_balance - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces - the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds - stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales - for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. - Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the - incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy - encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully - like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes - (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: - 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality - revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s - -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero + for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices. @@ -826,13 +848,10 @@ stages: rewrites: - main_residence_value - property_wealth - notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. - Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean - follows the incumbent behavior. + notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. - stage: lcfs_consumption survey: Living Costs and Food Survey 2023-24 - source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, - Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor. + source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -862,7 +881,7 @@ stages: kind: private_microdata format: tab vintage: '2018_20' - locator: was_round_8_hhold_eul_may_2025_230525.tab + locator: caller-supplied local input sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 size_bytes: 39073613 runtime_sha256_required: true @@ -1025,11 +1044,7 @@ stages: - electricity_consumption - gas_consumption - has_fuel_consumption - notes: Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed - seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity - at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating - is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build - year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh. + notes: Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh. - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. @@ -1082,14 +1097,10 @@ stages: outputs: - full_rate_vat_expenditure_rate nonnegative_outputs: [] - notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather - than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: - totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the - support gate is the guard - the net_financial_wealth precedent from was_wealth.' + notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth.' - stage: etb_services survey: Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table - source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost - table. + source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table. grain: household+person base_candidate: filename: populace_uk_2023.h5 @@ -1183,40 +1194,41 @@ stages: - nhs_a_and_e_spending - nhs_admitted_patient_spending - nhs_outpatient_spending - notes: Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS - visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves - to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides - with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this - vintage. + notes: Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this vintage. - stage: frs_hmrc_spine_leaves - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied - local input + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input grain: person artifacts: - role: frs_table table: adult kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: adult.tab - sha256: e09f9647d03585c81a528636028b2ed495f8f1fbcf64c5e7b4fe521b67367e06 - size_bytes: 35323384 + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 - role: frs_table table: benefits kind: licensed_microdata format: tab - vintage: '2023_24' + vintage: '2024_25' locator: benefits.tab - sha256: ff30d054cc659bcf23b44c492d98cfd701c0bfdb63e8e9aa9769b490ba9d636b - size_bytes: 4460292 + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + size_bytes: 2362329 runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 operations: - kind: retain_adjudicated_frs_hmrc_leaves population: uk_frs_raw_spine - source_vintage: 2023-24 - mapped_build_period: 2023 + source_vintage: 2024-25 + mapped_build_period: 2024 annualization: weekly raw FRS amounts * (365.25 / 7) status: adjudicated_partial_replay retained_full_constituents: @@ -1291,10 +1303,9 @@ stages: - ossben_identifiable_subset - srp_regular_code5 - employer_pension_contributions - notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, - then ports the incumbent employer-pension-contributions estimate. + notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then ports the incumbent employer-pension-contributions estimate. - stage: spi_support_channel - survey: Family Resources Survey 2023-24 + survey: Family Resources Survey 2024-25 source: Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning. grain: household artifacts: [] @@ -1344,8 +1355,7 @@ stages: - household_source_id - source_household_id - source_year - notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, - and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. + notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -1380,8 +1390,8 @@ stages: sheets: - Table_3_6 - Table_3_7 - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year runtime_sha256_required: true operations: - kind: verify_pinned_hmrc_source_pair @@ -1511,15 +1521,11 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine - gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; - it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM - + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates - and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -1528,8 +1534,7 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, - PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -1541,9 +1546,7 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income - fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain - exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -1571,9 +1574,7 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only - stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified - Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -1610,10 +1611,8 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not - a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training - source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -1639,8 +1638,8 @@ stages: rows: base_support_channel - kind: materialize_hmrc_income_bands_fail_closed artifact_role: published_fact_surface - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year column_index_base: 0 data_row_start_index: 5 stop_label: All ranges @@ -1723,9 +1722,7 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize - full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound - is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -1737,12 +1734,9 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce - complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this - is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an - imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -1753,12 +1747,9 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense - amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the - true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity - with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -1767,8 +1758,7 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable - mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -1779,8 +1769,7 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI - miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -1793,10 +1782,8 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be - split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate - FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -1808,10 +1795,8 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax - treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; - it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -1821,12 +1806,9 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination - of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; - it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published - state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -1838,12 +1820,9 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income - band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable - like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into - or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -1936,13 +1915,10 @@ stages: - hmrc_spi_pay - hmrc_spi_unemployment_benefit_income - hmrc_spi_incapacity_benefit_income - notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains - FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine - path. + notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path. - stage: frs_hmrc_retained_leaves - survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied - local input + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input grain: person artifacts: [] operations: @@ -1952,8 +1928,8 @@ stages: fail_on_mismatch: true - kind: retain_adjudicated_frs_hmrc_leaves population: certified_microcosm_uk_candidate_base_channel - source_vintage: 2023-24 - mapped_build_period: 2023 + source_vintage: 2024-25 + mapped_build_period: 2024 annualization: weekly raw FRS amounts * (365.25 / 7) status: adjudicated_partial_replay retained_full_constituents: @@ -2016,9 +1992,7 @@ stages: - hmrc_spi_incapacity_benefit_income - ossben_identifiable_subset - srp_regular_code5 - notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and - INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these - leaves.' + notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves.' - stage: hmrc_spi_income survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -2053,8 +2027,8 @@ stages: sheets: - Table_3_6 - Table_3_7 - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year runtime_sha256_required: true operations: - kind: verify_pinned_hmrc_source_pair @@ -2075,8 +2049,7 @@ stages: output_weight_kind: importance preserve_total_household_mass: true require_mass_change_record: true - mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; - total national mass is conserved. + mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved. fail_on_live_existing_spi_mass: true - kind: strict_read_private_table artifact_role: qrf_donor @@ -2197,15 +2170,11 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine - gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; - it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM - + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates - and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -2214,8 +2183,7 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, - PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -2227,9 +2195,7 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income - fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain - exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -2252,9 +2218,7 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only - stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified - Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -2291,10 +2255,8 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not - a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training - source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -2313,8 +2275,8 @@ stages: require_all_outputs: false - kind: materialize_hmrc_income_bands_fail_closed artifact_role: published_fact_surface - mapped_build_period: 2023 - period_mapping: tax_year_start + mapped_build_period: 2024 + period_mapping: latest_published_tax_year column_index_base: 0 data_row_start_index: 5 stop_label: All ranges @@ -2397,9 +2359,7 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize - full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound - is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -2411,12 +2371,9 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce - complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this - is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an - imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -2427,12 +2384,9 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense - amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the - true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity - with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -2441,8 +2395,7 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable - mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -2453,8 +2406,7 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI - miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -2467,10 +2419,8 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be - split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate - FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -2482,10 +2432,8 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax - treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; - it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -2495,12 +2443,9 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination - of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; - it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published - state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -2512,12 +2457,9 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income - band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable - like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into - or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -2569,15 +2511,5 @@ stages: - hmrc_spi_total_earned_income - hmrc_spi_total_investment_income - hmrc_spi_assessable_income - notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned - by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed - income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. - PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent - UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, - private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, - but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful - full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, - MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, - none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence - record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt - positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' + notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, + and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' diff --git a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json index 85cad6b335..72c6f4d685 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json +++ b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json @@ -1,8 +1,8 @@ { "version": 1, "country": "uk", - "build_year": 2023, - "policy": "UK stochastic assignment contract for the FRS 2023-24 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", + "build_year": 2024, + "policy": "UK stochastic assignment contract for the FRS 2024-25 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", "incumbent_provenance": { "commit": "ebf733c", "description": "incumbent UK pipeline stochastic layer", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py new file mode 100644 index 0000000000..c8472abe3b --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_release.py @@ -0,0 +1,112 @@ +"""Frozen UK FRS raw-vintage release loader.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass +from functools import lru_cache +from importlib.resources import files +from typing import Any + +__all__ = ["UKFRSRelease", "load_uk_frs_release"] + +_LOWER_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_VINTAGE = re.compile(r"^\d{4}_\d{2}$") + + +@dataclass(frozen=True) +class UKFRSRelease: + """Checked-in identity for the UK raw FRS release used by the build.""" + + version: int + country: str + policy: str + name: str + survey_year: int + base_year: int + calibration_year: int + time_period: str + vintage: str + ukds_study_number: int + doi: str + ukds_tab_zip: Mapping[str, Any] + acquisition: Mapping[str, Any] + notes: str + + +def _release_path(): + return files("microcosm.build.uk").joinpath("frs_release.json") + + +@lru_cache(maxsize=1) +def load_uk_frs_release() -> UKFRSRelease: + """Load and validate ``build/uk/frs_release.json``.""" + + raw = json.loads(_release_path().read_text(encoding="utf-8")) + if not isinstance(raw, Mapping): + raise ValueError("UK FRS release must contain a JSON object.") + release = UKFRSRelease( + version=_int(raw, "version"), + country=_str(raw, "country"), + policy=_str(raw, "policy"), + name=_str(raw, "name"), + survey_year=_int(raw, "survey_year"), + base_year=_int(raw, "base_year"), + calibration_year=_int(raw, "calibration_year"), + time_period=_str(raw, "time_period"), + vintage=_str(raw, "vintage"), + ukds_study_number=_int(raw, "ukds_study_number"), + doi=_str(raw, "doi"), + ukds_tab_zip=_mapping(raw, "ukds_tab_zip"), + acquisition=_mapping(raw, "acquisition"), + notes=_str(raw, "notes"), + ) + _validate(release) + return release + + +def _int(raw: Mapping[str, Any], key: str) -> int: + value = raw.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"UK FRS release requires integer {key}.") + return value + + +def _str(raw: Mapping[str, Any], key: str) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"UK FRS release requires non-empty string {key}.") + return value + + +def _mapping(raw: Mapping[str, Any], key: str) -> Mapping[str, Any]: + value = raw.get(key) + if not isinstance(value, Mapping): + raise ValueError(f"UK FRS release requires object {key}.") + return dict(value) + + +def _validate(release: UKFRSRelease) -> None: + if release.country != "uk": + raise ValueError("UK FRS release country must be 'uk'.") + if not (release.survey_year <= release.base_year < release.calibration_year): + raise ValueError( + "UK FRS release years must satisfy survey_year <= base_year < " + "calibration_year." + ) + if release.time_period != str(release.base_year): + raise ValueError("UK FRS release time_period must equal base_year.") + if not _VINTAGE.fullmatch(release.vintage): + raise ValueError("UK FRS release vintage must match YYYY_YY.") + if not release.vintage.startswith(str(release.survey_year)): + raise ValueError("UK FRS release vintage must start with survey_year.") + _validate_sha256(release.ukds_tab_zip, "ukds_tab_zip.sha256", "sha256") + _validate_sha256(release.acquisition, "acquisition.zip_sha256", "zip_sha256") + + +def _validate_sha256(raw: Mapping[str, Any], label: str, key: str) -> None: + value = raw.get(key) + if not isinstance(value, str) or not _LOWER_SHA256.fullmatch(value): + raise ValueError(f"UK FRS release {label} must be 64 lowercase hex chars.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 1ac77a432a..323feea41b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -55,7 +55,7 @@ ) WEEKS_IN_YEAR = 365.25 / 7 -TIME_PERIOD = "2023" +TIME_PERIOD = "2024" # FRS GVTREGNO uses skip-3 coding: code 3 (the retired Merseyside code) is # absent from the domain, so the real codes are [1, 2, 4..13] with diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py index 75f4f21c10..93785b56c1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_capital_gains.py @@ -69,7 +69,7 @@ HMRC_CGT_JOINT_ODS_SIZE_BYTES = 11_996 HMRC_CGT_SOURCE_LABEL = "HMRC capital gains table 3" HMRC_CGT_SOURCE_VINTAGE = "2023-24" -HMRC_CGT_BUILD_PERIOD = "2023" +HMRC_CGT_BUILD_PERIOD = "2024" #: Worksheet per tax year, newest first. The publication carries four years, #: so a stage can fit each year rather than aging one forward. @@ -325,7 +325,9 @@ def materialize_hmrc_capital_gains_joint_distribution( Args: path: Local copy of the published ODS. tax_year: Which published year to read, as ``"2023-24"``. - build_period: The build period these facts are declared against. + build_period: The build period these facts are declared against. The + latest published tax year 2023-24 is replayed against build period + 2024 (signed, microcosm#723). verify_fingerprint: Whether to require the pinned hash and size. Only a caller reading a different vintage should turn this off, and it then owns checking what it read. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py index c3b6236ea1..f6e9f58037 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_income.py @@ -59,7 +59,7 @@ HMRC_SPI_SOURCE_VINTAGE = "2023-24" HMRC_SPI_SOURCE_TAX_YEAR = HMRC_SPI_SOURCE_VINTAGE HMRC_SPI_SOURCE_TAX_YEAR_START = 2023 -HMRC_SPI_BUILD_PERIOD = "2023" +HMRC_SPI_BUILD_PERIOD = "2024" HMRC_SPI_ASSESSABLE_INCOME_COLUMN = "hmrc_spi_assessable_income" HMRC_SPI_INCOME_BAND_LOWER_BOUNDS = ( @@ -357,16 +357,16 @@ def materialize_hmrc_spi_income_band_targets( ) -> HMRCIncomeTargetSet: """Parse the complete 2023-24 HMRC SPI income-band target surface. - The source tax year is mapped by its tax-year start, so 2023-24 is valid - only for the Microcosm build period ``"2023"``. Any missing, duplicate, or - malformed source record raises; this function never narrows the family. + The latest published tax year 2023-24 is replayed against build period + ``"2024"`` (signed, microcosm#723). Any missing, duplicate, or malformed + source record raises; this function never narrows the family. """ period = str(build_period) if period != HMRC_SPI_BUILD_PERIOD: raise ValueError( - f"HMRC SPI source tax year {HMRC_SPI_SOURCE_TAX_YEAR} starts in " - f"{HMRC_SPI_SOURCE_TAX_YEAR_START} and maps to build period " + f"HMRC SPI source tax year {HMRC_SPI_SOURCE_TAX_YEAR} is the " + "latest published tax year 2023-24 replayed against build period " f"{HMRC_SPI_BUILD_PERIOD!r}; got {period!r}." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py index d8d715b9c0..b4f78b30df 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_source_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import json from collections.abc import Mapping, Sequence from importlib.resources import files @@ -69,6 +70,7 @@ ] UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE = "hmrc_income_source_stages.json" +UK_CANONICAL_SOURCE_STAGES_RESOURCE = "source_stages.json" HMRC_DISTRIBUTIONAL_INPUTS = ( "gift_aid", "charitable_investment_gifts", @@ -90,9 +92,10 @@ _STAGE2_REVIEWED_ABSENT_PREDICTORS = { "other_investment_income": ( "This remains a stage-1 SPI draw and an official HMRC fact component, " - "but it is not an FRS-only stage-2 predictor: policyengine-uk-data " - "frs_only.py defines exactly six income predictors and the certified " - "Microcosm UK base candidate has no other_investment_income column." + "but it is not an FRS-only stage-2 predictor: the incumbent UK data " + "build's frs_only.py defines exactly six income predictors and the " + "certified Microcosm UK base candidate has no other_investment_income " + "column." ) } @@ -239,7 +242,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "published_fact_surface.period_mapping", surface.get("period_mapping"), - "tax_year_start", + "latest_published_tax_year", ) _expect( failures, @@ -300,7 +303,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "frs_leaves.source_vintage", frs_leaves.get("source_vintage"), - "2023-24", + "2024-25", ) _expect( failures, @@ -708,7 +711,7 @@ def assert_uk_hmrc_income_source_contract_current( failures, "materialize.period_mapping", materialize.get("period_mapping"), - "tax_year_start", + "latest_published_tax_year", ) _expect( failures, @@ -905,7 +908,12 @@ def assert_uk_hmrc_income_source_contract_current( "stage.outputs", tuple(stage.get("outputs", ())), ( - *HMRC_SPI_INCOME_COMPONENTS, + *( + "hmrc_spi_state_pension_income" + if component == "state_pension" + else component + for component in HMRC_SPI_INCOME_COMPONENTS + ), *HMRC_DISTRIBUTIONAL_INPUTS, *SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, ), @@ -971,11 +979,63 @@ def uk_hmrc_weighted_qrf_output_columns( def _load_payload(resource: Any | None) -> Mapping[str, Any]: - target = ( - files("microcosm.build.uk").joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) - if resource is None - else resource - ) + if resource is None: + frozen_payload = json.loads( + files("microcosm.build.uk") + .joinpath(UK_HMRC_INCOME_SOURCE_STAGES_RESOURCE) + .read_text(encoding="utf-8") + ) + if not isinstance(frozen_payload, Mapping): + raise ValueError("UK HMRC source manifest root must be a JSON object.") + frozen_stages = frozen_payload.get("stages") + if not isinstance(frozen_stages, Sequence) or isinstance( + frozen_stages, (str, bytes) + ): + raise ValueError("UK HMRC source manifest stages must be a list.") + if len(frozen_stages) != 1 or not isinstance(frozen_stages[0], Mapping): + raise ValueError( + "UK HMRC source manifest must contain exactly one source stage." + ) + payload = json.loads( + files("microcosm.build.uk") + .joinpath(UK_CANONICAL_SOURCE_STAGES_RESOURCE) + .read_text(encoding="utf-8") + ) + if not isinstance(payload, Mapping): + raise ValueError("UK source manifest root must be a JSON object.") + stages = payload.get("stages") + if not isinstance(stages, Sequence) or isinstance(stages, (str, bytes)): + raise ValueError("UK source manifest stages must be a list.") + retained = [ + stage + for stage in stages + if isinstance(stage, Mapping) + and stage.get("stage") == "frs_hmrc_retained_leaves" + ] + hmrc = [ + stage + for stage in stages + if isinstance(stage, Mapping) and stage.get("stage") == "hmrc_spi_income" + ] + if len(retained) != 1 or len(hmrc) != 1: + raise ValueError( + "UK source manifest must contain exactly one " + "frs_hmrc_retained_leaves stage and one hmrc_spi_income stage." + ) + stage = copy.deepcopy(dict(hmrc[0])) + stage["base_candidate"] = copy.deepcopy( + dict(frozen_stages[0].get("base_candidate", {})) + ) + stage["operations"] = [ + *copy.deepcopy(list(retained[0].get("operations", ()))), + *copy.deepcopy(list(hmrc[0].get("operations", ()))), + ] + return { + "country": payload.get("country"), + "version": payload.get("version"), + "stages": [stage], + } + target = resource if hasattr(target, "read_text"): raw = target.read_text(encoding="utf-8") else: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 57b79833f0..6acb9a3bbe 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -250,12 +250,22 @@ def evidence(self) -> dict[str, object]: class UKFRSHMRCSpineLeavesStageTransform: frs_raw_dir: Path stage: SourceStageSpec + # A #627 scale-ladder receipt build: the spine was subsampled after + # frs_spine, so raw-tab person coverage legitimately exceeds the frame. + sampled_rung: bool = False # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKFRSHMRCSpineLeavesResult | None = field(default=None, init=False) - def __init__(self, frs_raw_dir: str | Path, *, stage: SourceStageSpec) -> None: + def __init__( + self, + frs_raw_dir: str | Path, + *, + stage: SourceStageSpec, + sampled_rung: bool = False, + ) -> None: object.__setattr__(self, "frs_raw_dir", Path(frs_raw_dir)) object.__setattr__(self, "stage", stage) + object.__setattr__(self, "sampled_rung", bool(sampled_rung)) object.__setattr__(self, "last_result", None) def __call__(self, frame: Frame) -> Frame: @@ -278,11 +288,18 @@ def __call__(self, frame: Frame) -> Frame: source_leaves = _materialize_source_leaves(adult, benefits) person = frame.table("person").copy() missing_ids = sorted(set(source_leaves.index) - set(person["person_id"])) - if missing_ids: + if missing_ids and not self.sampled_rung: raise ValueError( "Raw FRS retained leaves contain person identity value(s) absent " f"from the raw spine: {missing_ids[:5]}." ) + if missing_ids: + # A rung sample deliberately drops most source people; restrict + # the raw surface to the survivors (the full-scale fence above + # stays strict — mirrors frs_hmrc_leaves' sampled_rung posture). + source_leaves = source_leaves.loc[ + source_leaves.index.isin(person["person_id"].to_numpy()) + ] aligned = source_leaves.reindex(person["person_id"], fill_value=0.0) for column in FRS_HMRC_RETAINED_LEAF_COLUMNS: person[column] = aligned[column].to_numpy(dtype=float) @@ -339,6 +356,10 @@ def checkpoint_metadata(self) -> dict[str, object]: class UKSPISupportChannelStageTransform: stage: SourceStageSpec seed: int = 42 + # A #627 scale-ladder receipt build: scale the declared synthetic stack + # by the survey-side sample fraction so the prior-mass pairing holds at + # every rung (f100 keeps the declared count exactly). + sample_fraction: float = 1.0 # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPISupportResult | None = field(default=None, init=False) @@ -349,6 +370,8 @@ def __call__(self, frame: Frame) -> Frame: self.stage, seed=self.seed, ) + if self.sample_fraction != 1.0 and count is not None: + count = max(1, int(round(count * self.sample_fraction))) result = build_uk_spi_support_channel( tables["person"], tables["benunit"], @@ -416,6 +439,10 @@ class UKSPIIncomeSpineStageTransform: seed: int = 42 qrf_estimators: int = 100 donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE + # A #627 scale-ladder receipt build: a sampled stack cannot carry mass in + # ultra-sparse restored columns, so the effective-mass fence records the + # gap instead of raising (f100 keeps the strict fence). + sampled_rung: bool = False # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPIIncomeSpineResult | None = field(default=None, init=False) @@ -428,6 +455,7 @@ def __init__( seed: int = 42, qrf_estimators: int = 100, donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE, + sampled_rung: bool = False, ) -> None: object.__setattr__(self, "spi_tab_path", Path(spi_tab_path)) object.__setattr__(self, "hmrc_ods_path", Path(hmrc_ods_path)) @@ -435,6 +463,7 @@ def __init__( object.__setattr__(self, "seed", seed) object.__setattr__(self, "qrf_estimators", qrf_estimators) object.__setattr__(self, "donor_sample_size", donor_sample_size) + object.__setattr__(self, "sampled_rung", bool(sampled_rung)) object.__setattr__(self, "last_result", None) @property @@ -492,11 +521,16 @@ def __call__(self, frame: Frame) -> Frame: for name, share in distributional_mass_shares.items() if share < DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE } - if insufficient: + if insufficient and not self.sampled_rung: raise RuntimeError( "Rebuilt SPI spine channel did not restore required " f"effective-mass coverage: {insufficient}." ) + # On a sampled rung the scaled stack cannot carry mass in + # ultra-sparse restored columns (e.g. charitable_investment_gifts at + # ~0.02 percent density); the gap is receipted via the replay + # report's distributional_mass_shares — the full-scale fence above + # stays strict. report = _build_spine_replay_report( source_targets=source_targets, support=support, diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 943655020d..f6a434bc2d 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -281,6 +281,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "degenerate_reviewed_exclusions.json", "efrs_parity_known_gaps.json", "efrs_parity_reference.json", + "frs_release.json", "gates.json", "brma_rent_counts.json", "hmrc_income_release_gate_report.json", @@ -341,6 +342,7 @@ def test_uk_package_loads(self) -> None: "degenerate_reviewed_exclusions.json", "efrs_parity_known_gaps.json", "efrs_parity_reference.json", + "frs_release.json", "gates.json", "brma_rent_counts.json", "hmrc_income_release_gate_report.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index b60dccd8bb..bfd22497ef 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "942ad7df5537e244df7ede7230098e91c617dbdb11e4d0885a94bb3b4f912da5", + "54d6c1aa1e79c016081a645c0aaad86ba3fcc8568665804fbe46dfc5a08eb450", { "benunit.benunit_id", "household.household_id", @@ -118,8 +118,7 @@ def test_country_kernel_contract_ids_are_closed_in_the_compiler_registry() -> No assert country_contract_ids <= F0_KERNEL_REGISTRY.ids assert all(F0_KERNEL_REGISTRY.contains(value) for value in country_contract_ids) assert not any( - F0_KERNEL_REGISTRY.has_implementation(value) - for value in country_contract_ids + F0_KERNEL_REGISTRY.has_implementation(value) for value in country_contract_ids ) diff --git a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py index 1b9a86a151..d0d6f281c3 100644 --- a/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_cgt_source_manifest.py @@ -121,7 +121,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: ) compliant = SimpleNamespace( household_weight_kind=WeightKind.IMPORTANCE, - time_period="2023", + time_period="2024", mass_log=( MassChangeRecord( entity="household", @@ -152,7 +152,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: missing_receipt = SimpleNamespace( household_weight_kind=WeightKind.IMPORTANCE, - time_period="2023", + time_period="2024", mass_log=compliant.mass_log[:1], ) _, failures = _family_build_state_diagnostics(missing_receipt, manifest) @@ -160,7 +160,7 @@ def test_the_shipped_family_contracts_pass_the_terminal_gate_shape() -> None: wrong_kind = SimpleNamespace( household_weight_kind=WeightKind.DESIGN, - time_period="2023", + time_period="2024", mass_log=compliant.mass_log, ) _, failures = _family_build_state_diagnostics(wrong_kind, manifest) diff --git a/packages/microcosm-build/tests/test_uk_frs_release.py b/packages/microcosm-build/tests/test_uk_frs_release.py new file mode 100644 index 0000000000..48cfda58a2 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_frs_release.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import copy +import json +from importlib.resources import files + +import pytest + +from microcosm.build.source_manifest import SourceManifest +from microcosm.build.uk_runtime import frs_spine +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract + + +def _source_manifest() -> SourceManifest: + raw = json.loads( + files("microcosm.build.uk").joinpath("source_stages.json").read_text() + ) + return SourceManifest.from_mapping(raw) + + +def _reload_with(monkeypatch, mutated: dict) -> None: + load_uk_frs_release.cache_clear() + payload = json.dumps(mutated) + + class _FakePath: + def read_text(self, *args, **kwargs): + return payload + + monkeypatch.setattr( + "microcosm.build.uk_runtime.frs_release._release_path", + lambda: _FakePath(), + ) + + +def test_uk_frs_release_loads_and_controls_runtime_period() -> None: + release = load_uk_frs_release() + + assert release.country == "uk" + assert release.name == "frs_2024_25" + assert release.survey_year == 2024 + assert release.base_year == 2024 + assert release.calibration_year == 2025 + assert frs_spine.TIME_PERIOD == release.time_period == str(release.base_year) + assert release.vintage == "2024_25" + assert release.ukds_study_number == 9563 + assert release.doi == "10.5255/UKDA-SN-9563-1" + + +def test_release_lockstep_with_source_manifest_and_take_up_contract() -> None: + release = load_uk_frs_release() + manifest = _source_manifest() + + for stage in manifest.stages: + for artifact in stage.artifacts: + if artifact.get("role") != "frs_table": + continue + assert artifact["vintage"] == release.vintage + assert artifact["ukds_study_number"] == release.ukds_study_number + assert artifact["doi"] == release.doi + assert artifact["tax_year_start"] == release.base_year + assert load_uk_take_up_contract().build_year == release.base_year + assert "SN 9563" in manifest.stage_map()["frs_spine"].source + + +def test_release_validation_refuses_bad_years_and_hashes(monkeypatch) -> None: + raw = json.loads( + files("microcosm.build.uk").joinpath("frs_release.json").read_text() + ) + + mutated = copy.deepcopy(raw) + mutated["time_period"] = "2023" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="time_period"): + load_uk_frs_release() + + mutated = copy.deepcopy(raw) + mutated["ukds_tab_zip"]["sha256"] = "ABC" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="lowercase hex"): + load_uk_frs_release() + + mutated = copy.deepcopy(raw) + mutated["vintage"] = "2023_24" + _reload_with(monkeypatch, mutated) + with pytest.raises(ValueError, match="survey_year"): + load_uk_frs_release() diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index e01ed24a2a..822414392c 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -323,7 +323,7 @@ def _write_fixture( "table": table, "kind": "licensed_microdata", "format": "tab", - "vintage": "2023_24", + "vintage": "2024_25", "locator": path.name, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), "size_bytes": path.stat().st_size, @@ -761,13 +761,26 @@ def test_manifest_stage_and_runtime_agree_on_artifacts_and_operations() -> None: assert set(stage.outputs) == set(UKFRSSpineStageTransform.output_columns()) +def test_full_roster_artifact_pins_are_coherent() -> None: + # Regression armor for the #723 licensed-run finding: the WAS round-8 tab + # was declared by two stages (was_wealth, lcfs_consumption) with different + # locator strings, so `_artifact_pins()` refused the full-roster driver + # run — a break only a licensed build could see. Collecting pins over the + # complete manifest keeps the same-artifact-same-pin invariant CI-visible. + tool = _load_tool() + spec = load_country_spec("uk") + stages = list(spec.sources.stage_map().values()) + pins = tool._artifact_pins(stages) + assert pins, "full-roster pin collection must not be empty" + + def test_builds_structural_frame_from_pinned_tabs(tmp_path: Path) -> None: stage = _write_fixture(tmp_path) frame = build_uk_frs_spine_frame(tmp_path, stage=stage) validate_uk_national_frame(frame) - assert uk_time_period(frame) == "2023" + assert uk_time_period(frame) == "2024" assert frame.weights_for("household").kind is WeightKind.DESIGN np.testing.assert_array_equal( frame.weights_for("household").values, @@ -1058,7 +1071,7 @@ def _fake_value(column: str, rows: int): if column.endswith("_support_channel"): return ["frs"] * rows if column == "source_household_key": - return [f"2023:{index + 1}" for index in range(rows)] + return [f"2024:{index + 1}" for index in range(rows)] if column == "household_is_spi_synthetic": return [False] * rows return np.arange(1, rows + 1, dtype=float) @@ -1163,6 +1176,8 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( "stage1": 42, "stage2": 43, } + assert sidecar["source_vintages"] == {"frs": "2024_25"} + assert sidecar["sampling"] is None replay_bytes = output.with_suffix(".hmrc_replay.json").read_bytes() assert json.loads(replay_bytes) == {"report_kind": "fake_spine_replay"} # The synthetic spec declares no non-table pinned artifacts, so the pin @@ -1352,7 +1367,7 @@ def test_driver_refuses_misnamed_hmrc_ods(tmp_path: Path) -> None: tool._validate_args(args) -def test_driver_has_no_sample_fraction_path(tmp_path: Path) -> None: +def test_driver_rejects_non_rung_sample_fraction(tmp_path: Path) -> None: tool = _load_tool() with pytest.raises(SystemExit): @@ -1372,6 +1387,144 @@ def test_driver_has_no_sample_fraction_path(tmp_path: Path) -> None: ) +def test_driver_derives_rung_tokens_from_sample_fraction() -> None: + tool = _load_tool() + + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("0.01")] == "f001" + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("0.10")] == "f010" + assert tool.UK_SAMPLE_RUNG_TOKENS[tool._rung_sample_fraction("1.0")] == "f100" + + +def test_driver_refuses_checkpoint_dir_on_sampled_rung(tmp_path: Path) -> None: + tool = _load_tool() + + with pytest.raises(SystemExit): + tool._parse_args( + [ + "--frs-raw-dir", + str(tmp_path), + "--spine-h5", + str(tmp_path / "spine.h5"), + "--spi-tab", + str(tmp_path / "put2223uk.tab"), + "--hmrc-ods", + str(tmp_path / "hmrc.ods"), + "--sample-fraction", + "0.10", + "--checkpoint-dir", + str(tmp_path / "checkpoints"), + ] + ) + + +def test_driver_records_sampled_spine_sidecar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + pytest.importorskip("tables") + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "spine.h5" + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + monkeypatch.setattr( + tool, + "sample_frame_households", + lambda frame, **kwargs: ( + frame, + {"eligible_households": len(frame.table("household"))}, + ), + ) + monkeypatch.setattr( + tool, + "normalize_sampled_household_mass", + lambda frame, **kwargs: (frame, 1.0), + ) + + assert ( + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--sample-fraction", + "0.10", + "--sample-seed", + "999", + ] + ) + == 0 + ) + + sidecar = json.loads(output.with_suffix(".build.json").read_text()) + assert sidecar["sampling"] == { + "fraction": 0.1, + "seed": 999, + "rung_token": "f010", + "pre_household_count": 2, + "post_household_count": 2, + "normalization_factor": 1.0, + "receipt": {"eligible_households": 2}, + } + rows = load_spool_rows(tmp_path / "logbook-spool") + assert rows[0].rung == "f010" + + +def test_driver_sampled_named_edge_aborts_with_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + output = tmp_path / "spine.h5" + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + + def _raise_named_edge(*args, **kwargs): + raise ValueError(tool._RUNG_NAMED_EDGE_SIGNATURE) + + monkeypatch.setattr(tool, "sample_frame_households", _raise_named_edge) + + assert ( + tool.main( + [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--sample-fraction", + "0.10", + ] + ) + == tool._RUNG_ABORT_EXIT_CODE + ) + + receipt = json.loads(output.with_suffix(".rung_abort.json").read_text()) + assert receipt["named_edge"] == "spine_split_singleton_class" + assert receipt["sampling"]["rung_token"] == "f010" + assert not output.exists() + rows = load_spool_rows(tmp_path / "logbook-spool") + assert rows[0].disposition == "discarded" + assert rows[0].rung == "f010" + + def test_refuses_missing_tab(tmp_path: Path) -> None: stage = _write_fixture(tmp_path) (tmp_path / "adult.tab").unlink() diff --git a/packages/microcosm-build/tests/test_uk_hmrc_income.py b/packages/microcosm-build/tests/test_uk_hmrc_income.py index aaa151d3e1..eeeaa68079 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_income.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_income.py @@ -189,10 +189,10 @@ def should_not_parse(_path: Path): monkeypatch.setattr(hmrc_income, "_read_ods_tables", should_not_parse) - with pytest.raises(ValueError, match="maps to build period '2023'"): + with pytest.raises(ValueError, match="replayed against build period '2024'"): materialize_hmrc_spi_income_band_targets( ods_path, - build_period="2024", + build_period="2023", ) assert called is False diff --git a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py index c659a642ca..92a4009a32 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_income_source_manifest.py @@ -19,6 +19,14 @@ / "uk" / "hmrc_income_source_stages.json" ) +_CANONICAL_SOURCE_STAGES_PATH = ( + Path(__file__).resolve().parents[1] + / "src" + / "microcosm" + / "build" + / "uk" + / "source_stages.json" +) _COLLATED_ODS_URL = ( "https://assets.publishing.service.gov.uk/media/" "69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods" @@ -105,6 +113,29 @@ def _manifest() -> dict[str, object]: return json.loads(_MANIFEST_PATH.read_text(encoding="utf-8")) +def _runtime_manifest() -> dict[str, object]: + payload = json.loads(_CANONICAL_SOURCE_STAGES_PATH.read_text(encoding="utf-8")) + retained = [ + stage + for stage in payload["stages"] + if stage["stage"] == "frs_hmrc_retained_leaves" + ] + hmrc = [stage for stage in payload["stages"] if stage["stage"] == "hmrc_spi_income"] + assert len(retained) == 1 + assert len(hmrc) == 1 + stage = dict(hmrc[0]) + stage["base_candidate"] = dict(_manifest()["stages"][0]["base_candidate"]) + stage["operations"] = [ + *retained[0]["operations"], + *hmrc[0]["operations"], + ] + return { + "country": payload["country"], + "version": payload["version"], + "stages": [stage], + } + + def _stage(payload: dict[str, object]) -> dict[str, object]: stages = payload["stages"] assert isinstance(stages, list) @@ -638,7 +669,7 @@ def test_runtime_source_contract_rejects_manifest_drift( replacement, match, ) -> None: - payload = _manifest() + payload = _runtime_manifest() cursor = payload for segment in path[:-1]: cursor = cursor[segment] @@ -658,7 +689,7 @@ def test_runtime_source_contract_rejects_duplicate_keys( collection, key, ) -> None: - payload = _manifest() + payload = _runtime_manifest() values = payload["stages"][0][collection] values.append(dict(values[0])) tampered = tmp_path / f"duplicate_{key}.json" diff --git a/packages/microcosm-build/tests/test_uk_hmrc_replay.py b/packages/microcosm-build/tests/test_uk_hmrc_replay.py index 04569c8413..45e9869a14 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_replay.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_replay.py @@ -98,7 +98,7 @@ def _evidence() -> dict[str, dict[str, object]]: "hmrc_surface": {"sha256": "d" * 64, "size_bytes": 166_693}, }, "build_evidence": { - "period": "2023", + "period": HMRC_SPI_BUILD_PERIOD, "seed": 42, "spi_prior_mass_share": 0.5, }, @@ -255,7 +255,7 @@ def test_future_exact_and_directional_fact_invariants() -> None: component="employment_income", measure="amount", unit="GBP", - period="2023", + period=HMRC_SPI_BUILD_PERIOD, total_income_lower_bound=12_570, total_income_upper_bound=15_000, published_value=100.0, diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index e9c1011104..1da1aa7ab5 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -251,7 +251,7 @@ def flags(true_count: int) -> list[bool]: "child_benefit_opts_out": flags(23), "would_claim_pc": flags(70), "would_claim_uc": flags(55), - "would_claim_tfc": flags(59), + "would_claim_tfc": flags(88), "would_claim_extended_childcare": flags(81), "would_claim_universal_childcare": flags(56), "would_claim_targeted_childcare": flags(60), @@ -279,7 +279,7 @@ def flags(true_count: int) -> list[bool]: index % 4 >= 2 for index in range(n) ], "household_owns_tv": flags(95), - "would_evade_tv_licence_fee": flags(11), + "would_evade_tv_licence_fee": flags(13), "main_residential_property_purchased_is_first_home": flags(38), "property_purchased": flags(4), "brma": [ diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index a151d80fdc..8eef5ec377 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -145,6 +145,13 @@ def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( expected_operations[6]["reviewed_absent_predictors"][ "other_investment_income" ] = _rephrase_stage2_predictor_note(predictor_note) + # FRS retained leaves now come from the FRS 2024-25 spine while the + # frozen HMRC fact surface stays byte-pinned. + expected_operations[1]["source_vintage"] = "2024-25" + expected_operations[1]["mapped_build_period"] = 2024 + # Signed period re-map (#723) for materialized HMRC SPI facts. + expected_operations[7]["mapped_build_period"] = 2024 + expected_operations[7]["period_mapping"] = "latest_published_tax_year" assert stage1["operations"] + stage2["operations"] == expected_operations _assert_no_forbidden_dependency( @@ -155,6 +162,11 @@ def test_copy_is_lockstep_with_frozen_original_except_citation_rewrites( expected_artifacts = copy.deepcopy(frozen_stage["artifacts"]) expected_artifacts[0]["reviewed_source"] = _expected_reviewed_source() + # Signed period re-map (#723): the ODS source surface remains the + # frozen 2023-24 file, but the canonical manifest declares that it is + # replayed against build period 2024. + expected_artifacts[1]["mapped_build_period"] = 2024 + expected_artifacts[1]["period_mapping"] = "latest_published_tax_year" # Declared output-name correction (licensed-data acceptance finding): # the frozen original listed the SPI concept "state_pension", but the # stage writes the auxiliary column SPI_HMRC_STATE_PENSION_INCOME_COLUMN @@ -611,9 +623,7 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: ) assert ( tuple( - lcfs_ops["materialize_rules_engine_predictors"].parameters[ - "predictors" - ] + lcfs_ops["materialize_rules_engine_predictors"].parameters["predictors"] ) == UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS ) @@ -638,14 +648,9 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: tuple(stages["etb_services"].operations[1].parameters["predictors"]) == UK_ETB_SERVICES_ENGINE_VARIABLES ) - assert ( - set( - stages["etb_services"] - .operations[1] - .parameters["derived_predictors"] - ) - == set(UK_ETB_SERVICES_EDUCATION_COUNTS) - ) + assert set( + stages["etb_services"].operations[1].parameters["derived_predictors"] + ) == set(UK_ETB_SERVICES_EDUCATION_COUNTS) assert ( tuple(stages["etb_services"].operations[2].parameters["targets"]) == UK_ETB_SERVICES_OUTPUT_COLUMNS[:3] diff --git a/packages/microcosm-build/tests/test_uk_spi_spine.py b/packages/microcosm-build/tests/test_uk_spi_spine.py index 7543f26c0a..fe8e681c7e 100644 --- a/packages/microcosm-build/tests/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/test_uk_spi_spine.py @@ -181,6 +181,28 @@ def test_spine_leaves_fail_closed_on_unknown_raw_person(tmp_path: Path) -> None: UKFRSHMRCSpineLeavesStageTransform(tmp_path, stage=stage)(_base_frame()) +def test_spine_leaves_sampled_rung_restricts_to_surviving_people( + tmp_path: Path, +) -> None: + # A #627 rung subsamples households after frs_spine, so raw-tab person + # coverage legitimately exceeds the frame; the sampled posture restricts + # the raw surface to the survivors while f100 keeps the strict fence. + stage = _leaves_stage( + tmp_path, + extra_adult_rows=({"sernum": 9, "person": 1, "inearns": 1.0},), + ) + + result = UKFRSHMRCSpineLeavesStageTransform( + tmp_path, stage=stage, sampled_rung=True + )(_base_frame()) + person = result.table("person") + + assert person["person_id"].tolist() == [2001, 1001] + assert person["hmrc_spi_pay"].tolist() == pytest.approx( + [5.0 * 365.25 / 7.0, 4.0 * 365.25 / 7.0] + ) + + def _support_stage() -> SourceStageSpec: return SourceStageSpec.from_mapping( { diff --git a/packages/microcosm-build/tests/test_uk_take_up_contract.py b/packages/microcosm-build/tests/test_uk_take_up_contract.py index 990567911a..d9965b83b7 100644 --- a/packages/microcosm-build/tests/test_uk_take_up_contract.py +++ b/packages/microcosm-build/tests/test_uk_take_up_contract.py @@ -24,7 +24,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: contract = load_uk_take_up_contract() assert contract.country == "uk" - assert contract.build_year == 2023 + assert contract.build_year == 2024 assert [entry.key for entry in contract.programs] == [ "child_benefit", "child_benefit_opts_out_rate", @@ -44,7 +44,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: 0.7, 0.55, 0.5, - 0.586, + 0.88, 0.812, 0.563, 0.597, @@ -53,7 +53,7 @@ def test_uk_contract_loads_and_selects_build_year_rates() -> None: ] assert [contract.rate(entry.key) for entry in contract.stochastic] == [ 0.95, - 0.1058, + 0.1252, 0.384, 0.0385, ] diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index eae2d039c4..13f209bdc6 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -983,6 +983,7 @@ def test_no_incumbent_data_package_references_in_live_tree(self) -> None: # name only the immutable, sha-verified enhanced-FRS reference and # never import or execute the retired data package. "packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json", + "packages/microcosm-build/src/microcosm/build/uk/frs_release.json", "packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json", # The UK national contract's registry-parity accounting names the # retired data package by necessity: 651 rows at pinned ref ebf733c diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index e3a0f57ab9..d8b6682e7d 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -12,6 +12,10 @@ from pathlib import Path from microcosm.build.country_spec import country_stage_plan, load_country_spec +from microcosm.build.frame_sampling import ( + normalize_sampled_household_mass, + sample_frame_households, +) from microcosm.build.logbook import canonical_json_bytes from microcosm.build.logbook_adoption import ( AttemptState, @@ -46,6 +50,7 @@ UKFRSLegacyProxiesStageTransform, ) from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.frs_spine import ( UKFRSSpineStageTransform, uk_frs_spine_seed_frame, @@ -57,6 +62,10 @@ ) from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind +from microcosm.build.uk_runtime.national_sampling import ( + UK_SAMPLE_RUNG_TOKENS, + UK_SAMPLE_SEED_DEFAULT, +) from microcosm.build.uk_runtime.regional_uprating import ( UKRegionalPropertyUpratingStageTransform, ) @@ -70,8 +79,9 @@ from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine _PIPELINE = "uk-frs-spine" -_RUNG = "f100" _REPOSITORY = Path(__file__).resolve().parents[1] +_RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" +_RUNG_ABORT_EXIT_CODE = 3 _STAGE_NAMES = ( "frs_spine", "frs_employment", @@ -95,6 +105,22 @@ ) +def _rung_sample_fraction(value: str) -> float: + """CLI rung policy (#624) over the permissive library validator.""" + + try: + fraction = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError( + f"sample fraction must be a number; got {value!r}." + ) from error + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise argparse.ArgumentTypeError( + "sample fraction must be one of 0.01, 0.10, or 1.0 (the #624 rungs)." + ) + return fraction + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -107,7 +133,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--frs-raw-dir", type=Path, required=True, - help="Directory containing the 14 licensed FRS 2023-24 tab files.", + help="Directory containing the 14 licensed FRS 2024-25 tab files.", ) parser.add_argument( "--spine-h5", @@ -132,6 +158,22 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Optional directory for a copy of the completed spine checkpoint.", ) + parser.add_argument( + "--sample-fraction", + type=_rung_sample_fraction, + default=1.0, + help=( + "Scale-ladder rung (#624): 0.01 smoke, 0.10 dev, or 1.0 full. " + "Below 1.0 the raw FRS spine is sampled immediately after ingest, " + "renormalized to full household mass, and treated as a receipt." + ), + ) + parser.add_argument( + "--sample-seed", + type=int, + default=UK_SAMPLE_SEED_DEFAULT, + help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", + ) parser.add_argument( "--was-tab", type=Path, @@ -162,7 +204,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=sha256_argument, help="Optional current Logbook chain head.", ) - return parser.parse_args(argv) + args = parser.parse_args(argv) + if args.sample_seed < 0: + parser.error("sample seed must be a non-negative integer.") + if args.sample_fraction != 1.0 and args.checkpoint_dir is not None: + parser.error( + "sampled spine rungs refuse --checkpoint-dir; rung artifacts are " + "receipts, never releases." + ) + return args def _validate_args(args: argparse.Namespace) -> None: @@ -370,6 +420,8 @@ def _build_sidecar( input_artifact_pins: dict[str, dict[str, object]], hmrc_replay: dict[str, object], stochastic_contract_sha256: str, + frs_vintage: str, + sampling: dict[str, object] | None, ) -> dict[str, object]: household_weight = frame.weights_for("household") return { @@ -401,6 +453,8 @@ def _build_sidecar( for stage in stages }, "declared_seeds": _declared_seeds(stages), + "source_vintages": {"frs": frs_vintage}, + "sampling": sampling, "stochastic_contract_sha256": stochastic_contract_sha256, "rules_engine": _rules_engine_provenance(), } @@ -434,6 +488,7 @@ def _record_attempt( code_pin: str, disposition: str, predecessor: str | None, + rung: str, spool_dir: Path, ) -> Path: return record_terminal_attempt( @@ -441,7 +496,7 @@ def _record_attempt( started_at=started_at, started_ts=started_ts, pipeline=_PIPELINE, - rung=_RUNG, + rung=rung, seed=None, code_pin=code_pin, disposition=disposition, @@ -450,8 +505,92 @@ def _record_attempt( ) +def _sample_spine_frame( + frame, + *, + fraction: float, + seed: int, +) -> tuple[object, dict[str, object] | None]: + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( + frame, + fraction=fraction, + seed=seed, + source_name="UK FRS spine", + ) + normalized, factor = normalize_sampled_household_mass( + sampled, + target_mass=float(household_weight.total), + source_name="UK FRS spine", + ) + return normalized, { + "fraction": float(fraction), + "seed": int(seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, + "post_household_count": int(len(normalized.table("household"))), + "normalization_factor": float(factor), + "receipt": dict(receipt), + } + + +def _run_plan_with_spine_sampling( + plan, + *, + sample_fraction: float, + sample_seed: int, +) -> tuple[object, tuple[object, ...], dict[str, object] | None]: + if not plan.stages or plan.stages[0].name != "frs_spine": + frame, records = plan.run(uk_frs_spine_seed_frame()) + return frame, records, None + + from microcosm.build.plan import StagePlan + + spine_frame, spine_records = StagePlan(plan.stages[:1]).run( + uk_frs_spine_seed_frame() + ) + spine_frame, sampling = _sample_spine_frame( + spine_frame, + fraction=sample_fraction, + seed=sample_seed, + ) + if len(plan.stages) == 1: + return spine_frame, spine_records, sampling + frame, tail_records = StagePlan(plan.stages[1:]).run(spine_frame) + return frame, (*spine_records, *tail_records), sampling + + +def _rung_abort_receipt( + args: argparse.Namespace, + *, + error: BaseException, +) -> dict[str, object]: + return { + "schema_version": 1, + "artifact_kind": "uk_frs_spine_rung_abort_receipt", + "build_kind": "uk_frs_spine", + "sampling": { + "sample_fraction": float(args.sample_fraction), + "sample_seed": int(args.sample_seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + }, + "named_edge": "spine_split_singleton_class", + "stage": "frs_spine", + "error": str(error), + "disposition": "aborted_with_receipt", + "remedy": ( + "Re-roll --sample-seed; accepted dev-scale statistical edge. " + "The computation is never altered to avoid it." + ), + } + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) + rung = UK_SAMPLE_RUNG_TOKENS[args.sample_fraction] started_at = time.perf_counter() started_ts = datetime.now(UTC) predecessor = resolve_predecessor(args.logbook_prev_row_digest) @@ -481,6 +620,7 @@ def main(argv: list[str] | None = None) -> int: args.spine_h5, args.spine_h5.with_suffix(".build.json"), args.spine_h5.with_suffix(".hmrc_replay.json"), + args.spine_h5.with_suffix(".rung_abort.json"), ] if args.emit_nonzero_shares is not None: stale_outputs.append(args.emit_nonzero_shares) @@ -543,10 +683,12 @@ def main(argv: list[str] | None = None) -> int: append_phase(state, "inputs_pinned") engine = _rules_engine() stochastic_contract = load_uk_take_up_contract() + frs_release = load_uk_frs_release() hmrc_spine_transform = UKSPIIncomeSpineStageTransform( args.spi_tab, args.hmrc_ods, stage=stages_by_name["hmrc_spi_income_spine"], + sampled_rung=args.sample_fraction != 1.0, ) implementations = { "frs_spine": UKFRSSpineStageTransform( @@ -631,9 +773,11 @@ def main(argv: list[str] | None = None) -> int: implementations["frs_hmrc_spine_leaves"] = UKFRSHMRCSpineLeavesStageTransform( args.frs_raw_dir, stage=stages_by_name["frs_hmrc_spine_leaves"], + sampled_rung=args.sample_fraction != 1.0, ) implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( stage=stages_by_name["spi_support_channel"], + sample_fraction=args.sample_fraction, ) implementations["hmrc_spi_income_spine"] = hmrc_spine_transform plan = country_stage_plan( @@ -641,7 +785,11 @@ def main(argv: list[str] | None = None) -> int: implementations, stage_names=stage_names, ) - frame, records = plan.run(uk_frs_spine_seed_frame()) + frame, records, sampling = _run_plan_with_spine_sampling( + plan, + sample_fraction=args.sample_fraction, + sample_seed=args.sample_seed, + ) append_phase(state, "spine_built") output = write_uk_national_frame(frame, args.spine_h5) append_phase(state, "spine_written") @@ -673,6 +821,8 @@ def main(argv: list[str] | None = None) -> int: input_artifact_pins=input_artifact_pins, hmrc_replay=replay_binding, stochastic_contract_sha256=stochastic_contract.resource_sha256, + frs_vintage=frs_release.vintage, + sampling=sampling, ) atomic_write_json(sidecar_path, sidecar) append_phase(state, "build_sidecar_written") @@ -712,11 +862,64 @@ def main(argv: list[str] | None = None) -> int: code_pin=code_pin, disposition="iterating", predecessor=predecessor, + rung=rung, spool_dir=spool_dir, ) print(f"Wrote FRS spine H5: {output}", file=sys.stderr) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) return 0 + except ValueError as error: + if args.sample_fraction != 1.0 and _RUNG_NAMED_EDGE_SIGNATURE in str(error): + rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") + receipt = _rung_abort_receipt(args, error=error) + atomic_write_json(rung_abort_path, receipt) + state.gate_verdicts = { + "uk_frs_spine_rung_abort": { + "verdict": "aborted", + "receipt": ( + f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" + "#/named_edge" + ), + } + } + append_phase(state, "rung_aborted") + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="discarded", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + print(json.dumps(receipt, indent=2, sort_keys=True)) + return _RUNG_ABORT_EXIT_CODE + try: + receipt_path = write_error_receipt( + error_receipt_path(args.spine_h5.parent, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), + ) + _record_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + code_pin=code_pin, + disposition="failed", + predecessor=predecessor, + rung=rung, + spool_dir=spool_dir, + ) + except Exception: + pass + print(f"UK FRS spine build failed: {error}", file=sys.stderr) + return 1 except Exception as error: try: receipt_path = write_error_receipt( @@ -736,6 +939,7 @@ def main(argv: list[str] | None = None) -> int: code_pin=code_pin, disposition="failed", predecessor=predecessor, + rung=rung, spool_dir=spool_dir, ) except Exception: diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index f8ecee0b69..b2ca34d0fe 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -38,6 +38,7 @@ from microcosm.build.uk_runtime.frs_hmrc_leaves import ( UKFRSHMRCRetainedLeavesStageTransform, ) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report from microcosm.build.uk_runtime.hmrc_restoration import ( UKHMRCIncomeStageTransform, @@ -160,7 +161,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, required=True, help=( - "Raw FRS 2023-24 directory containing adult.tab and benefits.tab " + "Raw FRS 2024-25 directory containing adult.tab and benefits.tab " "for source-faithful retained HMRC leaves." ), ) @@ -1101,6 +1102,8 @@ def _aggregate_build_record( for record in result.frame.mass_log ] release_evidence = dict(result.gate_report["release_evidence"]) + source_vintages = dict(family_evidence.get("source_vintages", {})) + source_vintages["frs"] = load_uk_frs_release().vintage return { "schema_version": 3, "build_kind": "uk_national_staging_dataset", @@ -1148,7 +1151,7 @@ def _aggregate_build_record( dict(family_sources.get("spi_donor", {})).get("rows_used", 0) ), }, - "source_vintages": dict(family_evidence.get("source_vintages", {})), + "source_vintages": source_vintages, "terminal_gates": dict(result.gate_report), "input_coverage": { "passed": bool(result.input_coverage.passed), diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index c816571e6d..235933ee12 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -348,6 +348,7 @@ def e6_identity_receipt( allocate_nhs_by_age_gender, load_etb_services_anchors, ) + rail_fare_index = float( load_etb_services_anchors()["rail_fare_index_2023"]["value"] ) @@ -360,9 +361,7 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: if {"electricity_consumption", "gas_consumption"} <= set(household.columns): household_out["domestic_energy_consumption"] = household[ "electricity_consumption" - ].to_numpy(dtype=float) + household["gas_consumption"].to_numpy( - dtype=float - ) + ].to_numpy(dtype=float) + household["gas_consumption"].to_numpy(dtype=float) if "rail_subsidy_spending" in household.columns: household_out["rail_usage"] = ( household["rail_subsidy_spending"].to_numpy(dtype=float) @@ -424,9 +423,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: for op in channel["operations"] if op["kind"] == "allocate_zero_weight_prior_mass" ) - household["household_weight"] = household[ - "household_weight" - ].to_numpy(dtype=float) / (1.0 - float(share)) + household["household_weight"] = household["household_weight"].to_numpy( + dtype=float + ) / (1.0 - float(share)) original = recompute(person, benunit, household) rng = np.random.default_rng(permutation_seed) permuted = recompute( @@ -510,9 +509,14 @@ def main() -> int: from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine + # E4 draws ran on the pre-stack FRS spine and its take-up targets are + # unweighted int(rate * n_units): recompute on the survey channel + # only, or every target moves with the synthetic rows (the post-#717 + # scoping rule the E5 receipt already applies). + frame = _frs_only_frame(frame) engine = PolicyEngineUKEngine() lha_category = engine.materialize( - frame, ("LHA_category",), uk_time_period(frame) + _engine_safe_frame(frame), ("LHA_category",), uk_time_period(frame) )["LHA_category"] receipt = e4_identity_receipt( frame, @@ -549,6 +553,74 @@ def main() -> int: return 0 if ok else 1 +def _frs_only_frame(frame): + """Scope a post-#717 artifact to the survey channel (FRS rows only).""" + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + + household = frame.table("household") + if "household_is_spi_synthetic" not in household.columns: + return frame + keep = ~household["household_is_spi_synthetic"].astype(bool) + weights = frame.weights_for("household").values[keep.to_numpy()] + household = household.loc[keep].reset_index(drop=True) + ids = set(household["household_id"].tolist()) + person = ( + frame.table("person") + .loc[lambda t: t["person_household_id"].isin(ids)] + .reset_index(drop=True) + ) + benunit_ids = set(person["person_benunit_id"].tolist()) + benunit = ( + frame.table("benunit") + .loc[lambda t: t["benunit_id"].isin(benunit_ids)] + .reset_index(drop=True) + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=weights, + ) + + +def _engine_safe_frame(frame): + """Fill by-design NaN on channel-only auxiliary columns for engine reads. + + The #717 SPI channel leaves hmrc_spi_* auxiliaries (e.g. + other_investment_income) NaN on FRS rows by design; instruments fill 0, + matching stage-time semantics, because the engine adapter rejects NaN. + LHA_category derivation does not read these columns. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + + tables = {} + for entity in ("person", "benunit", "household"): + table = frame.table(entity).copy() + for column in table.columns: + if table[column].dtype.kind == "f" and table[column].isna().any(): + table[column] = table[column].fillna(0.0) + tables[entity] = table + return uk_national_frame( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + def _reverse_rows(frame): from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, From 49c1eb33f166e0a19daec46800334b436945e119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:11:45 +0200 Subject: [PATCH 02/26] Regenerate the parity instruments from enhanced_frs_2024_25 The regenerator pins move to enhanced_frs_2024_25.h5 at the v1.56.14 tag (= incumbent-at-pin ebf733c) and the committed reference re-freezes at the new vintage: 145 columns (surface unchanged), period "2024", entity counts now test-pinned with the re-derived record-count identity (16,288 raw + 10,000 SPI) x 2 + 270 CGT band donors = 52,846 - no raw household is dropped at 2024-25 and the donor count follows the HMRC band file (30 x 9). The release-input coverage manifest regenerates against the new reference with the certified 2023 candidate unchanged (candidate side moves at #686); known gaps stay empty and the restored-column receipts hold. Part of #723. Co-Authored-By: Claude Fable 5 --- .../build/uk/efrs_parity_reference.json | 272 +++++++++--------- .../uk/release_input_coverage_manifest.json | 27 +- .../tests/test_uk_efrs_weighted_totals.py | 2 +- .../tests/test_uk_parity_reference.py | 32 ++- ...test_uk_release_input_coverage_manifest.py | 3 +- tools/build_uk_efrs_parity_reference.py | 16 +- ...uild_uk_release_input_coverage_manifest.py | 28 +- 7 files changed, 214 insertions(+), 166 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json index 5fd03c09d2..36d94b40f4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/uk/efrs_parity_reference.json @@ -33,13 +33,18 @@ "household_id" ], "unknown_export_columns_excluded": [ + "bus_fare_spending", "clone_index", "constituency_code_oa", + "employment_sector", "esa_health_condition_proxy", "esa_support_group_proxy", "free_school_breakfasts", "has_fuel_consumption", "higher_earner_tie_break", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", + "household_is_spi_synthetic", "la_code_oa", "legacy_jobseeker_proxy", "lsoa_code", @@ -47,7 +52,10 @@ "oa_code", "region_code_oa", "salary_sacrifice_asked", - "salary_sacrifice_reported" + "salary_sacrifice_reported", + "sic_industry_division", + "source_household_id", + "source_year" ], "version": "2.89.0", "zero_share_input_columns_excluded": [ @@ -61,19 +69,19 @@ "export_columns": 11, "input_columns": 10, "populated_input_columns": 10, - "records": 61898 + "records": 61223 }, "household": { - "export_columns": 60, + "export_columns": 66, "input_columns": 51, "populated_input_columns": 51, - "records": 53508 + "records": 52846 }, "person": { - "export_columns": 97, + "export_columns": 99, "input_columns": 87, "populated_input_columns": 84, - "records": 115710 + "records": 113617 } }, "input_entities": { @@ -226,160 +234,160 @@ "nonzero_shares": { "a_and_e_visits": 1.0, "aa_category": 1.0, - "access_fund": 0.000207, + "access_fund": 0.000422, "accommodation_type": 1.0, "admitted_patient_visits": 1.0, - "adult_ema": 0.000259, - "afcs_reported": 0.006101, - "age": 0.989128, + "adult_ema": 0.000194, + "afcs_reported": 0.001884, + "age": 0.990829, "age_started_or_accepted_current_education_or_training": 1.0, - "alcohol_and_tobacco_consumption": 0.573372, + "alcohol_and_tobacco_consumption": 0.562975, "attends_private_school_random_draw": 1.0, "brma": 1.0, - "bsp_reported": 0.015349, - "bus_subsidy_spending": 0.226321, - "capital_gains": 0.231216, - "carers_allowance_reported": 0.019912, - "charitable_investment_gifts": 0.000173, - "child_benefit_opts_out": 0.229377, - "child_benefit_reported": 0.094495, - "child_ema": 0.000691, - "child_tax_credit_reported": 0.022401, - "childcare_expenses": 0.036609, - "clothing_and_footwear_consumption": 0.475817, - "communication_consumption": 0.79846, - "corporate_wealth": 0.810683, - "council_tax": 0.883905, + "bsp_reported": 0.004929, + "bus_subsidy_spending": 0.314347, + "capital_gains": 0.23375, + "carers_allowance_reported": 0.010535, + "charitable_investment_gifts": 0.00029, + "child_benefit_opts_out": 0.228084, + "child_benefit_reported": 0.08849, + "child_ema": 0.000907, + "child_tax_credit_reported": 0.002588, + "childcare_expenses": 0.037098, + "clothing_and_footwear_consumption": 0.500908, + "communication_consumption": 0.795084, + "corporate_wealth": 0.822465, + "council_tax": 0.819116, "council_tax_band": 1.0, - "council_tax_benefit_reported": 0.062829, + "council_tax_benefit_reported": 0.058468, "current_education": 1.0, - "dfe_education_spending": 0.000411, - "diesel_spending": 0.213688, - "dividend_income": 0.130706, + "dfe_education_spending": 0.000265, + "diesel_spending": 0.190346, + "dividend_income": 0.088455, "dla_m_category": 1.0, "dla_sc_category": 1.0, - "domestic_energy_consumption": 0.973163, - "domestic_rates": 0.110675, - "education_consumption": 0.130298, - "education_grants": 0.006032, - "electricity_consumption": 0.930141, - "employee_pension_contributions": 0.235079, - "employer_pension_contributions": 0.291194, - "employment_income": 0.481929, + "domestic_energy_consumption": 0.954093, + "domestic_rates": 0.106933, + "education_consumption": 0.125818, + "education_grants": 0.005483, + "electricity_consumption": 0.861352, + "employee_pension_contributions": 0.273489, + "employer_pension_contributions": 0.314865, + "employment_income": 0.492928, "employment_status": 1.0, - "esa_contrib_reported": 0.014796, - "esa_income_reported": 0.016438, - "external_child_payments": 0.011438, - "food_and_non_alcoholic_beverages_consumption": 0.992113, - "free_school_fruit_veg": 0.004719, - "free_school_meals": 0.034016, + "esa_contrib_reported": 0.008053, + "esa_income_reported": 0.009558, + "external_child_payments": 0.008875, + "food_and_non_alcoholic_beverages_consumption": 0.997086, + "free_school_fruit_veg": 0.004524, + "free_school_meals": 0.03422, "full_rate_vat_expenditure_rate": 1.0, - "gas_consumption": 0.969201, + "gas_consumption": 0.952314, "gender": 1.0, - "gift_aid": 0.037525, - "gross_financial_wealth": 0.977499, - "health_consumption": 0.484376, - "healthy_start_vouchers": 0.003094, + "gift_aid": 0.020904, + "gross_financial_wealth": 0.987189, + "health_consumption": 0.512849, + "healthy_start_vouchers": 0.002236, "highest_education": 1.0, - "hours_worked": 0.431717, - "household_furnishings_consumption": 0.829184, - "household_owns_tv": 0.951073, + "hours_worked": 0.428193, + "household_furnishings_consumption": 0.812625, + "household_owns_tv": 0.949022, "household_weight": 1.0, - "housing_benefit_reported": 0.030144, - "housing_service_charges": 0.105405, - "housing_water_and_electricity_consumption": 0.999813, - "iidb_reported": 0.016334, - "income_support_reported": 0.006274, - "is_before_universal_credit_qualifying_young_person_terminal_date": 0.000847, - "is_benunit_head": 0.534941, - "is_disabled_for_benefits": 0.064714, - "is_enhanced_disabled_for_benefits": 0.014657, - "is_household_head": 0.462432, - "is_in_non_advanced_education": 0.163581, - "is_married": 0.262916, - "is_parent": 0.204269, - "is_severely_disabled_for_benefits": 0.039098, - "jsa_contrib_reported": 0.00038, - "jsa_income_reported": 0.004408, - "lump_sum_income": 0.002783, - "main_residence_value": 0.590491, - "main_residential_property_purchased_is_first_home": 0.384541, - "maintenance_expenses": 0.008106, - "maintenance_income": 0.01184, + "housing_benefit_reported": 0.021423, + "housing_service_charges": 0.106971, + "housing_water_and_electricity_consumption": 0.999924, + "iidb_reported": 0.005589, + "income_support_reported": 0.000757, + "is_before_universal_credit_qualifying_young_person_terminal_date": 0.000898, + "is_benunit_head": 0.538854, + "is_disabled_for_benefits": 0.08908, + "is_enhanced_disabled_for_benefits": 0.045794, + "is_household_head": 0.465124, + "is_in_non_advanced_education": 0.161833, + "is_married": 0.256587, + "is_parent": 0.20335, + "is_severely_disabled_for_benefits": 0.056532, + "jsa_contrib_reported": 0.000378, + "jsa_income_reported": 0.000334, + "lump_sum_income": 0.003732, + "main_residence_value": 0.676078, + "main_residential_property_purchased_is_first_home": 0.380653, + "maintenance_expenses": 0.007437, + "maintenance_income": 0.01094, "marital_status": 1.0, - "maximum_extended_childcare_hours_usage": 0.998611, - "miscellaneous_consumption": 0.913471, - "miscellaneous_income": 0.012808, - "mortgage_capital_repayment": 0.252448, - "mortgage_interest_repayment": 0.250542, - "net_financial_wealth": 0.999551, + "maximum_extended_childcare_hours_usage": 0.99853, + "miscellaneous_consumption": 0.897532, + "miscellaneous_income": 0.013, + "mortgage_capital_repayment": 0.248098, + "mortgage_interest_repayment": 0.246206, + "net_financial_wealth": 0.998789, "nhs_a_and_e_spending": 1.0, "nhs_admitted_patient_spending": 1.0, "nhs_outpatient_spending": 1.0, - "non_residential_property_value": 0.056365, - "num_vehicles": 0.666368, - "other_residential_property_value": 0.125514, + "non_residential_property_value": 0.010786, + "num_vehicles": 0.760171, + "other_residential_property_value": 0.074991, "outpatient_visits": 1.0, - "owned_land": 0.045003, - "pension_contributions_via_salary_sacrifice": 0.077003, - "pension_credit_reported": 0.01305, - "personal_pension_contributions": 0.049417, - "petrol_spending": 0.507625, + "owned_land": 0.014306, + "pension_contributions_via_salary_sacrifice": 0.083139, + "pension_credit_reported": 0.016142, + "personal_pension_contributions": 0.035796, + "petrol_spending": 0.444177, "pip_dl_category": 1.0, "pip_m_category": 1.0, - "private_pension_income": 0.220569, - "private_transfer_income": 0.008366, - "property_income": 0.051508, - "property_purchased": 0.038985, - "property_wealth": 0.634073, - "rail_subsidy_spending": 0.044442, - "rail_usage": 0.044442, - "receives_benefits_in_own_right": 0.061239, - "recreation_consumption": 0.970173, + "private_pension_income": 0.231391, + "private_transfer_income": 0.009919, + "property_income": 0.039413, + "property_purchased": 0.038584, + "property_wealth": 0.708152, + "rail_subsidy_spending": 0.126443, + "rail_usage": 0.126443, + "receives_benefits_in_own_right": 0.081361, + "recreation_consumption": 0.973924, "region": 1.0, - "rent": 0.312252, - "restaurants_and_hotels_consumption": 0.574867, - "savings": 0.635344, - "savings_interest_income": 0.343566, - "sda_reported": 0.000104, - "self_employment_income": 0.061533, - "ssmg_reported": 0.000415, - "state_pension_reported": 0.23571, - "statutory_maternity_pay": 0.002506, - "statutory_sick_pay": 0.002057, - "structural_insurance_payments": 0.615983, - "student_loan_balance": 0.047256, + "rent": 0.311452, + "restaurants_and_hotels_consumption": 0.630549, + "savings": 0.662075, + "savings_interest_income": 0.424963, + "sda_reported": 8.8e-05, + "self_employment_income": 0.054939, + "ssmg_reported": 0.000405, + "state_pension_reported": 0.245694, + "statutory_maternity_pay": 0.002139, + "statutory_sick_pay": 0.001866, + "structural_insurance_payments": 0.616054, + "student_loan_balance": 0.01983, "student_loan_plan": 1.0, - "student_loan_repayments": 0.047584, - "student_loans": 0.011633, - "tax_free_savings_income": 0.157705, + "student_loan_repayments": 0.049042, + "student_loans": 0.012965, + "tax_free_savings_income": 0.189664, "tenure_type": 1.0, - "transport_consumption": 0.780145, - "universal_credit_reported": 0.042537, - "water_and_sewerage_charges": 0.852209, - "winter_fuel_allowance_reported": 0.23908, - "working_tax_credit_reported": 0.014761, - "would_claim_child_benefit": 0.893341, - "would_claim_extended_childcare": 0.808459, - "would_claim_marriage_allowance": 0.500147, - "would_claim_pc": 0.702091, - "would_claim_scp": 0.860254, - "would_claim_targeted_childcare": 0.594623, - "would_claim_tfc": 0.581667, - "would_claim_uc": 0.550163, - "would_claim_universal_childcare": 0.560923, - "would_evade_tv_licence_fee": 0.105666 + "transport_consumption": 0.86226, + "universal_credit_reported": 0.057359, + "water_and_sewerage_charges": 0.776937, + "winter_fuel_allowance_reported": 0.018598, + "working_tax_credit_reported": 0.000493, + "would_claim_child_benefit": 0.892573, + "would_claim_extended_childcare": 0.813893, + "would_claim_marriage_allowance": 0.498587, + "would_claim_pc": 0.70088, + "would_claim_scp": 0.858287, + "would_claim_targeted_childcare": 0.593404, + "would_claim_tfc": 0.879931, + "would_claim_uc": 0.550594, + "would_claim_universal_childcare": 0.560933, + "would_evade_tv_licence_fee": 0.127011 }, "schema_version": 3, "source": { - "filename": "enhanced_frs_2023_24.h5", - "period": "2023", + "filename": "enhanced_frs_2024_25.h5", + "period": "2024", "repo_id": "policyengine/policyengine-uk-data-private", "repo_type": "model", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "size_bytes": 125434652, - "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/655dd07e4bb9c777b00dac044949611f1feb824f/enhanced_frs_2023_24.h5", - "vintage": "2023_24" + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "size_bytes": 126579434, + "url": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/a2039519d3b92aecc06c66dfd175cb46ac24cada/enhanced_frs_2024_25.h5", + "vintage": "2024_25" } } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 72f881b317..e38bf29d8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -478,7 +478,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -497,7 +497,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -518,10 +518,10 @@ ], "required_mass_change_reason": "Amounts-only capital gains redraw: household weights pass through unchanged and total household mass is conserved.", "source_manifest": "cgt_source_stages.json", - "source_manifest_sha256": "16b68f1d222116278421b210d3d9de0d06edcdf25d68f5a29dbf0f52ca63384a", + "source_manifest_sha256": "71104111b4b9f2da00ce49ad5abec54a35d300196032c9742d62e02aa1730774", "source_vintages": { "hmrc_surface": "2023-24", - "mapped_build_period": "2023" + "mapped_build_period": "2024" }, "stage": "hmrc_cgt_gains", "status": "required_at_build" @@ -603,7 +603,8 @@ "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", "source_vintages": { "hmrc_surface": "2023-24", - "mapped_build_period": "2023", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", "spi_donor": "2022-23" }, "spi_prior_national_household_mass_share": 0.5, @@ -639,7 +640,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -659,7 +660,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -690,7 +691,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", + "source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" @@ -701,12 +702,12 @@ }, "reference": { "derived_from": "efrs_parity_reference.json", - "filename": "enhanced_frs_2023_24.h5", - "period": "2023", + "filename": "enhanced_frs_2024_25.h5", + "period": "2024", "populated_input_columns": 145, - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24" + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25" }, "restoration_evidence": { "derived_from": "efrs_parity_known_gaps.json", diff --git a/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py b/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py index 4f59962719..a4fe3a3c68 100644 --- a/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py +++ b/packages/microcosm-build/tests/test_uk_efrs_weighted_totals.py @@ -75,7 +75,7 @@ def synthetic_efrs(tmp_path, monkeypatch) -> Path: "council_tax": np.array([1_000.0, 2_000.0, 3_000.0]), } ) - path = tmp_path / "enhanced_frs_2023_24.h5" + path = tmp_path / "enhanced_frs_2024_25.h5" with pd.HDFStore(path) as store: store.put("person", person, format="table", data_columns=True) store.put("benunit", benunit, format="table", data_columns=True) diff --git a/packages/microcosm-build/tests/test_uk_parity_reference.py b/packages/microcosm-build/tests/test_uk_parity_reference.py index bc5c933709..d5d9c3aff9 100644 --- a/packages/microcosm-build/tests/test_uk_parity_reference.py +++ b/packages/microcosm-build/tests/test_uk_parity_reference.py @@ -88,15 +88,31 @@ def test_source_provenance_is_complete_and_immutable(self) -> None: source = load_efrs_parity_reference().source assert source.repo_id == "policyengine/policyengine-uk-data-private" assert source.repo_type == "model" - assert source.filename == "enhanced_frs_2023_24.h5" - assert source.revision == "655dd07e4bb9c777b00dac044949611f1feb824f" + assert source.filename == "enhanced_frs_2024_25.h5" + assert source.revision == "a2039519d3b92aecc06c66dfd175cb46ac24cada" assert source.sha256 == ( - "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" ) - assert source.size_bytes == 125_434_652 + assert source.size_bytes == 126_579_434 assert source.url.endswith(f"/{source.revision}/{source.filename}") - assert source.period == "2023" - assert source.vintage == "2023_24" + assert source.period == "2024" + assert source.vintage == "2024_25" + + def test_entity_record_counts_match_reviewed_2024_25_surface(self) -> None: + raw = json.loads(_resource(EFRS_PARITY_REFERENCE_RESOURCE).read_text()) + stats = raw["entity_stats"] + + # 52846 households = (16288 raw FRS + 10000 SPI) * 2 + 270 CGT band + # donors (30 * 9 bands; 181 FRS-side + 89 SPI-side). Unlike 2023-24, + # no raw household is dropped at 2024-25. + assert {entity: values["records"] for entity, values in stats.items()} == { + "person": 113_617, + "benunit": 61_223, + "household": 52_846, + } + assert set(stats) == {"person", "benunit", "household"} + for values in stats.values(): + assert {"records", "export_columns"} <= set(values) def test_recorded_sha256_is_lowercase_hex(self) -> None: sha = load_efrs_parity_reference().source.sha256 @@ -146,9 +162,7 @@ def test_uk_loader_migration_inputs_are_covered(self) -> None: def test_formula_owned_persisted_overrides_are_hard_covered(self) -> None: raw = json.loads(_resource(EFRS_PARITY_REFERENCE_RESOURCE).read_text()) - included = set( - raw["engine"]["formula_owned_persisted_overrides_included"] - ) + included = set(raw["engine"]["formula_owned_persisted_overrides_included"]) assert included == _FORMULA_OWNED_PERSISTED_OVERRIDES assert included <= set(raw["nonzero_shares"]) assert "formula_owned_export_columns_excluded" not in raw["engine"] diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index 5c5f90a1fd..b39143898d 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -168,7 +168,8 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No assert family["source_vintages"] == { "spi_donor": "2022-23", "hmrc_surface": "2023-24", - "mapped_build_period": "2023", + "mapped_build_period": "2024", + "period_mapping": "latest_published_tax_year", } assert family["spi_prior_national_household_mass_share"] == 0.5 assert family["required_mass_change_reason"] == ( diff --git a/tools/build_uk_efrs_parity_reference.py b/tools/build_uk_efrs_parity_reference.py index e8785bedcf..26dc7bfebb 100644 --- a/tools/build_uk_efrs_parity_reference.py +++ b/tools/build_uk_efrs_parity_reference.py @@ -17,7 +17,7 @@ Run with a licensed local artifact (the source SHA is always verified): python tools/build_uk_efrs_parity_reference.py \ - --input-h5 /path/to/enhanced_frs_2023_24.h5 + --input-h5 /path/to/enhanced_frs_2024_25.h5 Without ``--input-h5``, the tool first resolves the exact immutable revision from the Hugging Face cache and then downloads it if necessary. It accepts the @@ -47,12 +47,12 @@ # adjudication inputs. The licensed data lives in a private HF *model* repo. SOURCE_REPO_ID = "policyengine/policyengine-uk-data-private" SOURCE_REPO_TYPE = "model" -SOURCE_FILENAME = "enhanced_frs_2023_24.h5" -SOURCE_REVISION = "655dd07e4bb9c777b00dac044949611f1feb824f" -SOURCE_SHA256 = "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" -SOURCE_SIZE_BYTES = 125_434_652 -SOURCE_VINTAGE = "2023_24" -SOURCE_PERIOD = "2023" +SOURCE_FILENAME = "enhanced_frs_2024_25.h5" +SOURCE_REVISION = "a2039519d3b92aecc06c66dfd175cb46ac24cada" +SOURCE_SHA256 = "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" +SOURCE_SIZE_BYTES = 126_579_434 +SOURCE_VINTAGE = "2024_25" +SOURCE_PERIOD = "2024" SOURCE_URL = ( f"https://huggingface.co/{SOURCE_REPO_ID}/resolve/" f"{SOURCE_REVISION}/{SOURCE_FILENAME}" @@ -85,7 +85,7 @@ def _parse_args() -> argparse.Namespace: "--input-h5", type=Path, help=( - "Licensed enhanced_frs_2023_24.h5. If omitted, resolve the exact " + "Licensed enhanced_frs_2024_25.h5. If omitted, resolve the exact " "pinned revision from the HF cache/download API." ), ) diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 557eb7bd89..0151aa33b3 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -994,6 +994,20 @@ def _hmrc_family_coverage_contract( stage = stages[0] if not isinstance(stage, dict) or stage.get("stage") != "hmrc_spi_income": raise ValueError(f"{HMRC_SOURCE_STAGES_PATH}: expected hmrc_spi_income stage.") + canonical_payload = _load(SOURCE_STAGES_PATH) + canonical_stages = canonical_payload.get("stages") + if not isinstance(canonical_stages, list): + raise ValueError(f"{SOURCE_STAGES_PATH}: expected source stages list.") + canonical_matches = [ + candidate + for candidate in canonical_stages + if isinstance(candidate, dict) and candidate.get("stage") == "hmrc_spi_income" + ] + if len(canonical_matches) != 1: + raise ValueError( + f"{SOURCE_STAGES_PATH}: expected exactly one hmrc_spi_income stage." + ) + canonical_stage = canonical_matches[0] base_candidate = stage.get("base_candidate") if not isinstance(base_candidate, dict): raise ValueError( @@ -1011,6 +1025,11 @@ def _hmrc_family_coverage_contract( for artifact in stage.get("artifacts", []) if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) } + canonical_artifacts = { + artifact["role"]: artifact + for artifact in canonical_stage.get("artifacts", []) + if isinstance(artifact, dict) and isinstance(artifact.get("role"), str) + } operations = { operation["kind"]: operation for operation in stage.get("operations", []) @@ -1018,6 +1037,7 @@ def _hmrc_family_coverage_contract( } required_artifacts = {"qrf_donor", "published_fact_surface"} missing_artifacts = sorted(required_artifacts - set(artifacts)) + missing_canonical_artifacts = sorted(required_artifacts - set(canonical_artifacts)) required_operations = { "retain_adjudicated_frs_hmrc_leaves", "verify_pinned_hmrc_source_pair", @@ -1026,10 +1046,11 @@ def _hmrc_family_coverage_contract( "gate_distributional_effective_mass", } missing_operations = sorted(required_operations - set(operations)) - if missing_artifacts or missing_operations: + if missing_artifacts or missing_canonical_artifacts or missing_operations: raise ValueError( f"{HMRC_SOURCE_STAGES_PATH}: incomplete HMRC family contract; " f"missing_artifacts={missing_artifacts}, " + f"missing_canonical_artifacts={missing_canonical_artifacts}, " f"missing_operations={missing_operations}." ) classification = operations["classify_hmrc_income_facts_with_reviewed_fences"] @@ -1082,7 +1103,10 @@ def _hmrc_family_coverage_contract( "spi_donor": str(artifacts["qrf_donor"]["vintage"]), "hmrc_surface": str(artifacts["published_fact_surface"]["vintage"]), "mapped_build_period": str( - artifacts["published_fact_surface"]["mapped_build_period"] + canonical_artifacts["published_fact_surface"]["mapped_build_period"] + ), + "period_mapping": str( + canonical_artifacts["published_fact_surface"]["period_mapping"] ), }, "spi_prior_national_household_mass_share": float( From badc1213dfa3dfce52d62c7ff60203b45e6fc240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:12:02 +0200 Subject: [PATCH 03/26] Move the gate-side reference pins to the 2024-25 line The efrs-post-calibration input-mass descriptor is replaced in place (same name, #687's replacement model): enhanced_frs_2024_25.h5 identity and the totals_sha256 of the regenerated 131-column weighted-totals evidence, with the registry, gates.json, and the data-shard publication mirrors re-pinned in the same reviewed change per the gate-battery contract. No thresholds move - #723 records the re-measured baselines, #686 arms them. UK_REFERENCE_DATASET_NAME follows the incumbent's 2024-25 dataset name. Both per-reference reviewed exclusions are re-signed against the new reference (charitable_investment_gifts; owned_land on a fresh 2024-25 stability receipt), pending the approver's confirmation on the PR. Part of #723. Co-Authored-By: Claude Fable 5 --- .../src/microcosm/build/uk/gates.json | 12 ++--- .../uk/input_mass_reviewed_exclusions.json | 8 ++-- .../build/uk_runtime/terminal_gates.py | 2 +- .../build/uk_runtime/weighted_integrity.py | 47 ++++++++----------- .../tests/test_uk_terminal_gates.py | 20 ++++---- .../tests/test_uk_weighted_integrity.py | 34 +++++++------- .../src/microcosm/data/contract.py | 18 +++---- .../microcosm-data/tests/test_contract.py | 33 ++++++------- 8 files changed, 83 insertions(+), 91 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index b258907edd..0d73426efb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -289,13 +289,13 @@ "reference_registry": { "efrs-post-calibration": { "identity": { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24" + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25" }, - "totals_sha256": "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e", - "scope_note": "Post-calibration eFRS production incumbent; structurally lacks the SPI clone channel, so SPI-channel-exclusive columns are comparable only through per-reference reviewed exclusions." + "totals_sha256": "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659", + "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; structurally lacks the SPI clone channel, so SPI-channel-exclusive columns are comparable only through per-reference reviewed exclusions." } }, "relative_tolerance": 4.521811483823806, diff --git a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json index 298b6b8c80..995d4824a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/input_mass_reviewed_exclusions.json @@ -7,15 +7,15 @@ "reason": "SPI-channel-exclusive column on a channel-blind reference: the efrs-post-calibration incumbent structurally lacks the SPI clone channel, so its reference mass is survey-side scraps while the staged candidate's mass is the admin-captured SPI channel functioning as designed (microcosm#630 case 2). Compared normally against any future channel-aware reference.", "approved_by": "juaristi22", "adjudication": "microcosm#630", - "approved_on": "2026-08-17", - "expires_on": "2027-02-17" + "approved_on": "2026-08-20", + "expires_on": "2027-02-20" }, "owned_land": { "reason": "Sparse heavy-tailed WAS donor column (0.7 percent weighted nonzero share) whose weighted total is dominated by a handful of large farm/estate records: the E5 stability receipt (data/ukds/acceptance/e5/owned_land_stability_receipt.json) measures a 37.7 percent national and 2.41x London swing between adjacent seeds, the same realization-variance class the archived incumbent data repo records at uk-data#448 (4.6x Wales swing across releases). Register parity at this grain is not meaningful until the whole-spine comparison; the one-month expiry enforces the end-of-workstream revisit registered on microcosm#145 (winsorised donor or separate land imputation are the candidate remedies).", "approved_by": "juaristi22", "adjudication": "microcosm#714", - "approved_on": "2026-08-19", - "expires_on": "2026-09-19" + "approved_on": "2026-08-20", + "expires_on": "2026-09-20" } } } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 8759a77fcf..8be29e192f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -70,7 +70,7 @@ ] UK_CANDIDATE_DATASET_NAME = "populace_uk_2023" -UK_REFERENCE_DATASET_NAME = "enhanced_frs_2023_24_recalibrated" +UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25_recalibrated" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index b17a03c2b0..bc3b8f6aaf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -160,21 +160,12 @@ UK_DEGENERATE_EXCLUSION_REGISTER_RESOURCE = "degenerate_reviewed_exclusions.json" # Canonical sha256 of {"reference": {"identity": ..., "totals": ...}} for -# the 131-column weighted input surface emitted from the pinned enhanced-FRS -# artifact (sha 584ae33d...) by build_uk_efrs_parity_reference.py. The totals -# remain uncommitted under the UKDS EUL; this reviewed digest lets the gate and -# publication contract bind them without disclosing them (PR #610 review). -# -# Re-frozen keys-only under the #611 A4 dedup: totals keys went from -# "entity.column" to flat frame column names, every value byte-identical to -# the #610 freeze (whose saved totals reproduce the prior digest 11b22dd4… -# exactly), 131 columns before and after, zero cross-entity collisions — -# also enforced at mint time by the migrated tool's collision guard. The -# identity is unchanged. Reviewed by María Juaristi (2026-08-12); verified -# by loading the re-keyed totals through load_uk_input_mass_reference -# unpatched. +# the weighted input surface emitted from the pinned enhanced-FRS artifact by +# build_uk_efrs_parity_reference.py. The totals remain uncommitted under the +# UKDS EUL; this reviewed digest lets the gate and publication contract bind +# them without disclosing them. UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) @@ -216,7 +207,9 @@ def __post_init__(self) -> None: ) for field_name in ("sha256", "totals_sha256"): value = getattr(self, field_name) - if len(value) != 64 or any(character not in _SHA256_HEX for character in value): + if len(value) != 64 or any( + character not in _SHA256_HEX for character in value + ): raise ValueError( f"UK input-mass reference descriptor {field_name} must be " "a lowercase sha256." @@ -245,15 +238,16 @@ def spec_payload(self) -> dict[str, object]: _UK_INPUT_MASS_REFERENCE_DESCRIPTOR = UKInputMassReferenceDescriptor( name="efrs-post-calibration", - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note=( - "Post-calibration eFRS production incumbent; structurally lacks the " - "SPI clone channel, so SPI-channel-exclusive columns are comparable " - "only through per-reference reviewed exclusions." + "Channel-blind post-calibration enhanced-FRS production incumbent, " + "pinned to the 2024-25 line; structurally lacks the SPI clone " + "channel, so SPI-channel-exclusive columns are comparable only " + "through per-reference reviewed exclusions." ), ) @@ -439,8 +433,7 @@ def coerce_input_mass_reference_registry( identity = entry["identity"] if not isinstance(identity, Mapping): raise TypeError( - f"{label} reference_registry entry {name!r} identity must be " - "an object." + f"{label} reference_registry entry {name!r} identity must be an object." ) expected_identity_fields = {"filename", "revision", "sha256", "vintage"} if set(identity) != expected_identity_fields: @@ -542,9 +535,9 @@ def _premature_exclusion_failure( ) -def _read_register_payload(source: str | Path | None, *, resource: str) -> tuple[ - Mapping[str, object], str -]: +def _read_register_payload( + source: str | Path | None, *, resource: str +) -> tuple[Mapping[str, object], str]: if source is None: raw = files("microcosm.build.uk").joinpath(resource).read_text("utf-8") label = resource diff --git a/packages/microcosm-build/tests/test_uk_terminal_gates.py b/packages/microcosm-build/tests/test_uk_terminal_gates.py index b8fe346840..2a67eace52 100644 --- a/packages/microcosm-build/tests/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/test_uk_terminal_gates.py @@ -284,20 +284,20 @@ def test_ported_june_parity_gates_reject_empty_evidence() -> None: def _input_mass_reference(totals=None) -> UKInputMassReference: return UKInputMassReference( totals=({"employment_income": 10.0} if totals is None else totals), - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256=("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256=("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + vintage="2024_25", ) def _input_mass_descriptor() -> UKInputMassReferenceDescriptor: return UKInputMassReferenceDescriptor( name="efrs-post-calibration", - filename="enhanced_frs_2023_24.h5", - revision="655dd07e4bb9c777b00dac044949611f1feb824f", - sha256="584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - vintage="2023_24", + filename="enhanced_frs_2024_25.h5", + revision="a2039519d3b92aecc06c66dfd175cb46ac24cada", + sha256="97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + vintage="2024_25", totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note="Seeded scoped-reference note.", ) @@ -332,9 +332,7 @@ def test_zeroed_input_column_fails_by_name() -> None: assert gate.name == "input_mass_parity" assert "employment_income" in gate.failures[0] assert "mass is zero" in gate.failures[0] - assert gate.details["reference_identity"]["filename"] == ( - "enhanced_frs_2023_24.h5" - ) + assert gate.details["reference_identity"]["filename"] == ("enhanced_frs_2024_25.h5") def test_999_permille_mass_loss_fails_by_name() -> None: diff --git a/packages/microcosm-build/tests/test_uk_weighted_integrity.py b/packages/microcosm-build/tests/test_uk_weighted_integrity.py index 31b8fab61c..c73eaaf2e0 100644 --- a/packages/microcosm-build/tests/test_uk_weighted_integrity.py +++ b/packages/microcosm-build/tests/test_uk_weighted_integrity.py @@ -91,10 +91,10 @@ def _frame( def _reference(totals, **overrides) -> UKInputMassReference: fields = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": ("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "vintage": "2024_25", } fields.update(overrides) return UKInputMassReference(totals=totals, **fields) @@ -103,10 +103,10 @@ def _reference(totals, **overrides) -> UKInputMassReference: def _descriptor(**overrides) -> UKInputMassReferenceDescriptor: fields = { "name": "efrs-post-calibration", - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", "totals_sha256": UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, "scope_note": "Seeded scoped-reference note.", } @@ -272,10 +272,10 @@ def test_input_mass_reference_identity_is_recorded() -> None: ) assert gate.details["reference_identity"] == { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": ("584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d"), - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": ("97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e"), + "vintage": "2024_25", } @@ -1001,12 +1001,12 @@ def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> No { "schema_version": 1, "identity": { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", "sha256": ( - "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e" ), - "vintage": "2023_24", + "vintage": "2024_25", }, "totals": {"employment_income": 10.5}, } @@ -1023,7 +1023,7 @@ def test_input_mass_reference_round_trips_the_measurement_schema(tmp_path) -> No ): reference = load_uk_input_mass_reference(path) - assert reference.filename == "enhanced_frs_2023_24.h5" + assert reference.filename == "enhanced_frs_2024_25.h5" assert dict(reference.totals) == {"employment_income": 10.5} with pytest.raises(ValueError, match="schema_version"): path.write_text(json.dumps({"schema_version": 9})) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index fc0790570b..d1ed7667f7 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -187,10 +187,10 @@ # uk/gates.json reference_registry["efrs-post-calibration"].identity. _UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" _UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", } # Independent publication pin for the canonical # {"reference": {"identity": ..., "totals": ...}} evidence emitted from the @@ -198,7 +198,7 @@ # under the UKDS EUL; keep this in lockstep with # UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 in the build shard. _UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) _UK_TERMINAL_GATE_DETAIL_FIELDS = { "uk_release_input_coverage": frozenset( @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "728a5fe2f543f59e2797e4227269fe4516508a274b4fa5fe49559387d6b9d686" + "a9b4887b08f628c86fc4ee679b41f050ed26f3c157cf424c9a02043d510509b5" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f01e5459debc6f3ebfa097591749377b640a5d633e3df3825575b4ec15eeacb2" + "263569ea70081420a1dc06dcf262f6581f3225ebdaf464106ec3fcf571455fae" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "f358121fefc6e0e2371956dc0628c1d997ee3735e558ca3dd4526326a6add78b" + "a3dfe036599981665ac019ce1902283d6672522ee80d866a0fb62135af953cd9" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -420,7 +420,7 @@ # canonical hash; this pins the wrapped digest so the entry's evidence line # still binds the enhanced-FRS incumbent totals. _UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "df74556909990345bc9032f6f5db7f817d9c273f75522a85ab6e332dd8dc7355" + "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" ) # The degenerate binding's evidence payload digests the resolved exclusion # records; for a release that must be the committed register, so its digest diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 6d5cc69cf7..7da9cb4bca 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -64,19 +64,20 @@ "qrf_tail_concentration": ("qrf_tail_concentration",), } UK_INPUT_MASS_REFERENCE_IDENTITY = { - "filename": "enhanced_frs_2023_24.h5", - "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", - "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d", - "vintage": "2023_24", + "filename": "enhanced_frs_2024_25.h5", + "revision": "a2039519d3b92aecc06c66dfd175cb46ac24cada", + "sha256": "97a07f9ccb54019e4550e70980c561c985523e6bbc43d21938d01536e37d6c3e", + "vintage": "2024_25", } UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256 = ( - "c36c015a60f796ad9199a4a5652706f5310909cb572b1c90092ef9df1fa7187e" + "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659" ) UK_INPUT_MASS_ACTIVE_REFERENCE = "efrs-post-calibration" UK_INPUT_MASS_REFERENCE_SCOPE_NOTE = ( - "Post-calibration eFRS production incumbent; structurally lacks the SPI " - "clone channel, so SPI-channel-exclusive columns are comparable only " - "through per-reference reviewed exclusions." + "Channel-blind post-calibration enhanced-FRS production incumbent, pinned " + "to the 2024-25 line; structurally lacks the SPI clone channel, so " + "SPI-channel-exclusive columns are comparable only through per-reference " + "reviewed exclusions." ) UK_INPUT_MASS_REVIEWED_EXCLUSIONS = { "charitable_investment_gifts": { @@ -90,8 +91,8 @@ ), "approved_by": "juaristi22", "adjudication": "microcosm#630", - "approved_on": "2026-08-17", - "expires_on": "2027-02-17", + "approved_on": "2026-08-20", + "expires_on": "2027-02-20", }, "owned_land": { "reason": ( @@ -111,8 +112,8 @@ ), "approved_by": "juaristi22", "adjudication": "microcosm#714", - "approved_on": "2026-08-19", - "expires_on": "2026-09-19", + "approved_on": "2026-08-20", + "expires_on": "2026-09-20", }, } GIT_COMMIT = "5fa48f07436a806ad75ff76fd22cfb8613bddbe0" @@ -144,19 +145,19 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "728a5fe2f543f59e2797e4227269fe4516508a274b4fa5fe49559387d6b9d686" + "a9b4887b08f628c86fc4ee679b41f050ed26f3c157cf424c9a02043d510509b5" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f01e5459debc6f3ebfa097591749377b640a5d633e3df3825575b4ec15eeacb2" + "263569ea70081420a1dc06dcf262f6581f3225ebdaf464106ec3fcf571455fae" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "f358121fefc6e0e2371956dc0628c1d997ee3735e558ca3dd4526326a6add78b" + "a3dfe036599981665ac019ce1902283d6672522ee80d866a0fb62135af953cd9" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" ) UK_GATE_BATTERY_INPUT_MASS_EVIDENCE_SHA256 = ( - "df74556909990345bc9032f6f5db7f817d9c273f75522a85ab6e332dd8dc7355" + "806f46de90a0bf08c70c977ab63dad1ed644088c89e40df1869ce07b97f63c0c" ) #: Spec entry id -> (neutral gate name, phase, legacy detail-schema name). UK_GATE_BATTERY_ENTRIES = { From 2ac2a9ca3b3d5ba50793baf27046326c025dc479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:44:39 +0200 Subject: [PATCH 04/26] Review adjustments: reference label matches the pinned artifact; precise channel-blind scope note UK_REFERENCE_DATASET_NAME drops the _recalibrated suffix (adjudicated 2026-08-20): the pinned reference is the published enhanced_frs_2024_25.h5 itself and no recalibrated variant exists at this vintage, so the gate-report label now names the artifact exactly (June report strings keep their own label). The registry scope notes stop claiming the incumbent "structurally lacks" the SPI clone channel - the 2024-25 artifact carries the synthetic rows structurally but no admin-restored mass in the channel-exclusive columns, which is the fact the reviewed exclusions rest on; the approved exclusion reasons are untouched. Gate digests re-cut over the post-#729 union in the same change. Part of #723. Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/src/microcosm/build/uk/gates.json | 2 +- .../src/microcosm/build/uk_runtime/terminal_gates.py | 5 ++++- .../src/microcosm/build/uk_runtime/weighted_integrity.py | 7 ++++--- packages/microcosm-data/src/microcosm/data/contract.py | 6 +++--- packages/microcosm-data/tests/test_contract.py | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 0d73426efb..45f4cb9eba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -295,7 +295,7 @@ "vintage": "2024_25" }, "totals_sha256": "e70a45387c6adc13df5d7eb7da3c2cada7972a2f293a9238c8c29c9e885e4659", - "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; structurally lacks the SPI clone channel, so SPI-channel-exclusive columns are comparable only through per-reference reviewed exclusions." + "scope_note": "Channel-blind post-calibration enhanced-FRS production incumbent, pinned to the 2024-25 line; its artifact carries the SPI-synthetic rows structurally but no admin-restored mass in the SPI-channel-exclusive columns, so those columns are comparable only through per-reference reviewed exclusions." } }, "relative_tolerance": 4.521811483823806, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 8be29e192f..cefeeb86ec 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -70,7 +70,10 @@ ] UK_CANDIDATE_DATASET_NAME = "populace_uk_2023" -UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25_recalibrated" +# The label names the pinned reference artifact exactly: the 2024-25 line's +# published enhanced_frs_2024_25.h5 (no separate "recalibrated" variant +# exists at this vintage; the June report strings keep their own label). +UK_REFERENCE_DATASET_NAME = "enhanced_frs_2024_25" UK_MAX_TARGET_ABS_RELATIVE_ERROR = 0.25 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py index bc3b8f6aaf..e82962d62d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/weighted_integrity.py @@ -245,9 +245,10 @@ def spec_payload(self) -> dict[str, object]: totals_sha256=UK_INPUT_MASS_REFERENCE_EVIDENCE_SHA256, scope_note=( "Channel-blind post-calibration enhanced-FRS production incumbent, " - "pinned to the 2024-25 line; structurally lacks the SPI clone " - "channel, so SPI-channel-exclusive columns are comparable only " - "through per-reference reviewed exclusions." + "pinned to the 2024-25 line; its artifact carries the SPI-synthetic " + "rows structurally but no admin-restored mass in the " + "SPI-channel-exclusive columns, so those columns are comparable " + "only through per-reference reviewed exclusions." ), ) diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index d1ed7667f7..f5dbc34b20 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "a9b4887b08f628c86fc4ee679b41f050ed26f3c157cf424c9a02043d510509b5" + "20546f88fc2724c7ab4dcabf011d393ccb2229b0a256980bd8988b1ae44a300f" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "263569ea70081420a1dc06dcf262f6581f3225ebdaf464106ec3fcf571455fae" + "aed10d02f0c9c75d7c231278d5e8562a762b895f015735ae26df35f415302dde" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "a3dfe036599981665ac019ce1902283d6672522ee80d866a0fb62135af953cd9" + "3ba226748038e8fefcb25537a27e65cc34a1eebde1c9eb4abe754ba59fb13db6" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 7da9cb4bca..138791ae36 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -145,13 +145,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "a9b4887b08f628c86fc4ee679b41f050ed26f3c157cf424c9a02043d510509b5" + "20546f88fc2724c7ab4dcabf011d393ccb2229b0a256980bd8988b1ae44a300f" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "263569ea70081420a1dc06dcf262f6581f3225ebdaf464106ec3fcf571455fae" + "aed10d02f0c9c75d7c231278d5e8562a762b895f015735ae26df35f415302dde" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "a3dfe036599981665ac019ce1902283d6672522ee80d866a0fb62135af953cd9" + "3ba226748038e8fefcb25537a27e65cc34a1eebde1c9eb4abe754ba59fb13db6" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" From e9cbb2ed6aab5125225ad84b8b60f81758db8920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:57:04 +0200 Subject: [PATCH 05/26] Adversarial-review dispositions: dual source pins on the HMRC family block; self-describing June freeze Finding 1 (accepted): the family-coverage block rendered the #723 re-mapped period fields from the canonical manifest while hashing only the frozen mirror - evidence fields and their hash must name the same bytes. The block now carries a dual pin (source_manifest for the frozen June identity, canonical_source_manifest for the bytes the re-mapped fields come from) and a test binds each field set to the sha256 of the file it actually derives from. Finding 2 (rejected with armor): the committed replay report is the June evidence freeze and deliberately keeps mapped_build_period 2023 - it is evidence for the grandfathered release, not the 2024 line, and retires with the frozen manifest after #686 per #687; instead of regenerating it, a new assertion binds it to the FROZEN manifest's declared mapping so the partition is self-describing. Part of #723. Co-Authored-By: Claude Fable 5 --- .../uk/release_input_coverage_manifest.json | 2 + .../tests/test_uk_hmrc_replay_artifacts.py | 16 ++++++ ...test_uk_release_input_coverage_manifest.py | 51 +++++++++++++++++++ ...uild_uk_release_input_coverage_manifest.py | 7 +++ 4 files changed, 76 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index e38bf29d8c..7c166b3cb6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -531,6 +531,8 @@ "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", "calibration_permitted": false, + "canonical_source_manifest": "source_stages.json", + "canonical_source_manifest_sha256": "31818338ef62a19d9aebad7cdcf79af7cf05c2f0657ecc5e53b5954390ffa93b", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", diff --git a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py index 081a07486f..cf2c149c92 100644 --- a/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py +++ b/packages/microcosm-build/tests/test_uk_hmrc_replay_artifacts.py @@ -121,6 +121,22 @@ def test_real_replay_binds_sources_identity_and_positive_mass() -> None: } assert sources["hmrc_surface"]["sha256"] == HMRC_SPI_COLLATED_ODS_SHA256 assert sources["hmrc_surface"]["mapped_build_period"] == "2023" + # June-freeze partition, made self-describing (adversarial-review + # disposition, microcosm#723): this report is evidence for the + # grandfathered June release and binds to the FROZEN manifest's period + # mapping - it deliberately does NOT follow the live build period, which + # moved to "2024" with the #723 signed re-map. It retires with the frozen + # manifest after #686 (#687's disposition), never regenerates against a + # different vintage. + frozen_stage = _resource("hmrc_income_source_stages.json")["stages"][0] + frozen_surface = next( + artifact + for artifact in frozen_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + assert sources["hmrc_surface"]["mapped_build_period"] == str( + frozen_surface["mapped_build_period"] + ) assert qrf["fits"] == { "uk_frs_only_spi_fill": {"weight_kind": "importance"}, "uk_spi_2022_23_income": {"weight_kind": "design"}, diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py index b39143898d..c9e2d39040 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage_manifest.py @@ -126,6 +126,55 @@ def test_cached_candidate_regeneration_matches_committed_evidence() -> None: assert regenerated == committed +def test_hmrc_family_period_fields_come_from_the_bytes_their_hash_names() -> None: + # Adversarial-review finding (2026-08-20): the family block renders the + # #723 re-mapped period fields from the CANONICAL manifest while the + # frozen mirror keeps its June bytes; each field set must bind to the + # sha256 of the file it actually came from. + import hashlib + + manifest = _resource("release_input_coverage_manifest.json") + family = manifest["family_coverage"]["hmrc_spi_income"] + + frozen_bytes = ( + files(_UK_PACKAGE).joinpath("hmrc_income_source_stages.json").read_bytes() + ) + canonical_bytes = files(_UK_PACKAGE).joinpath("source_stages.json").read_bytes() + assert family["source_manifest_sha256"] == hashlib.sha256(frozen_bytes).hexdigest() + assert ( + family["canonical_source_manifest_sha256"] + == hashlib.sha256(canonical_bytes).hexdigest() + ) + + frozen_stage = json.loads(frozen_bytes)["stages"][0] + frozen_surface = next( + artifact + for artifact in frozen_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + canonical_stage = next( + stage + for stage in json.loads(canonical_bytes)["stages"] + if stage.get("stage") == "hmrc_spi_income" + ) + canonical_surface = next( + artifact + for artifact in canonical_stage["artifacts"] + if artifact.get("role") == "published_fact_surface" + ) + # The re-mapped fields equal the canonical declaration; the frozen mirror + # still declares the June mapping (its bytes are pinned elsewhere). + assert family["source_vintages"]["mapped_build_period"] == str( + canonical_surface["mapped_build_period"] + ) + assert ( + family["source_vintages"]["period_mapping"] + == canonical_surface["period_mapping"] + ) + assert str(frozen_surface["mapped_build_period"]) == "2023" + assert frozen_surface["period_mapping"] == "tax_year_start" + + def test_promoted_manifest_requires_the_full_reference_surface() -> None: reference = _resource("efrs_parity_reference.json") manifest = _resource("release_input_coverage_manifest.json") @@ -172,6 +221,8 @@ def test_hmrc_stage_is_required_while_the_208_fact_replay_remains_fenced() -> No "period_mapping": "latest_published_tax_year", } assert family["spi_prior_national_household_mass_share"] == 0.5 + assert family["canonical_source_manifest"] == "source_stages.json" + assert len(family["canonical_source_manifest_sha256"]) == 64 assert family["required_mass_change_reason"] == ( "Allocate 50% of certified UK national household prior mass to the " "rebuilt 2022-23 SPI support channel; total national mass is conserved." diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 0151aa33b3..f0b17562e1 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -1097,6 +1097,13 @@ def _hmrc_family_coverage_contract( "stage": "hmrc_spi_income", "source_manifest": HMRC_SOURCE_STAGES_PATH.name, "source_manifest_sha256": _sha256(HMRC_SOURCE_STAGES_PATH), + # The two re-mapped period fields below come from the CANONICAL + # manifest (the #723 signed re-map lives there; the frozen mirror + # keeps its June bytes), so the bytes they derive from are pinned + # separately - evidence fields and their hash must name the same + # source (adversarial-review finding, 2026-08-20). + "canonical_source_manifest": SOURCE_STAGES_PATH.name, + "canonical_source_manifest_sha256": _sha256(SOURCE_STAGES_PATH), "base_candidate_sha256": str(base_candidate["sha256"]), "base_candidate_tier": base_candidate_tier, "source_vintages": { From 3ec5e15e743c77a44032af0dbbea523469f21e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:22:26 +0200 Subject: [PATCH 06/26] Re-measure the UK spec-bundle and gate-battery digests on the merged tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both parents edit uk/gates.json and uk/country_package.json, so the merged manifest digests differ from either side's pins. Re-pinned by recomputation (never by picking a side): spec bundle e12a2cb8…, policy 404968fb…, gates manifest 59c7808d…, spec fingerprint bfb98736…. Co-Authored-By: Claude Fable 5 --- .../tests/test_spec_engine_country_bundles.py | 2 +- packages/microcosm-data/src/microcosm/data/contract.py | 6 +++--- packages/microcosm-data/tests/test_contract.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5b2fece4cd..617eff5599 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e65bbdfba881d64618713df765cf0ccd21c119b05a5b11ba7fbcf336065654dd", + "e12a2cb87c0e096af0173bd51fbede4b7df5e7c118ffe07ef616bbb30640e4ea", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 124f7b7da9..3e5190c2cf 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -344,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "47d41dc3643818e049034b4333392aeb352bed5f76fc4556c1da5ba2fe8fb86d" + "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "687bb4ae4b085ac9abf2e77a7a0c8bcf11204eb9b00fdac1fd7232e16f81161f" + "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e54e2f5880f2380854d9764dcd5482aaa51e37ca0915a3ffaa0c9897626ba656" + "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 268f6b8381..18a88423d0 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -145,13 +145,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "47d41dc3643818e049034b4333392aeb352bed5f76fc4556c1da5ba2fe8fb86d" + "404968fba9a626d4b534dfbef87721ff9d98c5af356758b2bab49dbaf004fdc3" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "687bb4ae4b085ac9abf2e77a7a0c8bcf11204eb9b00fdac1fd7232e16f81161f" + "59c7808d50a9ef84d37f524779a7518b4fb4f62dc7d17eb47e4a108d830c3798" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e54e2f5880f2380854d9764dcd5482aaa51e37ca0915a3ffaa0c9897626ba656" + "bfb987361037e6475ea9906894cb16e5b3cd0ff515096bd55d4918dfd7331d2c" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" From 2ccc963acca096a56120a0a54f372692f427ecfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:00:06 +0200 Subject: [PATCH 07/26] Land the #623 run-readiness tooling: doctrine, honest parity trio, US-format diagnostics, scorer, staging posture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Work items 1-7 of the narrowed #623 plan (runs held per the 2026-08-21 adjudications; everything here is synthetic/hermetic): - uk_runtime/national_doctrine.py: every solver constant the national calibration inherits becomes a declared, tamper-tested doctrine value (epochs 256, lr 0.02, ratio 10.0, seed 0, loss cap 10.0, l0 0.0, free mass, uniform weights). - Single compile path: the stage consumes the driver's compiled registry at the release calibration year and materializes bindings through the shared target_materialization interpreter; prepared scratch columns are restored post-solve, so the staged frame survives the real HDFStore writer (closes #729 dispositions finding 4). - Canonical mass/weight conventions mirroring the rowwise path: declared mass reason, post-solve fence (CALIBRATED kind, exactly one appended record), household_weight_kind_chain + calibration_mass_change manifest. - The parity trio evaluates for real on armed builds: candidate side from the staged frame and solve diagnostics, reference side from the frozen eFRS parity instrument and the declared registry at name@period grain — never a copied reference; unarmed builds keep evidence_absent. - calibration_diagnostics.json is now produced by the same shared diagnostics producer the US release path uses (per-epoch loss trajectory, per-target rows), with a pinned US<->UK format-parity test; the build block carries the chronicle artifact provenance (facts/manifest shas, profile ids) and reserves score_vs_enhanced_frs. - tools/score_uk_national_candidate.py: #578 rule-1 scoring, both artifacts rescored on one frozen register, June-schema score block with a declared none_declared holdout basis. - Declared non-certified staging-candidate input posture (sha-gated with a mid-read race guard, refused for release candidates) and the held-run runbook, unblocking on WS-E E8+E10 or an explicitly named base. Implemented by Codex from the reviewed plan; review pass fixed the materialization period (declared calibration year, never the frame's base-year time_period) and the parity-evidence sourcing above. Part of #623 under #665. Co-Authored-By: Claude Fable 5 --- .../623-uk-national-run-readiness.added.md | 3 + docs/uk-national-calibration-runbook-623.md | 52 +++ .../microcosm/build/target_materialization.py | 42 ++ .../microcosm/build/uk_runtime/__init__.py | 20 + .../build/uk_runtime/hmrc_restoration.py | 54 +++ .../build/uk_runtime/national_build.py | 72 ++- .../build/uk_runtime/national_calibration.py | 419 ++++++++++++------ .../build/uk_runtime/national_doctrine.py | 129 ++++++ .../tests/test_score_uk_national_candidate.py | 144 ++++++ .../tests/test_target_materialization.py | 46 +- .../tests/test_uk_diagnostics.py | 27 ++ .../tests/test_uk_national_build.py | 153 ++++++- .../tests/test_uk_national_build_driver.py | 89 ++++ .../tests/test_uk_national_calibration.py | 188 +++++++- tools/build_uk_national_dataset.py | 117 ++++- tools/score_uk_national_candidate.py | 183 ++++++++ 16 files changed, 1561 insertions(+), 177 deletions(-) create mode 100644 changelog.d/623-uk-national-run-readiness.added.md create mode 100644 docs/uk-national-calibration-runbook-623.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py create mode 100644 packages/microcosm-build/tests/test_score_uk_national_candidate.py create mode 100644 tools/score_uk_national_candidate.py diff --git a/changelog.d/623-uk-national-run-readiness.added.md b/changelog.d/623-uk-national-run-readiness.added.md new file mode 100644 index 0000000000..0ccf2d0383 --- /dev/null +++ b/changelog.d/623-uk-national-run-readiness.added.md @@ -0,0 +1,3 @@ +Add UK national Ledger-calibration run-readiness tooling, shared diagnostics, +synthetic incumbent scoring, and the held-run staging-candidate posture for +the first calibrated candidate. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md new file mode 100644 index 0000000000..23adfa15c8 --- /dev/null +++ b/docs/uk-national-calibration-runbook-623.md @@ -0,0 +1,52 @@ +# UK national calibration runbook (#623) + +This runbook documents the held licensed run for the first Ledger-calibrated UK +national candidate. It is not an instruction to run it in PR CI. + +## Unblock conditions + +Run only after one of these is true: + +- WS-E spine is complete through E8 (#684) and E10 (#686). +- Maria explicitly names a different base spine for this run. + +The input posture is a non-certified staging candidate at pre-clone grain, +matching the incumbent published national surface. The staging input must be +SHA-pinned with `--staging-candidate-input-sha256`; release-candidate posture +continues to require the certified input path. + +## Command shape + +```bash +uv run --no-sync python tools/build_uk_national_dataset.py \ + --input-h5 data/ukds/acceptance/623-first-calibrated-candidate/input-spine.h5 \ + --staging-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ + --staging-candidate-input-sha256 \ + --release-id populace-uk-2023-frs-k535080 \ + --calibration-diagnostics-sha256 \ + --national-calibration-diagnostics-json data/ukds/acceptance/623-first-calibrated-candidate/calibration_diagnostics.json \ + --frs-raw-dir \ + --spi-tab \ + --hmrc-ods \ + --cgt-ods \ + --ledger-facts \ + --ledger-facts-sha256 \ + --ledger-manifest-sha256 \ + --terminal-gates-json data/ukds/acceptance/623-first-calibrated-candidate/terminal_gates.json \ + --build-record-json data/ukds/acceptance/623-first-calibrated-candidate/build_record.json +``` + +## Evidence directory + +`data/ukds/acceptance/623-first-calibrated-candidate/` should contain: + +- `build_record.json` +- `terminal_gates.json` +- `calibration_diagnostics.json` +- calibration diagnostics SHA-256 receipt +- `score_vs_enhanced_frs.json` +- twin-build payload identity receipt + +Acceptance follows #578: the candidate must not regress incumbent battery +observables, and the score block decides rule 1. A rule-1 loss is evidence for +#686/#736, not a threshold-edit instruction. diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 2a3298f00e..2215aea188 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -74,6 +74,12 @@ def materialize_target_bindings( values = provider(adapter, binding, period) else: values = _prepared_column_values(adapter, spec.entity, binding) + values = _apply_household_conditions( + adapter, + spec.entity, + values, + binding.get("household_conditions", ()), + ) adapter.set_column(spec.entity, spec.measure, values) except (KeyError, TypeError, ValueError) as error: skipped.append( @@ -153,6 +159,7 @@ def _prepared_column_values( entity: str, binding: Mapping[str, Any], ) -> np.ndarray: + entity = str(binding.get("from_entity") or entity) if "value_expression" in binding: values = _expression(adapter, entity, str(binding["value_expression"])) else: @@ -165,6 +172,25 @@ def _prepared_column_values( return np.where(mask, values, 0.0) +def _apply_household_conditions( + adapter: Any, + entity: str, + values: np.ndarray, + predicates: object, +) -> np.ndarray: + conditions = tuple(predicates or ()) + if not conditions: + return values + mask = np.ones_like(values, dtype=bool) + for predicate in conditions: + if hasattr(adapter, "household_condition_mask"): + condition_mask = adapter.household_condition_mask(entity, predicate) + else: + condition_mask = _predicate_mask(adapter, entity, predicate) + mask &= np.asarray(condition_mask, dtype=bool) + return np.where(mask, values, 0.0) + + def _predicate_mask( adapter: Any, default_entity: str, @@ -181,6 +207,8 @@ def _compare(values: np.ndarray, operator: str, expected: object) -> np.ndarray: return values == expected if operator == "!=": return values != expected + if operator == "in": + return np.isin(values, list(expected)) numeric = values.astype(float) threshold = float(expected) if operator == ">": @@ -208,7 +236,21 @@ def _expression(adapter: Any, entity: str, expression: str) -> np.ndarray: def _column(adapter: Any, entity: str, variable: object) -> np.ndarray: + if str(variable) in {"person_count", "household_count", "benunit_count"}: + return np.ones(_entity_length(adapter, entity), dtype=float) if hasattr(adapter, "column"): return np.asarray(adapter.column(entity, str(variable)), dtype=float) table = adapter.tables[entity] return np.asarray(table[str(variable)], dtype=float) + + +def _entity_length(adapter: Any, entity: str) -> int: + if hasattr(adapter, "entity_length"): + return int(adapter.entity_length(entity)) + table = adapter.tables[entity] + if isinstance(table, Mapping): + first = next(iter(table.values())) + return len(first) + if hasattr(table, "__len__"): + return len(table) + raise TypeError(f"cannot determine row count for entity {entity!r}") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 79072bf57d..a7e4538103 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -305,6 +305,17 @@ load_uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UKNationalSolveDoctrine, +) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, UKStagingProvenance, @@ -639,8 +650,17 @@ "UKHMRCIncomeStageTransform", "UKHMRCTargetMaterialization", "UKNationalBuildResult", + "UKNationalSolveDoctrine", "UKStagingProvenance", + "UK_NATIONAL_L0_LAMBDA", + "UK_NATIONAL_LEARNING_RATE", + "UK_NATIONAL_MASS_RULE", + "UK_NATIONAL_MAX_WEIGHT_RATIO", "UK_NATIONAL_SCHEMA", + "UK_NATIONAL_SEED", + "UK_NATIONAL_SOLVE_DOCTRINE", + "UK_NATIONAL_SOLVE_EPOCHS", + "UK_NATIONAL_TARGET_LOSS_CAP", "UKNationalStage", "UKRowwiseDatasetResult", "UKReleaseInputColumn", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py index d67b823ff1..1f02c66554 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py @@ -12,6 +12,7 @@ from __future__ import annotations import hashlib +import re from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path @@ -91,6 +92,8 @@ "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" ) CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 +STAGING_CANDIDATE_TIER = "staging_candidate" +STAGING_CANDIDATE_REVISION = "non_certified_staging_candidate" _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN = object() @@ -494,6 +497,50 @@ def verify_certified_uk_candidate(path: str | Path) -> UKCertifiedCandidateIdent return identity +def verify_staging_candidate_uk_input( + path: str | Path, + *, + expected_sha256: str, +) -> UKCertifiedCandidateIdentity: + """Hash-gate a declared non-certified staging-candidate input.""" + + candidate = Path(path).expanduser().resolve() + if not candidate.is_file(): + raise FileNotFoundError(f"UK staging-candidate input not found: {candidate}.") + expected = str(expected_sha256) + if not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ValueError( + "--staging-candidate-input-sha256 must be a lowercase SHA-256." + ) + fingerprint_before = _uk_source_file_fingerprint(candidate) + digest = _sha256(candidate) + fingerprint_after = _uk_source_file_fingerprint(candidate) + if fingerprint_after != fingerprint_before: + raise RuntimeError( + "UK staging-candidate input changed while its SHA-256 was being verified." + ) + if digest != expected: + raise ValueError( + f"{candidate}: sha256 {digest} does not match declared " + f"staging-candidate input {expected}." + ) + identity = UKCertifiedCandidateIdentity( + path=candidate, + filename=candidate.name, + tier=STAGING_CANDIDATE_TIER, + revision=STAGING_CANDIDATE_REVISION, + sha256=digest, + size_bytes=fingerprint_after.size_bytes, + ) + object.__setattr__( + identity, + "_verification_token", + _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN, + ) + object.__setattr__(identity, "_source_file_fingerprint", fingerprint_after) + return identity + + def restore_uk_hmrc_income_family( frame: Frame, *, @@ -801,6 +848,13 @@ def _validate_certified_candidate_identity( identity.sha256, identity.size_bytes, ) + if identity.tier == STAGING_CANDIDATE_TIER: + if identity.revision != STAGING_CANDIDATE_REVISION: + raise ValueError( + "UK staging-candidate input identity has an invalid revision " + f"{identity.revision!r}." + ) + return if actual != expected: raise ValueError( "HMRC replay base identity does not match the certified Microcosm UK " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index d8a16a8972..bc2c5a2da3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -24,6 +24,7 @@ from dataclasses import dataclass from datetime import date from pathlib import Path +from types import SimpleNamespace from typing import Any import pandas as pd @@ -57,6 +58,7 @@ UK_SAMPLE_SEED_DEFAULT, sample_uk_national_frame, ) +from microcosm.build.uk_runtime.parity_reference import EfrsParityReference from microcosm.build.uk_runtime.release_input_coverage import ( PolicyEngineUKCoverageEngine, ) @@ -360,6 +362,7 @@ def build_uk_national_dataset( now: date | None = None, gate_registry: Mapping[str, GateBinding] | None = None, ledger_target_registry: Mapping[int | str, TargetRegistry] | None = None, + parity_reference: EfrsParityReference | None = None, ) -> UKNationalBuildResult: """Run ordered national stages, hard-gate the result, and stage an H5. @@ -552,6 +555,13 @@ def build_uk_national_dataset( calibration_evidence = _stage_calibration_evidence(materialized_stages) if calibration_evidence is not None: artifacts["national_calibration"] = calibration_evidence + parity_evidence = _stage_parity_evidence( + materialized_stages, + frame=frame, + parity_reference=parity_reference, + ) + if parity_evidence is not None: + artifacts["parity_evidence"] = parity_evidence if input_mass_reference is not None: artifacts["input_mass_reference"] = input_mass_reference if reviewed_input_mass_exclusions is not None: @@ -825,7 +835,67 @@ def _stage_calibration_evidence( for stage in stages: if stage.name == "national_calibration": manifest = getattr(stage.transform, "manifest", None) - return dict(manifest) if isinstance(manifest, Mapping) else {} + if not isinstance(manifest, Mapping): + raise RuntimeError( + "national_calibration stage did not produce a manifest; " + "refusing to build calibration gate evidence." + ) + return dict(manifest) + return None + + +def _stage_parity_evidence( + stages: tuple[PlanStage, ...], + *, + frame: Frame, + parity_reference: EfrsParityReference | None, +) -> object | None: + """Build the parity-trio evidence from independently sourced sides. + + The candidate side comes from the staged frame and the solve diagnostics; + the reference side comes from the frozen parity instrument and the + declared registry. The two sides must never alias each other — a copied + reference would make the trio pass by construction. + """ + + for stage in stages: + if stage.name != "national_calibration": + continue + if parity_reference is None: + return None + diagnostics = getattr(stage.transform, "diagnostics", None) + if not isinstance(diagnostics, tuple): + raise RuntimeError( + "national_calibration stage did not produce target diagnostics; " + "refusing to build parity evidence." + ) + registry = getattr(stage.transform, "registry", None) + specs = getattr(registry, "specs", None) + if specs is None: + raise RuntimeError( + "national_calibration stage carries no compiled registry; " + "refusing to build parity evidence." + ) + target_relative_errors = { + str(row["name"]): float(row["relative_error"]) for row in diagnostics + } + return SimpleNamespace( + candidate_columns={ + f"{entity}.{column}" + for entity in frame.entities + for column in frame.table(entity).columns + }, + reference_columns={ + f"{entity}.{name}" + for name, entity in parity_reference.input_entities.items() + }, + candidate_targets=set(target_relative_errors), + # Solver diagnostics label rows as ``name@period``; the declared + # side is compared at the same labeled grain so a target bound at + # the wrong period cannot satisfy the surface. + reference_targets={f"{spec.name}@{spec.period}" for spec in specs}, + target_relative_errors=target_relative_errors, + ) return None diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index e56cd725c1..e09dc59b40 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -2,20 +2,35 @@ from __future__ import annotations -import json from collections.abc import Mapping, Sequence -from importlib import resources from typing import Any +import numpy as np import pandas as pd -from microcosm.build.ledger_targets import compile_ledger_target_references from microcosm.build.plan import Stage from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity -from microcosm.calibrate import calibrate, effective_sample_size -from microcosm.frame import Frame +from microcosm.build.uk_runtime.ledger_targets import ( + UKLedgerTargetCompilation, + materialize_uk_ledger_targets, +) +from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_SOLVE_DOCTRINE, + UKNationalSolveDoctrine, +) +from microcosm.calibrate import ( + CalibrationResult, + TargetRegistry, + calibrate, + effective_sample_size, +) +from microcosm.frame import Frame, WeightKind, Weights -__all__ = ["UKNationalCalibrationStage", "uk_national_calibration_stage"] +__all__ = [ + "UKNationalCalibrationStage", + "national_calibration_mass_reason", + "uk_national_calibration_stage", +] class UKNationalCalibrationStage: @@ -23,50 +38,69 @@ class UKNationalCalibrationStage: def __init__( self, - facts: Sequence[Mapping[str, Any]], + registry: UKLedgerTargetCompilation | TargetRegistry, *, - references: Sequence[object] | None = None, - epochs: int = 256, - learning_rate: float = 0.02, - max_weight_ratio: float = 10.0, - seed: int = 0, + period: int, + doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, ) -> None: - from microcosm.build.country_spec import load_country_spec - - self.facts = tuple(facts) - self.references = tuple( - load_country_spec("uk").target_references - if references is None - else references + self.compilation = ( + registry + if isinstance(registry, UKLedgerTargetCompilation) + else UKLedgerTargetCompilation(registry=registry, unsupported=()) ) - self.epochs = epochs - self.learning_rate = learning_rate - self.max_weight_ratio = max_weight_ratio - self.seed = seed + self.registry = self.compilation.registry + # The materialization period is the declared calibration year the + # registry was compiled at — never the input frame's base-year + # time_period, which lags it (survey 2024, calibration 2025). + if not isinstance(period, int) or isinstance(period, bool) or period <= 0: + raise ValueError( + f"period must be the declared calibration year, got {period!r}." + ) + self.period = period + self.doctrine = doctrine self.manifest: dict[str, object] | None = None self.diagnostics: tuple[dict[str, object], ...] = () + self.solve_result: CalibrationResult | None = None self.output_content_identity: str | None = None def __call__(self, frame: Frame) -> Frame: - registry = compile_ledger_target_references( - self.facts, self.references, country="uk" - ) - declared = len(self.references) - resolved = len(registry.specs) - if resolved != declared: + declared = len(self.registry.specs) + len(self.compilation.unsupported) + resolved = len(self.registry.specs) + if self.compilation.unsupported: raise RuntimeError( "UK national calibration resolved " - f"{resolved} of {declared} activated target references." + f"{resolved} of {declared} activated target references; " + f"unsupported={self.compilation.unsupported!r}." ) - prepared = _prepare_target_columns(frame, registry.specs) + adapter = _FrameTargetAdapter(frame) + materialized = materialize_uk_ledger_targets( + adapter, + self.registry, + period=self.period, + ) + if materialized.skipped: + skipped = [skip.__dict__ for skip in materialized.skipped] + raise RuntimeError( + "UK national calibration could not materialize every activated " + f"target reference: skipped={skipped}." + ) + prepared = adapter.frame() + mass_reason = national_calibration_mass_reason( + spec.family for spec in self.registry.specs + ) + mass_log_records_before_calibration = len(frame.mass_log) result = calibrate( prepared, - registry.to_target_set(), + self.registry.to_target_set(), weight_entity="household", - epochs=self.epochs, - learning_rate=self.learning_rate, - max_weight_ratio=self.max_weight_ratio, - seed=self.seed, + epochs=self.doctrine.epochs, + learning_rate=self.doctrine.learning_rate, + mass=self.doctrine.mass_rule, + mass_reason=mass_reason, + max_weight_ratio=self.doctrine.max_weight_ratio, + seed=self.doctrine.seed, + l0_lambda=self.doctrine.l0_lambda, + target_loss_cap=self.doctrine.target_loss_cap, ) if result.skipped or len(result.problem.names) != declared: skipped = [item.name for item in result.skipped] @@ -75,6 +109,13 @@ def __call__(self, frame: Frame) -> Frame: f"reference: declared={declared}, rows={len(result.problem.names)}, " f"skipped={skipped}." ) + clean_frame = adapter.restore(result.frame) + self.solve_result = result + calibration_record = _post_solve_calibration_record( + frame, + clean_frame, + before_count=mass_log_records_before_calibration, + ) self.diagnostics = tuple( { "name": row.name, @@ -85,6 +126,10 @@ def __call__(self, frame: Frame) -> Frame: for row in result.diagnostics ) ratios = result.weights / result.initial_weights + old_total = float(calibration_record.old_total) + new_total = float(calibration_record.new_total) + before_kind = frame.weights_for("household").kind + after_kind = clean_frame.weights_for("household").kind self.manifest = { "activated_reference_count": declared, "resolved_reference_count": resolved, @@ -92,10 +137,38 @@ def __call__(self, frame: Frame) -> Frame: "loss": result.final_loss, "effective_sample_size": effective_sample_size(result.weights), "max_weight_ratio": float(ratios.max()), - "max_weight_ratio_bound": self.max_weight_ratio, + "max_weight_ratio_bound": self.doctrine.max_weight_ratio, + "target_materialization": materialized.report(), + "weights": { + "household_weight_kind": after_kind.value, + "household_weight_kind_chain": [ + {"stage": "staging", "kind": before_kind.value}, + {"stage": "national_calibration", "kind": after_kind.value}, + ], + "mass_log_records_before_calibration": ( + mass_log_records_before_calibration + ), + "mass_log_records": len(clean_frame.mass_log), + "calibration_mass_change": { + "entity": str(calibration_record.entity), + "old_total": old_total, + "new_total": new_total, + "relative_shift": (new_total - old_total) / old_total, + "declared_factor": calibration_record.declared_factor, + "reason": str(calibration_record.reason), + }, + }, + "solve": { + "n_targets": len(result.problem.names), + "n_households": len(clean_frame.table("household")), + "initial_loss": float(result.initial_loss), + "final_loss": float(result.final_loss), + "n_nonzero": int(np.count_nonzero(result.weights)), + }, + "parameters": {"doctrine": _doctrine_bounds(self.doctrine)}, } - self.output_content_identity = uk_frame_content_identity(result.frame) - return result.frame + self.output_content_identity = uk_frame_content_identity(clean_frame) + return clean_frame def checkpoint_metadata(self) -> Mapping[str, object]: if self.manifest is None: @@ -147,81 +220,192 @@ def resume_from_checkpoint( def uk_national_calibration_stage( - facts: Sequence[Mapping[str, Any]], **kwargs: Any + registry: UKLedgerTargetCompilation, **kwargs: Any ) -> Stage: """Return the named ordered-stage entry for national calibration.""" - transform = UKNationalCalibrationStage(facts, **kwargs) + transform = UKNationalCalibrationStage(registry, **kwargs) return Stage(name="national_calibration", transform=transform) -def _contract_targets() -> dict[str, Mapping[str, Any]]: - payload = json.loads( - resources.files("microcosm.build.uk") - .joinpath("uk_national_targets.json") - .read_text() +def national_calibration_mass_reason(bound_families: Sequence[str]) -> str: + """The mass-record reason a national doctrine calibration declares.""" + + families = sorted({str(name) for name in bound_families}) + if not families or any(not name.strip() for name in families): + raise ValueError("bound_families must name at least one target family.") + return ( + "National doctrine calibration to bound target family(ies) " + f"{', '.join(families)}; total household mass moved with the targets." ) - return {row["target_id"]: row for row in payload["targets"]} - - -def _prepare_target_columns(frame: Frame, specs: Sequence[object]) -> Frame: - tables = {entity: frame.table(entity).copy() for entity in frame.entities} - tables.update({name: frame.link(name).copy() for name in frame.links}) - contracts = _contract_targets() - for spec in specs: - target_id = spec.metadata["contract_target_id"] - binding = contracts[target_id]["bindings"]["policyengine"] - entity = spec.entity - table = tables[entity] - if spec.measure in table: - continue - value_name = binding["value_variable"] - if value_name in {"person_count", "household_count", "benunit_count"}: - values = pd.Series(1.0, index=table.index) + + +def _post_solve_calibration_record( + before: Frame, + after: Frame, + *, + before_count: int, +): + if after.weights_for("household").kind is not WeightKind.CALIBRATED: + raise RuntimeError( + "UK national calibration returned household weights whose kind is " + f"{after.weights_for('household').kind.value!r}, not 'calibrated'." + ) + if len(after.mass_log) != before_count + 1: + raise RuntimeError( + "UK national calibration must append exactly one mass record; " + f"before={before_count}, after={len(after.mass_log)}." + ) + if before.mass_log != after.mass_log[:before_count]: + raise RuntimeError( + "UK national calibration changed pre-existing mass-log records." + ) + record = after.mass_log[-1] + if record.entity != "household" or "calibration" not in record.reason: + raise RuntimeError( + "UK national calibration latest mass record is not the calibration " + f"record: entity={record.entity!r}, reason={record.reason!r}." + ) + return record + + +def _doctrine_bounds(doctrine: UKNationalSolveDoctrine) -> dict[str, object]: + return { + "epochs": doctrine.epochs, + "learning_rate": doctrine.learning_rate, + "max_weight_ratio": doctrine.max_weight_ratio, + "seed": doctrine.seed, + "target_loss_cap": doctrine.target_loss_cap, + "scale_rule": doctrine.scale_rule, + "target_weight_rule": doctrine.target_weight_rule, + "mass_rule": doctrine.mass_rule, + "l0_lambda": doctrine.l0_lambda, + } + + +class _FrameTargetAdapter: + def __init__(self, frame: Frame) -> None: + self._frame = frame + self.tables = {entity: frame.table(entity).copy() for entity in frame.entities} + self.tables.update({name: frame.link(name).copy() for name in frame.links}) + self._original_tables = { + name: table.copy() for name, table in self.tables.items() + } + self._scratch_columns: dict[str, set[str]] = { + name: set() for name in self.tables + } + + def entity_length(self, entity: str) -> int: + return len(self.tables[entity]) + + def column(self, entity: str, variable: str) -> np.ndarray: + if variable not in self.tables[entity]: + raise KeyError(variable) + return np.asarray(self.tables[entity][variable]) + + def set_column(self, entity: str, variable: str, values: object) -> None: + table = self.tables[entity] + if variable not in self._original_tables[entity]: + self._scratch_columns[entity].add(variable) + table[variable] = np.asarray(values, dtype=float) + + def parameter(self, parameter: str, period: int | str) -> float: + from microcosm.build.uk_runtime.ledger_targets import UKPolicyEngineAdapter + + return UKPolicyEngineAdapter(None).parameter(parameter, period) + + def household_condition_mask( + self, + target_entity: str, + condition: Mapping[str, Any], + ) -> np.ndarray: + if target_entity != "household": + raise ValueError( + "household_conditions can only gate household-grain targets, " + f"got {target_entity!r}." + ) + entity = str(condition.get("entity") or "household") + source = self.tables[entity] + if entity == "household": + household_ids = source["household_id"] else: - if value_name not in table: - raise ValueError( - f"Activated UK target {target_id!r} requires missing " - f"{entity} column {value_name!r}." - ) - values = table[value_name].astype(float) - mask = pd.Series(True, index=table.index) - for condition in binding.get("filters", ()): - variable = condition["variable"] - if variable not in table: - raise ValueError( - f"Activated UK target {target_id!r} requires missing " - f"{entity} filter column {variable!r}." - ) - mask &= _compare(table[variable], condition) - if binding.get("household_conditions"): - if entity != "household": - raise ValueError( - f"Activated UK target {target_id!r} declares household " - f"conditions on entity {entity!r}." + household_ids = self._group_household_ids(entity) + reduce = str(condition.get("reduce") or "any") + if reduce in {"any", "any_child_under"}: + matched = _compare_series(source[str(condition["variable"])], condition) + aggregate = matched.groupby(household_ids).any().astype(float) + comparison = {"operator": "==", "value": True} + elif reduce == "sum": + aggregate = source[str(condition["variable"])].groupby(household_ids).sum() + comparison = condition + elif reduce == "count": + aggregate = ( + source[str(condition["variable"])].groupby(household_ids).count() + ) + comparison = condition + else: + raise ValueError(f"Unsupported UK household reduction {reduce!r}.") + values = self.tables["household"]["household_id"].map(aggregate).fillna(0.0) + return np.asarray(_compare_series(values, comparison), dtype=bool) + + def frame(self) -> Frame: + return Frame( + self.tables, + self._frame.schema, + { + entity: self._frame.weights_for(entity) + for entity in self._frame.weighted_entities + }, + self._frame.strata, + mass_log=self._frame.mass_log, + metadata=self._frame.metadata, + ) + + def restore(self, calibrated: Frame) -> Frame: + tables = {name: table.copy() for name, table in self._original_tables.items()} + return Frame( + tables, + calibrated.schema, + { + entity: Weights( + calibrated.weights_for(entity).values, + calibrated.weights_for(entity).kind, ) - for condition in binding["household_conditions"]: - mask &= _household_condition(tables, condition, table) - table[spec.measure] = values.where(mask, 0.0) - weights = {entity: frame.weights_for(entity) for entity in frame.weighted_entities} - return Frame( - tables, - frame.schema, - weights, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) + for entity in calibrated.weighted_entities + }, + calibrated.strata, + mass_log=calibrated.mass_log, + metadata=calibrated.metadata, + ) + def _group_household_ids(self, entity: str) -> pd.Series: + people = self.tables["person"] + entity_membership = f"person_{entity}_id" + if entity_membership not in people: + raise ValueError(f"UK person table is missing {entity_membership!r}.") + group_to_household = ( + people[[entity_membership, "person_household_id"]] + .drop_duplicates() + .set_index(entity_membership)["person_household_id"] + ) + if group_to_household.index.has_duplicates: + raise ValueError(f"UK {entity} groups span multiple households.") + return self.tables[entity][f"{entity}_id"].map(group_to_household) -def _compare(values: pd.Series, condition: Mapping[str, Any]) -> pd.Series: + +def _compare_series(values: pd.Series, condition: Mapping[str, Any]) -> pd.Series: operator = condition.get("operator") if operator is None: operator, expected = "==", condition["equals"] else: expected = condition["value"] + if operator == "==": + return values.eq(expected) + if operator == "!=": + return values.ne(expected) + if operator == "in": + return values.isin(expected) operations = { - "==": values.eq, ">": values.gt, ">=": values.ge, "<": values.lt, @@ -230,44 +414,3 @@ def _compare(values: pd.Series, condition: Mapping[str, Any]) -> pd.Series: if operator not in operations: raise ValueError(f"Unsupported UK target condition operator {operator!r}.") return operations[operator](expected) - - -def _household_condition( - tables: Mapping[str, pd.DataFrame], - condition: Mapping[str, Any], - households: pd.DataFrame, -) -> pd.Series: - entity = condition["entity"] - source = tables[entity] - if entity == "household": - household_ids = source["household_id"] - else: - people = tables["person"] - entity_membership = f"person_{entity}_id" - if entity_membership not in people: - raise ValueError(f"UK person table is missing {entity_membership!r}.") - group_to_household = ( - people[[entity_membership, "person_household_id"]] - .drop_duplicates() - .set_index(entity_membership)["person_household_id"] - ) - if group_to_household.index.has_duplicates: - raise ValueError(f"UK {entity} groups span multiple households.") - household_ids = source[f"{entity}_id"].map(group_to_household) - reduce = condition["reduce"] - if reduce == "any": - matched = _compare(source[condition["variable"]], condition) - aggregate = matched.groupby(household_ids).any().astype(float) - expected_condition = {"operator": "==", "value": True} - elif reduce == "sum": - aggregate = source[condition["variable"]].groupby(household_ids).sum() - expected_condition = condition - elif reduce == "count": - aggregate = source[condition["variable"]].groupby(household_ids).count() - expected_condition = condition - else: - raise ValueError(f"Unsupported UK household reduction {reduce!r}.") - ids = households["household_id"] - return _compare(ids.map(aggregate).fillna(0.0), expected_condition).set_axis( - households.index - ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py new file mode 100644 index 0000000000..40687602dc --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py @@ -0,0 +1,129 @@ +"""UK national calibration doctrine: reviewed solve constants for #623. + +The national release path exposes no calibration knobs. Every solver option +that the first Ledger-backed national calibration currently inherits is +declared here so a future change must edit this module and its pinned tests, +forcing review before any armed run. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +__all__ = [ + "UK_NATIONAL_LEARNING_RATE", + "UK_NATIONAL_L0_LAMBDA", + "UK_NATIONAL_MASS_RULE", + "UK_NATIONAL_MAX_WEIGHT_RATIO", + "UK_NATIONAL_SEED", + "UK_NATIONAL_SOLVE_DOCTRINE", + "UK_NATIONAL_SOLVE_EPOCHS", + "UK_NATIONAL_TARGET_LOSS_CAP", + "UKNationalSolveDoctrine", +] + +UK_NATIONAL_SOLVE_EPOCHS = 256 +UK_NATIONAL_LEARNING_RATE = 0.02 +UK_NATIONAL_MAX_WEIGHT_RATIO = 10.0 +UK_NATIONAL_SEED = 0 +UK_NATIONAL_TARGET_LOSS_CAP = 10.0 +UK_NATIONAL_L0_LAMBDA = 0.0 +UK_NATIONAL_MASS_RULE = "free" + +_ALLOWED_SCALE_RULES = ("default_target_loss_scales",) +_ALLOWED_TARGET_WEIGHT_RULES = ("uniform",) +_ALLOWED_MASS_RULES = ("free",) + + +@dataclass(frozen=True) +class UKNationalSolveDoctrine: + """The declared constants of the UK national calibration solve.""" + + epochs: int = UK_NATIONAL_SOLVE_EPOCHS + learning_rate: float = UK_NATIONAL_LEARNING_RATE + max_weight_ratio: float | None = UK_NATIONAL_MAX_WEIGHT_RATIO + seed: int = UK_NATIONAL_SEED + target_loss_cap: float = UK_NATIONAL_TARGET_LOSS_CAP + scale_rule: str = "default_target_loss_scales" + target_weight_rule: str = "uniform" + mass_rule: str = UK_NATIONAL_MASS_RULE + l0_lambda: float = UK_NATIONAL_L0_LAMBDA + + def __post_init__(self) -> None: + if ( + not isinstance(self.epochs, int) + or isinstance(self.epochs, bool) + or self.epochs <= 0 + ): + raise ValueError( + f"doctrine epochs must be a positive integer, got {self.epochs!r}." + ) + if ( + not isinstance(self.learning_rate, int | float) + or isinstance(self.learning_rate, bool) + or not np.isfinite(self.learning_rate) + or self.learning_rate <= 0 + ): + raise ValueError( + "doctrine learning_rate must be a positive finite number, " + f"got {self.learning_rate!r}." + ) + if self.max_weight_ratio is not None and ( + not isinstance(self.max_weight_ratio, int | float) + or isinstance(self.max_weight_ratio, bool) + or not np.isfinite(self.max_weight_ratio) + or self.max_weight_ratio <= 1 + ): + raise ValueError( + "doctrine max_weight_ratio must be None or a finite number " + f"greater than 1, got {self.max_weight_ratio!r}." + ) + if ( + not isinstance(self.seed, int) + or isinstance(self.seed, bool) + or self.seed < 0 + ): + raise ValueError( + f"doctrine seed must be a non-negative integer, got {self.seed!r}." + ) + if ( + not isinstance(self.target_loss_cap, int | float) + or isinstance(self.target_loss_cap, bool) + or not np.isfinite(self.target_loss_cap) + or self.target_loss_cap <= 0 + ): + raise ValueError( + "doctrine target_loss_cap must be a positive finite number, " + f"got {self.target_loss_cap!r}." + ) + if self.scale_rule not in _ALLOWED_SCALE_RULES: + raise ValueError( + f"doctrine scale_rule must be one of {_ALLOWED_SCALE_RULES}, " + f"got {self.scale_rule!r}." + ) + if self.target_weight_rule not in _ALLOWED_TARGET_WEIGHT_RULES: + raise ValueError( + "doctrine target_weight_rule must be one of " + f"{_ALLOWED_TARGET_WEIGHT_RULES}, got " + f"{self.target_weight_rule!r}." + ) + if self.mass_rule not in _ALLOWED_MASS_RULES: + raise ValueError( + f"doctrine mass_rule must be one of {_ALLOWED_MASS_RULES}, " + f"got {self.mass_rule!r}." + ) + if ( + not isinstance(self.l0_lambda, int | float) + or isinstance(self.l0_lambda, bool) + or not np.isfinite(self.l0_lambda) + or self.l0_lambda < 0 + ): + raise ValueError( + "doctrine l0_lambda must be a non-negative finite number, " + f"got {self.l0_lambda!r}." + ) + + +UK_NATIONAL_SOLVE_DOCTRINE = UKNationalSolveDoctrine() diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py new file mode 100644 index 0000000000..36961df295 --- /dev/null +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.national_build import write_uk_national_frame +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind +from tools.score_uk_national_candidate import ( + _load_registry, + main, + score_uk_national_candidate, +) + + +def _registry() -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="target_a", + entity="household", + measure="measure_a", + value=10.0, + source="synthetic", + family="family_a", + ), + TargetSpec( + name="target_b", + entity="household", + measure="measure_b", + value=20.0, + source="synthetic", + family="family_b", + ), + ], + country="uk", + ) + + +def _write(path, *, measure_a, measure_b) -> None: + n = 2 + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_household_id": np.arange(n), + "person_benunit_id": np.arange(n), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = pd.DataFrame( + { + "household_id": np.arange(n), + "measure_a": measure_a, + "measure_b": measure_b, + } + ) + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2025", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.ones(n), + ) + write_uk_national_frame(frame, path) + + +def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + + score = score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + target_registry=_registry(), + ) + + assert score["candidate_full_loss"] == pytest.approx(0.25) + assert score["incumbent_full_loss"] == pytest.approx(0.1) + assert score["candidate_train_loss"] == score["candidate_full_loss"] + assert score["candidate_holdout_loss"] == score["candidate_full_loss"] + assert score["holdout_basis"] == "none_declared" + assert score["candidate_target_wins"] == 1 + assert score["incumbent_target_wins"] == 1 + assert score["target_wins_by_family"] == { + "family_a": { + "candidate_target_wins": 1, + "incumbent_target_wins": 0, + "ties": 0, + }, + "family_b": { + "candidate_target_wins": 0, + "incumbent_target_wins": 1, + "ties": 0, + }, + } + + +def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + registry_json = tmp_path / "registry.json" + output_json = tmp_path / "score.json" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + registry_json.write_text( + json.dumps( + { + "country": "uk", + "specs": [ + asdict for asdict in (_spec.__dict__ for _spec in _registry().specs) + ], + } + ), + encoding="utf-8", + ) + + assert ( + main( + [ + "--candidate-h5", + str(candidate), + "--incumbent-h5", + str(incumbent), + "--target-registry-json", + str(registry_json), + "--output-json", + str(output_json), + ] + ) + == 0 + ) + + payload = json.loads(output_json.read_text(encoding="utf-8")) + assert payload["score_vs_enhanced_frs"]["holdout_basis"] == "none_declared" + assert _load_registry(registry_json).version == _registry().version diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 7bcf2bd211..2846d26b68 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -170,9 +170,7 @@ def test_missing_materialization_inputs_are_reported_as_skips(): country="uk", ) contract = { - "missing": { - "bindings": {"policyengine": {"value_variable": "not_present"}} - } + "missing": {"bindings": {"policyengine": {"value_variable": "not_present"}}} } result = materialize_target_bindings(StubAdapter(), registry, contract, period=2025) @@ -180,3 +178,45 @@ def test_missing_materialization_inputs_are_reported_as_skips(): assert len(result.skipped) == 1 assert result.skipped[0].name == "missing" assert result.report()["skipped_count"] == 1 + + +def test_count_aliases_and_in_predicates_materialize_prepared_columns(): + registry = TargetRegistry( + [ + TargetSpec( + name="affected_households", + entity="household", + measure="affected_households_measure", + value=2.0, + source="test", + metadata={"contract_target_id": "affected_households"}, + ) + ], + country="uk", + ) + contract = { + "affected_households": { + "bindings": { + "policyengine": { + "value_variable": "household_count", + "filters": [ + { + "variable": "children", + "operator": "in", + "value": [3.0, 4.0], + } + ], + } + } + } + } + adapter = StubAdapter() + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + assert adapter.tables["household"]["affected_households_measure"].tolist() == [ + 1.0, + 0.0, + 1.0, + ] diff --git a/packages/microcosm-build/tests/test_uk_diagnostics.py b/packages/microcosm-build/tests/test_uk_diagnostics.py index 9dc8f65c79..af25c584f5 100644 --- a/packages/microcosm-build/tests/test_uk_diagnostics.py +++ b/packages/microcosm-build/tests/test_uk_diagnostics.py @@ -21,6 +21,7 @@ CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, TargetRegistry, TargetSpec, + diagnostics_payload, score_targets, ) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights @@ -344,6 +345,32 @@ def test_payload_preserves_common_schema_and_adds_versioned_uk_evidence() -> Non assert json.loads(json.dumps(payload, allow_nan=False)) == payload +def test_uk_payload_shared_layer_matches_shared_diagnostics_format() -> None: + result, frame, registry, geography = _diagnostics_case() + build = {"release_id": "fixture"} + + shared = diagnostics_payload(result, target_registry=registry, build=build) + uk = uk_calibration_diagnostics_payload( + result, + frame, + target_geography_levels=geography, + target_registry=registry, + build=build, + ) + + uk_shared_layer = { + key: value for key, value in uk.items() if key != "uk_diagnostics" + } + assert uk_shared_layer == shared + assert set(uk) == {*shared.keys(), "uk_diagnostics"} + assert set(uk["uk_diagnostics"]) == { + "schema_version", + "weights", + "zero_weight_rows_by_stratum", + "target_pass_rates_by_geography_level", + } + + def test_payload_requires_exact_explicit_geography_mapping() -> None: result, frame, registry, geography = _diagnostics_case() missing = dict(geography) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index e8edf9ea7e..53815df963 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -31,15 +31,21 @@ from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, ) +from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( _uk_gate_surface, uk_household_weight_kind, uk_national_frame, uk_time_period, ) +from microcosm.build.uk_runtime.parity_reference import ( + EfrsParityReference, + EfrsParitySource, +) from microcosm.build.uk_runtime.release_input_coverage import ( uk_release_input_coverage_gate, ) +from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import Frame, MassChangeRecord, WeightKind TEST_UK_RELEASE_ID = "populace-uk-2023-frs-k535080" @@ -355,6 +361,13 @@ def _registry_with_calibration() -> dict[str, UKGateBinding]: return registry +def _registry_with_calibration_and_parity_trio() -> dict[str, UKGateBinding]: + registry = _registry_with_calibration() + for name in ("export_surface", "target_surface", "target_fit"): + registry[name] = UK_GATE_REGISTRY[name] + return registry + + def _uc_reference(**overrides) -> LedgerTargetReference: values = { "name": "dwp.uc.households", @@ -390,6 +403,22 @@ def _calibration_fact(value: float = 60.0) -> dict: } +def _calibration_registry(value: float = 60.0) -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=value, + source="test", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + def test_driver_validates_the_uk_residue_after_each_stage( monkeypatch, tmp_path ) -> None: @@ -523,16 +552,16 @@ def test_resumed_national_calibration_feeds_reference_coverage_gate( _write_two_row_h5(input_h5, include_calibration_columns=True) frame, _provenance = load_uk_national_frame(input_h5) stage = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) staged = stage(frame) metadata = json.loads(json.dumps(stage.checkpoint_metadata())) resumed = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) resumed.resume_from_checkpoint(metadata, staged) @@ -1204,6 +1233,108 @@ def test_national_build_parity_trio_is_evidence_absent( assert gates["uk_qrf_tail_concentration"]["status"] == "failed" +def _fixture_parity_reference(input_h5) -> EfrsParityReference: + """A synthetic frozen instrument matching the fixture's export surface. + + Reference and candidate stay independently derived in production; the + test constructs the reference from the *input* artifact, before any + stage runs, so a stage that leaked scratch columns would still fail + the export-surface comparison. + """ + + frame, _provenance = load_uk_national_frame(input_h5) + input_entities = { + str(column): entity + for entity in frame.entities + for column in frame.table(entity).columns + } + return EfrsParityReference( + source=EfrsParitySource( + repo_id="example/synthetic", + repo_type="model", + filename="synthetic_reference.h5", + revision="0" * 40, + sha256="0" * 64, + url="https://example.invalid/synthetic_reference.h5", + vintage="2024_25", + period="2024", + size_bytes=1, + ), + nonzero_shares={name: 1.0 for name in input_entities}, + input_entities=input_entities, + ) + + +def test_national_build_parity_trio_evaluates_for_armed_calibration( + tmp_path, +) -> None: + pytest.importorskip("tables") + + input_h5 = tmp_path / "healthy.h5" + terminal_json = tmp_path / "terminal_gates.json" + _write_two_row_h5(input_h5, include_calibration_columns=True) + + _run_national_build( + input_h5=input_h5, + staging_h5=tmp_path / "staging.h5", + stages=( + Stage( + name="national_calibration", + transform=UKNationalCalibrationStage( + _calibration_registry(55.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ), + ), + ), + coverage_engine=object(), + terminal_gate_path=terminal_json, + gate_registry=_registry_with_calibration_and_parity_trio(), + parity_reference=_fixture_parity_reference(input_h5), + ) + + gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] + for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + assert gates[entry_id]["status"] == "passed", entry_id + assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + + +def test_armed_calibration_without_parity_reference_stays_evidence_absent( + tmp_path, +) -> None: + """No frozen instrument, no trio evidence — never a copied reference.""" + + pytest.importorskip("tables") + + input_h5 = tmp_path / "healthy.h5" + terminal_json = tmp_path / "terminal_gates.json" + _write_two_row_h5(input_h5, include_calibration_columns=True) + + _run_national_build( + input_h5=input_h5, + staging_h5=tmp_path / "staging.h5", + stages=( + Stage( + name="national_calibration", + transform=UKNationalCalibrationStage( + _calibration_registry(55.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ), + ), + ), + coverage_engine=object(), + terminal_gate_path=terminal_json, + gate_registry=_registry_with_calibration_and_parity_trio(), + ) + + gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] + for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + assert gates[entry_id]["status"] == "evidence_absent", entry_id + assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" + assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + + def test_national_build_rejects_both_gate_path_names_and_h5_collisions( tmp_path, ) -> None: @@ -1528,15 +1659,19 @@ class _CountingCalibrationStage: def __init__(self, calls: list[str]) -> None: self.calls = calls self.inner = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) @property def manifest(self) -> dict[str, object] | None: return self.inner.manifest + @property + def diagnostics(self) -> tuple[dict[str, object], ...]: + return self.inner.diagnostics + def __call__(self, frame: Frame) -> Frame: self.calls.append("national_calibration") return self.inner(frame) diff --git a/packages/microcosm-build/tests/test_uk_national_build_driver.py b/packages/microcosm-build/tests/test_uk_national_build_driver.py index 54e39138eb..4be972b64f 100644 --- a/packages/microcosm-build/tests/test_uk_national_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_national_build_driver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import importlib.util import json import sys @@ -927,6 +928,40 @@ def test_national_driver_accepts_legacy_input_coverage_path_alias( assert args.logbook_prev_row_digest == "d" * 64 +def test_national_driver_accepts_staging_candidate_input_sha( + monkeypatch, + tmp_path, +) -> None: + builder = _load_builder_module() + declared_sha = "a" * 64 + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_national_dataset.py", + *_IDENTITY_CLI_ARGUMENTS, + "--input-h5", + "base.h5", + "--staging-h5", + "staging.h5", + "--staging-candidate-input-sha256", + declared_sha, + "--frs-raw-dir", + "frs_2023_24", + "--spi-tab", + "put2223uk.tab", + "--hmrc-ods", + "hmrc.ods", + "--cgt-ods", + "cgt.ods", + ], + ) + + args = builder._parse_args() + + assert args.staging_candidate_input_sha256 == declared_sha + + def test_national_driver_forwards_legacy_output_to_compatibility_serializer( monkeypatch, tmp_path, @@ -1283,6 +1318,60 @@ def test_national_driver_refuses_release_candidate_with_the_legacy_alias( assert "signed schema-4 report" in capsys.readouterr().err +def test_national_driver_refuses_release_candidate_with_staging_candidate_input( + monkeypatch, + capsys, +) -> None: + builder = _load_builder_module() + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_national_dataset.py", + *_IDENTITY_CLI_ARGUMENTS, + "--input-h5", + "base.h5", + "--staging-h5", + "staging.h5", + "--staging-candidate-input-sha256", + "a" * 64, + "--frs-raw-dir", + "frs_2023_24", + "--spi-tab", + "put2223uk.tab", + "--hmrc-ods", + "hmrc.ods", + "--cgt-ods", + "cgt.ods", + "--release-candidate", + ], + ) + + with pytest.raises(SystemExit): + builder._parse_args() + assert "non-certified spine" in capsys.readouterr().err + + +def test_staging_candidate_input_verifier_requires_declared_sha(tmp_path) -> None: + builder = _load_builder_module() + source = tmp_path / "synthetic-spine.h5" + source.write_bytes(b"synthetic") + digest = hashlib.sha256(b"synthetic").hexdigest() + + identity = builder.verify_staging_candidate_uk_input( + source, + expected_sha256=digest, + ) + + assert identity.tier == "staging_candidate" + assert identity.sha256 == digest + with pytest.raises(ValueError, match="does not match declared"): + builder.verify_staging_candidate_uk_input( + source, + expected_sha256="0" * 64, + ) + + def test_staging_run_config_pins_the_sampling_identity(monkeypatch, tmp_path) -> None: builder = _load_builder_module() for name in ("adult.tab", "benefits.tab", "put2223uk.tab", "hmrc.ods", "cgt.ods"): diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index a299d1782c..4526f279b3 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -10,10 +10,26 @@ import pytest from microcosm.build.ledger_targets import LedgerTargetReference +from microcosm.build.uk_runtime import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UKNationalSolveDoctrine, +) +from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation +from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, + _post_solve_calibration_record, + national_calibration_mass_reason, ) from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import EntitySchema, Frame, WeightKind, Weights ACTIVE_REFERENCE_COUNT = 387 @@ -59,6 +75,23 @@ def _fact( } +def _registry(*, value: float = 30.0) -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=value, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + def _frame() -> Frame: ids = np.arange(4, dtype="int64") return Frame( @@ -97,9 +130,7 @@ def _nested_frame() -> Frame: "universal_credit": [1.0, 0.0, 1.0, 1.0], } ), - "household": pd.DataFrame( - {"household_id": np.arange(3, dtype="int64")} - ), + "household": pd.DataFrame({"household_id": np.arange(3, dtype="int64")}), }, EntitySchema(group_entities=("benunit", "household")), {"household": Weights(np.array([10.0, 20.0, 30.0]), WeightKind.DESIGN)}, @@ -110,7 +141,9 @@ def _nested_frame() -> Frame: def test_uc_calibration_compiles_and_moves_weighted_count_towards_fact() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=200, learning_rate=0.05 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=200, learning_rate=0.05), ) result = stage(frame) @@ -122,12 +155,27 @@ def test_uc_calibration_compiles_and_moves_weighted_count_towards_fact() -> None assert stage.manifest["resolved_reference_count"] == 1 assert stage.manifest["matrix_target_count"] == 1 assert stage.diagnostics[0]["target"] == 30.0 + assert result.weights_for("household").kind is WeightKind.CALIBRATED + assert len(result.mass_log) == 1 + mass_change = stage.manifest["weights"]["calibration_mass_change"] + assert mass_change["entity"] == "household" + assert "National doctrine calibration" in mass_change["reason"] + assert stage.manifest["weights"]["household_weight_kind_chain"] == [ + {"stage": "staging", "kind": "design"}, + {"stage": "national_calibration", "kind": "calibrated"}, + ] + assert stage.manifest["weights"]["mass_log_records_before_calibration"] == 0 + assert stage.manifest["weights"]["mass_log_records"] == 1 + assert stage.manifest["solve"]["n_targets"] == 1 + assert stage.manifest["solve"]["n_households"] == 4 def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() -> None: frame = _nested_frame() stage = UKNationalCalibrationStage( - [_fact(value=60.0)], references=[_uc_reference()], epochs=5 + _registry(value=60.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) result = stage(frame) @@ -140,12 +188,23 @@ def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() validate_uk_national_frame(result) -def test_activated_unresolvable_reference_aborts_loudly() -> None: +def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: stage = UKNationalCalibrationStage( - [_fact(concept="different")], references=[_uc_reference()], epochs=1 + UKLedgerTargetCompilation( + registry=TargetRegistry([], country="uk"), + unsupported=( + { + "name": "dwp.uc.households", + "period": "2025", + "reason": "did not match a Ledger fact selector", + }, + ), + ), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) - with pytest.raises(ValueError, match="did not match a Ledger fact selector"): + with pytest.raises(RuntimeError, match="did not match a Ledger fact selector"): stage(_frame()) @@ -191,7 +250,9 @@ def test_chronicle_184_uc_and_obr_references_compile_fail_closed() -> None: def test_calibration_preserves_entity_ids_and_national_integrity() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) result = stage(frame) @@ -205,14 +266,18 @@ def test_calibration_preserves_entity_ids_and_national_integrity() -> None: def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) staged = stage(frame) metadata = json.loads(json.dumps(stage.checkpoint_metadata())) resumed = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) resumed.resume_from_checkpoint(metadata, staged) @@ -221,13 +286,17 @@ def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: assert resumed.output_content_identity == metadata["output_content_identity"] drifted = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="drifted record"): drifted.resume_from_checkpoint(metadata, frame) empty = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="calibration counts"): empty.resume_from_checkpoint({}, staged) @@ -242,7 +311,98 @@ def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: empty.resume_from_checkpoint(missing_count, staged) unrun = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="has not run"): unrun.checkpoint_metadata() + + +def test_prepared_slash_columns_are_not_returned_to_the_writer(tmp_path) -> None: + pytest.importorskip("tables") + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + ) + + result = stage(_frame()) + + assert "dwp/uc/households" not in result.table("benunit") + write_uk_national_frame(result, tmp_path / "staging.h5") + + +def test_national_calibration_mass_reason_is_canonical() -> None: + assert national_calibration_mass_reason( + ["dwp_universal_credit", "hmrc", "dwp_universal_credit"] + ) == ( + "National doctrine calibration to bound target family(ies) " + "dwp_universal_credit, hmrc; total household mass moved with the targets." + ) + with pytest.raises(ValueError, match="bound_families"): + national_calibration_mass_reason([]) + + +def test_post_solve_fence_requires_calibrated_kind_and_mass_record() -> None: + before = _frame() + uncalibrated = Frame( + {entity: before.table(entity) for entity in before.entities}, + before.schema, + {"household": Weights(np.full(4, 10.0), WeightKind.DESIGN)}, + before.strata, + mass_log=before.mass_log, + metadata=before.metadata, + ) + + with pytest.raises(RuntimeError, match="not 'calibrated'"): + _post_solve_calibration_record(before, uncalibrated, before_count=0) + + calibrated_without_record = Frame( + {entity: before.table(entity) for entity in before.entities}, + before.schema, + {"household": Weights(np.full(4, 10.0), WeightKind.CALIBRATED)}, + before.strata, + mass_log=before.mass_log, + metadata=before.metadata, + ) + with pytest.raises(RuntimeError, match="exactly one mass record"): + _post_solve_calibration_record( + before, + calibrated_without_record, + before_count=0, + ) + + +def test_national_doctrine_constants_are_the_declared_contract() -> None: + assert UK_NATIONAL_SOLVE_EPOCHS == 256 + assert UK_NATIONAL_LEARNING_RATE == 0.02 + assert UK_NATIONAL_MAX_WEIGHT_RATIO == 10.0 + assert UK_NATIONAL_SEED == 0 + assert UK_NATIONAL_TARGET_LOSS_CAP == 10.0 + assert UK_NATIONAL_L0_LAMBDA == 0.0 + assert UK_NATIONAL_MASS_RULE == "free" + assert UK_NATIONAL_SOLVE_DOCTRINE == UKNationalSolveDoctrine() + assert UK_NATIONAL_SOLVE_DOCTRINE.scale_rule == "default_target_loss_scales" + assert UK_NATIONAL_SOLVE_DOCTRINE.target_weight_rule == "uniform" + + +def test_national_doctrine_rejects_tampered_bounds() -> None: + with pytest.raises(ValueError, match="epochs"): + UKNationalSolveDoctrine(epochs=0) + with pytest.raises(ValueError, match="learning_rate"): + UKNationalSolveDoctrine(learning_rate=0.0) + with pytest.raises(ValueError, match="max_weight_ratio"): + UKNationalSolveDoctrine(max_weight_ratio=1.0) + with pytest.raises(ValueError, match="seed"): + UKNationalSolveDoctrine(seed=-1) + with pytest.raises(ValueError, match="target_loss_cap"): + UKNationalSolveDoctrine(target_loss_cap=float("nan")) + with pytest.raises(ValueError, match="scale_rule"): + UKNationalSolveDoctrine(scale_rule="bespoke") + with pytest.raises(ValueError, match="target_weight_rule"): + UKNationalSolveDoctrine(target_weight_rule="per_target") + with pytest.raises(ValueError, match="mass_rule"): + UKNationalSolveDoctrine(mass_rule="conserve") + with pytest.raises(ValueError, match="l0_lambda"): + UKNationalSolveDoctrine(l0_lambda=-0.1) diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index 7392b9884d..2ee157d9da 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -10,6 +10,7 @@ import time import uuid from datetime import UTC, datetime +from importlib import resources as importlib_resources from itertools import combinations from pathlib import Path @@ -38,6 +39,7 @@ from microcosm.build.uk_runtime.cgt_imputation import ( uk_capital_gains_imputation_stage, ) +from microcosm.build.uk_runtime.diagnostics import write_uk_calibration_diagnostics from microcosm.build.uk_runtime.frs_hmrc_leaves import ( UKFRSHMRCRetainedLeavesStageTransform, ) @@ -46,6 +48,7 @@ from microcosm.build.uk_runtime.hmrc_restoration import ( UKHMRCIncomeStageTransform, verify_certified_uk_candidate, + verify_staging_candidate_uk_input, ) from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry from microcosm.build.uk_runtime.national_build import build_uk_national_dataset @@ -60,6 +63,9 @@ UK_SAMPLE_RUNG_TOKENS, UK_SAMPLE_SEED_DEFAULT, ) +from microcosm.build.uk_runtime.parity_reference import ( + load_efrs_parity_reference, +) from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIERS from microcosm.build.uk_runtime.source_runtime import uk_stage_implementations from microcosm.build.uk_runtime.terminal_gates import ( @@ -131,6 +137,14 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Compact UK single-year H5 supplying the national base tables.", ) + parser.add_argument( + "--staging-candidate-input-sha256", + type=sha256_argument, + help=( + "Declare --input-h5 as a non-certified staging-candidate spine and " + "bind it to this SHA-256. Refused with --release-candidate." + ), + ) parser.add_argument( "--staging-h5", type=Path, @@ -335,6 +349,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "schema-1 alias is last-written over the report path and a " "candidate must keep its signed schema-4 report." ) + if args.release_candidate and args.staging_candidate_input_sha256 is not None: + parser.error( + "--release-candidate requires the certified input posture; " + "--staging-candidate-input-sha256 declares a non-certified spine." + ) if args.sample_seed < 0: parser.error("sample seed must be a non-negative integer.") if args.sample_fraction != 1.0 and ( @@ -434,6 +453,18 @@ def _source_pins( return pins +def _input_posture(candidate: object) -> dict[str, object]: + tier = str(getattr(candidate, "tier", "frs")) + return { + "posture": "staging_candidate" if tier == "staging_candidate" else "certified", + "filename": str(getattr(candidate, "filename", "")), + "tier": tier, + "revision": str(getattr(candidate, "revision", "")), + "sha256": str(candidate.sha256), + "size_bytes": int(candidate.size_bytes), + } + + def _gate_verdicts_from_report( report: dict[str, object], *, @@ -786,7 +817,13 @@ def _main_recording( for period in (2023, 2025) } ) - candidate = verify_certified_uk_candidate(args.input_h5) + if args.staging_candidate_input_sha256 is None: + candidate = verify_certified_uk_candidate(args.input_h5) + else: + candidate = verify_staging_candidate_uk_input( + args.input_h5, + expected_sha256=args.staging_candidate_input_sha256, + ) evidence_path.unlink(missing_ok=True) replay_path.unlink(missing_ok=True) build_record_path.unlink(missing_ok=True) @@ -836,11 +873,12 @@ def _main_recording( append_phase(state, "configured") append_phase(state, "candidate_verified") append_phase(state, "inputs_pinned") - # This staging path performs no calibration and therefore has no real - # target-surface or target-fit evidence; the schema-4 battery records - # the missing evidence explicitly. Input-mass evidence joins only when - # the caller supplies the licensed frozen reference sidecar; QRF-tail is - # spec-armed and runs whenever the frame evidence is present. + # Without Ledger facts, this staging path has no real target-surface or + # target-fit evidence; the schema-4 battery records the missing evidence + # explicitly. Armed calibration builds add target evidence from the solve. + # Input-mass evidence joins only when the caller supplies the licensed + # frozen reference sidecar; QRF-tail is spec-armed and runs whenever the + # frame evidence is present. gate_path_argument = ( {"input_coverage_path": legacy_input_coverage_path} if legacy_input_coverage_path is not None @@ -865,7 +903,12 @@ def _main_recording( calibration_stages: tuple[PlanStage, ...] = () if args.ledger_facts is not None: assert ledger_artifact is not None # resolved and pin-checked above - calibration_transform = UKNationalCalibrationStage(ledger_artifact.facts) + assert ledger_compilations is not None + calibration_year = load_uk_frs_release().calibration_year + calibration_transform = UKNationalCalibrationStage( + ledger_compilations[calibration_year], + period=calibration_year, + ) calibration_stages = ( PlanStage( name="national_calibration", @@ -910,14 +953,35 @@ def _main_recording( for period, compilation in ledger_compilations.items() } ), + # The frozen instrument supplies the parity trio's reference side; + # the candidate side comes from the staged frame and the solve. + parity_reference=( + load_efrs_parity_reference() if calibration_transform is not None else None + ), ) if calibration_transform is not None: - _write_json( + if calibration_transform.solve_result is None: + raise RuntimeError( + "national calibration diagnostics require the intact solve " + "result; checkpoint-resumed runs must rerun calibration before " + "writing calibration_diagnostics.json." + ) + assert ledger_artifact is not None + write_uk_calibration_diagnostics( + calibration_transform.solve_result, args.national_calibration_diagnostics_json, - { - "schema_version": 1, - "targets": list(calibration_transform.diagnostics), - "calibration": calibration_transform.manifest, + result.frame, + target_geography_levels=_uk_target_geography_levels( + calibration_transform.registry + ), + target_registry=calibration_transform.registry, + build={ + "build_id": state.build_id, + "ledger_facts": ledger_artifact.provenance(), + "code_pin": attempt_context["code_pin"], + "source_pins": source_pins, + "input_posture": _input_posture(candidate), + "score_vs_enhanced_frs": None, }, ) append_phase(state, "build_completed") @@ -959,6 +1023,7 @@ def _main_recording( ledger_artifact_provenance=( None if ledger_artifact is None else ledger_artifact.provenance() ), + input_posture=_input_posture(candidate), ) _write_json(build_record_path, build_record) append_phase(state, "build_record_written") @@ -1053,6 +1118,7 @@ def _staging_run_config( "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], }, "certified_candidate": dict(pins["certified_candidate"]), + "input_posture": _input_posture(candidate), "sources": { "adult_tab": dict(pins["adult_tab"]), "benefits_tab": dict(pins["benefits_tab"]), @@ -1100,6 +1166,7 @@ def _aggregate_build_record( sample_seed: int = UK_SAMPLE_SEED_DEFAULT, degenerate_exclusions_override: bool = False, ledger_artifact_provenance: dict[str, object] | None = None, + input_posture: dict[str, object] | None = None, ) -> dict[str, object]: """Return commit-safe aggregate evidence for one successful staging build.""" @@ -1157,6 +1224,7 @@ def _aggregate_build_record( # honestly disagree, which is why they carry distinct names. "degenerate_exclusions_override_supplied": (degenerate_exclusions_override), }, + "input_posture": dict(input_posture or {}), "sampling": ( None if result.sampling_receipt is None else dict(result.sampling_receipt) ), @@ -1348,6 +1416,31 @@ def _validate_distinct_paths( ) +def _uk_target_geography_levels(registry) -> dict[str, str]: + payload = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + targets = {row["target_id"]: row for row in payload["targets"]} + levels: dict[str, str] = {} + for spec in registry.specs: + target_id = spec.metadata.get("contract_target_id") + target = targets.get(str(target_id)) + if target is None: + raise ValueError( + f"UK calibration target {spec.name!r} references unknown " + f"contract target {target_id!r}." + ) + geography_levels = tuple(target.get("geography_levels") or ()) + if not geography_levels: + raise ValueError( + f"UK calibration target {spec.name!r} has no geography level." + ) + levels[spec.to_target().row_name] = str(geography_levels[0]) + return levels + + def _paths_alias(left: Path, right: Path) -> bool: """Conservatively identify aliases before any build path is unlinked.""" diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py new file mode 100644 index 0000000000..3f98959fba --- /dev/null +++ b/tools/score_uk_national_candidate.py @@ -0,0 +1,183 @@ +"""Score a UK national candidate against the incumbent on one frozen register.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from microcosm.build.uk_runtime.national_build import load_uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec, score_targets + +UK_SCORE_LOSS_CAP = 10.0 + + +def score_uk_national_candidate( + *, + candidate_h5: str | Path, + incumbent_h5: str | Path, + target_registry: TargetRegistry, + candidate_label: str = "populace_uk_2023", + incumbent_label: str = "enhanced_frs_2024_25", +) -> dict[str, Any]: + """Return the #578 rule-1 score block on a shared target registry.""" + + if target_registry.country != "uk" or not target_registry.specs: + raise ValueError("UK candidate scoring requires a non-empty UK registry.") + candidate_frame, _candidate_provenance = load_uk_national_frame(candidate_h5) + incumbent_frame, _incumbent_provenance = load_uk_national_frame(incumbent_h5) + candidate = score_targets( + candidate_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + incumbent = score_targets( + incumbent_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + candidate_errors = _relative_errors(candidate) + incumbent_errors = _relative_errors(incumbent) + names = sorted(candidate_errors) + missing = sorted(set(names) ^ set(incumbent_errors)) + if missing: + raise RuntimeError( + "candidate and incumbent scores produced different target rows: " + f"{missing[:10]}." + ) + wins = _target_wins(candidate_errors, incumbent_errors) + return { + "candidate_train_loss": float(candidate.final_loss), + "candidate_holdout_loss": float(candidate.final_loss), + "candidate_full_loss": float(candidate.final_loss), + "incumbent_train_loss": float(incumbent.final_loss), + "incumbent_holdout_loss": float(incumbent.final_loss), + "incumbent_full_loss": float(incumbent.final_loss), + "candidate_target_wins": wins["candidate"], + "incumbent_target_wins": wins["incumbent"], + "holdout_basis": "none_declared", + "loss": { + "objective": "relative_error_loss", + "target_loss_cap": UK_SCORE_LOSS_CAP, + "train_equals_full": True, + }, + "register": { + "country": target_registry.country, + "version": target_registry.version, + "n_specs": len(target_registry), + }, + "artifacts": { + "candidate": candidate_label, + "incumbent": incumbent_label, + }, + "signed_asymmetries": [ + { + "id": "incumbent_own_registry", + "description": ( + "The incumbent was calibrated to its own registry; both " + "artifacts are rescored here on the supplied frozen register." + ), + }, + { + "id": "national_direct_vs_collapsed_local", + "description": ( + "Candidate national weights are a direct national solve; " + "incumbent national weights are the published national " + "surface being compared on the same frozen register." + ), + }, + ], + "target_wins_by_family": _target_wins_by_family( + target_registry, + candidate_errors, + incumbent_errors, + ), + } + + +def _relative_errors(result) -> dict[str, float]: + return { + diagnostic.name: float(diagnostic.relative_error) + for diagnostic in result.diagnostics + } + + +def _target_wins( + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, int]: + candidate = 0 + incumbent = 0 + for name, candidate_error in candidate_errors.items(): + incumbent_error = incumbent_errors[name] + if abs(candidate_error) < abs(incumbent_error): + candidate += 1 + elif abs(incumbent_error) < abs(candidate_error): + incumbent += 1 + return {"candidate": candidate, "incumbent": incumbent} + + +def _target_wins_by_family( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, dict[str, int]]: + by_row_name = {spec.to_target().row_name: spec.family for spec in registry.specs} + result: dict[str, dict[str, int]] = {} + for name, family in by_row_name.items(): + bucket = result.setdefault( + family, + {"candidate_target_wins": 0, "incumbent_target_wins": 0, "ties": 0}, + ) + candidate_error = abs(candidate_errors[name]) + incumbent_error = abs(incumbent_errors[name]) + if candidate_error < incumbent_error: + bucket["candidate_target_wins"] += 1 + elif incumbent_error < candidate_error: + bucket["incumbent_target_wins"] += 1 + else: + bucket["ties"] += 1 + return result + + +def _load_registry(path: str | Path) -> TargetRegistry: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if isinstance(payload, dict): + specs_payload = payload.get("specs") + country = str(payload.get("country", "uk")) + else: + specs_payload = payload + country = "uk" + if not isinstance(specs_payload, list): + raise ValueError("target registry JSON must contain a specs list.") + return TargetRegistry( + (TargetSpec(**item) for item in specs_payload), + country=country, + ) + + +def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: + output = Path(path) + output.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + return output + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--incumbent-h5", required=True, type=Path) + parser.add_argument("--target-registry-json", required=True, type=Path) + parser.add_argument("--output-json", required=True, type=Path) + args = parser.parse_args(argv) + score = score_uk_national_candidate( + candidate_h5=args.candidate_h5, + incumbent_h5=args.incumbent_h5, + target_registry=_load_registry(args.target_registry_json), + ) + _write_json(args.output_json, {"score_vs_enhanced_frs": score}) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From ccb80a3d42c46910d1e40b11c040079a95351c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:13:37 +0200 Subject: [PATCH 08/26] Drop the duplicate spine-driver failure handler reintroduced by the main merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The post-#735/#733 merge at 9a8bd46d resurrected the standalone `except ValueError` block that #733's review commit 1d066e5d had folded into the single generic handler. Because the narrower clause catches first, the named-edge branch inside `except Exception` became unreachable, silently reverting a reviewed structural fix. This branch has no business touching the spine driver at all — its scope is the national calibration seam — so the file returns to main's version exactly. Co-Authored-By: Claude Fable 5 --- tools/build_uk_frs_spine.py | 52 ------------------------------------- 1 file changed, 52 deletions(-) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index 2ca036af84..5b4822a9b2 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -868,58 +868,6 @@ def main(argv: list[str] | None = None) -> int: print(f"Wrote FRS spine H5: {output}", file=sys.stderr) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) return 0 - except ValueError as error: - if args.sample_fraction != 1.0 and _RUNG_NAMED_EDGE_SIGNATURE in str(error): - rung_abort_path = args.spine_h5.with_suffix(".rung_abort.json") - receipt = _rung_abort_receipt(args, error=error) - atomic_write_json(rung_abort_path, receipt) - state.gate_verdicts = { - "uk_frs_spine_rung_abort": { - "verdict": "aborted", - "receipt": ( - f"{local_artifact_reference(rung_abort_path, repository_hint=_REPOSITORY)}" - "#/named_edge" - ), - } - } - append_phase(state, "rung_aborted") - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="discarded", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - print(json.dumps(receipt, indent=2, sort_keys=True)) - return _RUNG_ABORT_EXIT_CODE - try: - receipt_path = write_error_receipt( - error_receipt_path(args.spine_h5.parent, build_id=state.build_id), - state=state, - pipeline=_PIPELINE, - error=error, - ) - apply_error_verdict( - state, - local_artifact_reference(receipt_path, repository_hint=_REPOSITORY), - ) - _record_attempt( - state=state, - started_at=started_at, - started_ts=started_ts, - code_pin=code_pin, - disposition="failed", - predecessor=predecessor, - rung=rung, - spool_dir=spool_dir, - ) - except Exception: - pass - print(f"UK FRS spine build failed: {error}", file=sys.stderr) - return 1 except Exception as error: if ( isinstance(error, ValueError) From 43d3e2a5051fea346bd5b624ce5278fa37df8c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:12:53 +0200 Subject: [PATCH 09/26] Pin the Ledger feed by content and size in the Logbook role pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An armed UK national build crashed before its first stage: _source_pins stored the full Ledger provenance block under the 'ledger_facts' role, but role_pins_digest requires each role to carry exactly sha256 and size_bytes, so every armed run raised ValueError at input_pins_digest. The two halves landed independently — the strict pin contract with Logbook adoption (#666) and the Ledger role pin with the compile-parity wiring — and neither PR branch fires it alone, because only an armed run (--ledger-facts) reaches this path and the licensed runs were held. It reproduces on both #743 and #747 branches and on main. The pin now carries the feed's verified digest and byte size; the richer Ledger identity block already travels in safe_artifacts, source_vintages, and the diagnostics build block, so nothing is lost. This fix belongs upstream in #743 (the run-readiness PR whose runbook documents the armed command); it rides the build branch until then. --- tools/build_uk_national_dataset.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index 1dc65be8ef..00c4fdc4d7 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -449,10 +449,28 @@ def _source_pins( "cgt_ods": _artifact_pin(cgt_ods_path), } if ledger_artifact is not None: - pins["ledger_facts"] = dict(ledger_artifact.provenance()) + pins["ledger_facts"] = _ledger_facts_pin(ledger_artifact) return pins +def _ledger_facts_pin(ledger_artifact: object) -> dict[str, object]: + """Pin the consumer feed by content and size. + + Logbook role pins are exactly ``sha256`` and ``size_bytes``; the richer + Ledger identity block travels separately in ``safe_artifacts`` and + ``source_vintages``. The feed digest is already verified against the + manifest and the CLI pin at load, so it is reused rather than recomputed + over a multi-hundred-megabyte file. + """ + + path = Path(getattr(ledger_artifact, "path")) + facts_path = path / "consumer_facts.jsonl" if path.is_dir() else path + return { + "sha256": str(getattr(ledger_artifact, "facts_sha256")), + "size_bytes": int(facts_path.stat().st_size), + } + + def _input_posture(candidate: object) -> dict[str, object]: tier = str(getattr(candidate, "tier", "frs")) return { From 8eef7eb436135b7b884f5f1393183877902b3ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:55:05 +0200 Subject: [PATCH 10/26] Slice banded target measures: implement the declared groupby 243 of the 388 UK references are banded, and none of them were ever sliced. Every employment-income band materialized 35,351,186 (roughly everyone with employment income) and every state-pension band 13,518,447 (roughly every state pensioner), so a band's apparent overshoot -- up to 6,759x on the state-pension 1m+ cell -- was an unsliced total compared against a band value, not a data defect. The slice was declared but unconsumed: the contract binding names groupby_variable, each compiled spec carries its own band's lower edge in Ledger filter metadata, and _prepared_column_values read neither. The binding's own filters list did work (verified: a family_type == SINGLE binding correctly masks a COUPLE row), so this is specifically the band dimension. Both published encodings reduce to one lower edge -- a numeric *_lower_bound (HMRC SPI) or a range label in monthly units scaled by band_period_factor (DWP awards) -- because no reference anywhere declares an upper bound. A band's upper edge is its sibling's lower edge within the same contract target, grouped per contract target rather than per dimension so two measures sharing a dimension cannot slice each other on the wrong boundaries; the top band runs to infinity. Validated against the real register: 243 banded references, zero unreadable, and the derived bounds match the measure names (150000-200000, 1000000-inf, and UC monthly 500.01-600.00 to annual 6000.12-7200.12). A band whose edge cannot be read now raises, so the measure is skipped and reported rather than silently reporting the whole population as one band -- the failure mode that hid this. Tests cover the partition property, an adjacent-bands-differ regression, the monthly label conversion, and the refusal. Lives in the shared materialization module, so it cherry-picks into any branch. --- .../microcosm/build/target_materialization.py | 127 +++++++++++++- .../tests/test_target_materialization.py | 155 ++++++++++++++++++ 2 files changed, 280 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index e5209134f9..039dbcd3e2 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -2,7 +2,9 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +import math +import re +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any @@ -12,6 +14,26 @@ Provider = Callable[[Any, Mapping[str, Any], int | str], np.ndarray] +# Banded measures declare their slice in two places: the contract binding +# names the variable to band on (``groupby_variable``), and each compiled +# spec carries its own band's lower edge in Ledger filter metadata. Two +# encodings are in use — a numeric lower bound (HMRC SPI income bands) and a +# published range label (DWP award bands, in monthly units, hence +# ``band_period_factor``) — and both reduce to one lower edge, because no +# reference anywhere declares an upper bound. A band's upper edge is +# therefore its sibling's lower edge within the same contract target, and the +# top band runs to infinity. +_BAND_LOWER_BOUND_SUFFIX = "_lower_bound" +_LEDGER_FILTER_PREFIX = "ledger_filter_" +_RANGE_LABEL = re.compile( + r"([\d,]+(?:\.\d+)?)\s*(?:to|–|—|-)\s*(?:£\s*)?([\d,]+(?:\.\d+)?)", + re.IGNORECASE, +) +_OPEN_LABEL = re.compile( + r"([\d,]+(?:\.\d+)?)\s*(?:or more|or over|and over|and above|\+)", + re.IGNORECASE, +) + @dataclass(frozen=True) class MaterializationSkip: @@ -40,6 +62,91 @@ def report(self) -> dict[str, object]: } +def _band_lower_edge( + spec: Any, binding: Mapping[str, Any] +) -> float | None: + """This spec's band lower edge in model units, or None if unbanded. + + Reads the compiled spec's Ledger filter metadata: a ``*_lower_bound`` + entry when the publisher gives numeric edges, otherwise a range label + such as ``"£500.01 to £600.00"`` or ``"5 or more"``. ``band_period_factor`` + converts published units to the model's (monthly award bands to annual). + Non-band filters — ``ledger_filter_family_type = "Single, no children"`` + — carry no numeric range and are ignored here; the binding's own + ``filters`` list applies those. + """ + + metadata = getattr(spec, "metadata", None) or {} + factor = float(binding.get("band_period_factor", 1) or 1) + for key, value in sorted(metadata.items()): + if not key.startswith(_LEDGER_FILTER_PREFIX): + continue + if key.endswith(_BAND_LOWER_BOUND_SUFFIX): + return float(str(value).replace(",", "")) * factor + for key, value in sorted(metadata.items()): + if not key.startswith(_LEDGER_FILTER_PREFIX): + continue + match = _RANGE_LABEL.search(str(value)) or _OPEN_LABEL.search(str(value)) + if match is not None: + return float(match.group(1).replace(",", "")) * factor + return None + + +def _band_edges_by_group( + registry: TargetRegistry, + contract_targets: Mapping[str, Mapping[str, Any]], +) -> dict[str, list[float]]: + """Sorted band lower edges per contract target that declares bands. + + Grouped by contract target rather than by dimension: two measures can + share a dimension while publishing different band sets, and taking the + union of their edges would slice each measure on the other's boundaries. + """ + + edges: dict[str, set[float]] = {} + for spec in registry.specs: + contract_target_id = str( + (getattr(spec, "metadata", None) or {}).get("contract_target_id", "") + ) + target = contract_targets.get(contract_target_id) + if target is None: + continue + binding = target.get("bindings", {}).get("policyengine", {}) + if not binding.get("groupby_variable"): + continue + lower = _band_lower_edge(spec, binding) + if lower is not None: + edges.setdefault(contract_target_id, set()).add(lower) + return {key: sorted(values) for key, values in edges.items()} + + +def _band_bounds( + spec: Any, + binding: Mapping[str, Any], + band_edges: Sequence[float], +) -> tuple[float, float]: + """The half-open ``[lower, upper)`` this spec's band covers. + + Raises rather than returning an unsliced column: a banded measure whose + band cannot be read would otherwise report the whole population as if it + were one band, which is the failure this function exists to end. + """ + + lower = _band_lower_edge(spec, binding) + if lower is None: + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} declares " + f"groupby_variable {binding['groupby_variable']!r} but its spec " + "carries no readable band edge" + ) + upper = math.inf + for edge in band_edges: + if edge > lower: + upper = edge + break + return lower, upper + + def materialize_target_bindings( adapter: Any, registry: TargetRegistry, @@ -51,6 +158,7 @@ def materialize_target_bindings( """Prepare measure columns declared by compiled Ledger target specs.""" provider_registry = {**default_provider_registry(), **(providers or {})} + band_edges = _band_edges_by_group(registry, contract_targets) skipped: list[MaterializationSkip] = [] for spec in registry.specs: if hasattr(adapter, "has_column") and adapter.has_column( @@ -77,7 +185,16 @@ def materialize_target_bindings( raise ValueError(f"unsupported binding kind {kind!r}") values = provider(adapter, binding, period) else: - values = _prepared_column_values(adapter, spec.entity, binding) + band = None + if binding.get("groupby_variable"): + band = _band_bounds( + spec, + binding, + band_edges.get(str(contract_target_id), ()), + ) + values = _prepared_column_values( + adapter, spec.entity, binding, band=band + ) adapter.set_column(spec.entity, spec.measure, values) except (KeyError, TypeError, ValueError) as error: skipped.append( @@ -169,6 +286,8 @@ def _prepared_column_values( adapter: Any, entity: str, binding: Mapping[str, Any], + *, + band: tuple[float, float] | None = None, ) -> np.ndarray: entity = str(binding.get("from_entity") or entity) if "value_expression" in binding: @@ -180,6 +299,10 @@ def _prepared_column_values( mask &= _predicate_mask(adapter, entity, predicate) for predicate in binding.get("household_conditions", ()): mask &= _predicate_mask(adapter, entity, predicate) + if band is not None: + lower, upper = band + banded = _column(adapter, entity, binding["groupby_variable"]).astype(float) + mask &= (banded >= lower) & (banded < upper) return np.where(mask, values, 0.0) diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index ddbcb000d0..0844ddc87d 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -225,3 +225,158 @@ def test_count_aliases_and_in_predicates_materialize_prepared_columns(): 0.0, 1.0, ] + + +def _banded_registry(*, count_measure: bool = True) -> TargetRegistry: + """Three adjacent income bands over one contract target.""" + + specs = [] + for lower, label in ((0, "0"), (20, "20"), (40, "40")): + specs.append( + TargetSpec( + name=f"band_{label}", + entity="person", + measure=f"income_band_{label}", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + "ledger_filter_total_income_lower_bound": str(lower), + }, + ) + ) + return TargetRegistry(specs, country="uk") + + +_BANDED_CONTRACT = { + "spi.income_by_band": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + } + } + } +} + + +def test_bands_slice_the_population_and_partition_it(): + adapter = StubAdapter() + registry = _banded_registry() + + result = materialize_target_bindings( + adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + # income is [10, 20, 30]; bands are [0,20), [20,40), [40,inf). + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] + assert list(adapter.tables["person"]["income_band_40"]) == [0.0, 0.0, 0.0] + # Every record lands in exactly one band: the bands partition the surface. + total = sum( + adapter.tables["person"][f"income_band_{label}"] + for label in ("0", "20", "40") + ) + assert list(total) == [1.0, 1.0, 1.0] + + +def test_adjacent_bands_are_not_identical(): + # The regression that would have caught the unsliced-measure defect: + # before banding was implemented every band returned the same unsliced + # column, so two adjacent bands compared equal. + adapter = StubAdapter() + materialize_target_bindings( + adapter, _banded_registry(), _BANDED_CONTRACT, period=2025 + ) + + assert not np.array_equal( + adapter.tables["person"]["income_band_0"], + adapter.tables["person"]["income_band_20"], + ) + + +def test_published_range_labels_band_in_model_units(): + adapter = StubAdapter() + # Monthly published bands, annual model values: "£1.01 to £2.00" x12 + # covers [12.12, 24.12) and the sibling opens at 24.12. + registry = TargetRegistry( + [ + TargetSpec( + name="award_low", + entity="person", + measure="award_low", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_family_type": "Single, no children", + "ledger_filter_monthly_award_bands": "£1.01 to £2.00", + }, + ), + TargetSpec( + name="award_high", + entity="person", + measure="award_high", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_monthly_award_bands": "£2.01 to £3.00", + }, + ), + ], + country="uk", + ) + contract = { + "uc.award_bands": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + "band_period_factor": 12, + } + } + } + } + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + # income is [10, 20, 30]: only 20 falls inside [12.12, 24.12). + assert list(adapter.tables["person"]["award_low"]) == [0.0, 1.0, 0.0] + assert list(adapter.tables["person"]["award_high"]) == [0.0, 0.0, 1.0] + + +def test_unreadable_band_is_skipped_not_silently_unsliced(): + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="mystery_band", + entity="person", + measure="mystery_band", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + "ledger_filter_total_income_band": "not a range", + }, + ) + ], + country="uk", + ) + + result = materialize_target_bindings( + adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert [skip.name for skip in result.skipped] == ["mystery_band"] + assert "no readable band edge" in result.skipped[0].reason + assert "mystery_band" not in adapter.tables["person"] From f8ecfa71e51b651f0d2795cca7c5df6f72fee2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:23:08 +0200 Subject: [PATCH 11/26] Bind the two-child-limit references to real model variables All 15 two-child-limit references failed to materialize. Two naming faults, both in the contract rather than the provider: - eight children references declared value_variable "children_count", which is not a policyengine-uk variable. They now bind what they actually count: uc_is_child_limit_affected for the affected-children rows (mapped to household it sums to the count of flagged children) and is_child for the children-in-affected-households rows (the total children there). - fifteen bindings put a prose label in count_of ("affected_households", "affected_children", "children_in_affected_households"). count_of is a column fallback consulted when value_variable is an entity-count indicator, so the provider looked those labels up as columns and raised. The labels move to notes, leaving household references on the household_count unit indicator as intended. The provider keeps its documented behaviour; the pre-existing test covering count_of as a real column still passes. --- .../microcosm/build/target_materialization.py | 7 +- .../build/uk/uk_national_targets.json | 61 ++++++--------- .../tests/test_target_materialization.py | 75 +++++++++++++++++++ 3 files changed, 104 insertions(+), 39 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 039dbcd3e2..3c3e85191c 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -23,6 +23,11 @@ # reference anywhere declares an upper bound. A band's upper edge is # therefore its sibling's lower edge within the same contract target, and the # top band runs to infinity. +# Entity-count indicators: a value of one per record of the owning entity. +_COUNT_VALUE_VARIABLES = frozenset( + {"household_count", "person_count", "benunit_count"} +) + _BAND_LOWER_BOUND_SUFFIX = "_lower_bound" _LEDGER_FILTER_PREFIX = "ledger_filter_" _RANGE_LABEL = re.compile( @@ -256,7 +261,7 @@ def baseline_flag_crosstab( flag = _column(adapter, entity, binding["affected_flag_variable"]).astype(bool) value_variable = str(binding.get("value_variable") or "") count_of = str(binding.get("count_of") or "") - if value_variable and value_variable not in {"household_count", "person_count"}: + if value_variable and value_variable not in _COUNT_VALUE_VARIABLES: values = _column(adapter, entity, value_variable) elif count_of and count_of not in {"household", "person", "household_count"}: values = _column(adapter, entity, count_of) diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json index f36048e626..600162b7bf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json @@ -3619,10 +3619,9 @@ "metric_name": "dwp/uc/two_child_limit/households_affected", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/households_affected", @@ -3653,10 +3652,9 @@ "metric_name": "dwp/uc/two_child_limit/children_affected", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/children_affected", @@ -3687,10 +3685,9 @@ "metric_name": "dwp/uc/two_child_limit/children_in_affected_households", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/children_in_affected_households", @@ -3730,10 +3727,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/3_children_households", @@ -3773,10 +3769,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/3_children_households_total_children", @@ -3816,10 +3811,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/4_children_households", @@ -3859,10 +3853,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/4_children_households_total_children", @@ -3902,10 +3895,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/5_children_households", @@ -3945,10 +3937,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/5_children_households_total_children", @@ -3988,10 +3979,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Publisher's top band is 6-or-more children." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Publisher's top band is 6-or-more children. Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/6_children_households", @@ -4031,10 +4021,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/6_children_households_total_children", @@ -4074,10 +4063,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/adult_pip_households", @@ -4117,10 +4105,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/adult_pip_children", @@ -4160,10 +4147,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/disabled_child_element_households", @@ -4203,10 +4189,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/disabled_child_element_children", diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 0844ddc87d..22dc8e6edb 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -380,3 +380,78 @@ def test_unreadable_band_is_skipped_not_silently_unsliced(): assert [skip.name for skip in result.skipped] == ["mystery_band"] assert "no readable band edge" in result.skipped[0].reason assert "mystery_band" not in adapter.tables["person"] + + +def test_entity_count_binding_yields_the_unit_indicator(): + # An entity-count value_variable with no count_of column counts records: + # the shape every two-child-limit household reference needs. Those + # references previously put a prose label in count_of, which the provider + # then looked up as a column and failed on. + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="affected", + entity="household", + measure="affected_households", + value=1.0, + source="test", + family="dwp_two_child_limit", + metadata={"contract_target_id": "tcl.households"}, + ) + ], + country="uk", + ) + contract = { + "tcl.households": { + "bindings": { + "policyengine": { + "kind": "baseline_flag_crosstab", + "affected_flag_variable": "affected", + "value_variable": "household_count", + "from_entity": "household", + } + } + } + } + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + assert list(adapter.tables["household"]["affected_households"]) == [1.0, 0.0, 1.0] + + +def test_crosstab_counts_a_real_value_variable_per_record(): + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="children", + entity="household", + measure="affected_children", + value=1.0, + source="test", + family="dwp_two_child_limit", + metadata={"contract_target_id": "tcl.children"}, + ) + ], + country="uk", + ) + contract = { + "tcl.children": { + "bindings": { + "policyengine": { + "kind": "baseline_flag_crosstab", + "affected_flag_variable": "affected", + "count_of": "affected_children", + "value_variable": "children", + "from_entity": "household", + } + } + } + } + + materialize_target_bindings(adapter, registry, contract, period=2025) + + # children is [3, 2, 4], masked by affected [1, 0, 1]. + assert list(adapter.tables["household"]["affected_children"]) == [3.0, 0.0, 4.0] From f24b401bcea5e3aac2f244eb0326f6306b7269f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:26:00 +0200 Subject: [PATCH 12/26] Update the two-child-limit fixtures to the corrected binding The stub adapter and national-stage frame both modelled the old children_count column. Mapped to household, uc_is_child_limit_affected sums to the number of flagged children, so it serves as both the affected flag and the affected-children count; the fixtures now carry counts rather than indicators. Caught by running the suites properly: the earlier chain piped pytest into tail, so its exit status was tail's and two real failures rode through into 760fe8b4. --- packages/microcosm-build/tests/test_uk_ledger_targets.py | 6 ++++-- .../microcosm-build/tests/test_uk_national_calibration.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_ledger_targets.py b/packages/microcosm-build/tests/test_uk_ledger_targets.py index 023ddb7f82..c3257ed8da 100644 --- a/packages/microcosm-build/tests/test_uk_ledger_targets.py +++ b/packages/microcosm-build/tests/test_uk_ledger_targets.py @@ -21,8 +21,10 @@ def __init__(self): "capital_gains": np.array([0.0, 5_000.0, 20_000.0]), }, "household": { - "uc_is_child_limit_affected": np.array([1.0, 0.0, 1.0]), - "children_count": np.array([3.0, 2.0, 4.0]), + # Mapped to household, uc_is_child_limit_affected sums to the + # number of flagged children, so it is both the affected flag + # and the affected-children count. + "uc_is_child_limit_affected": np.array([3.0, 0.0, 4.0]), }, } diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 34ce0cf461..79b1b33b1c 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -183,8 +183,10 @@ def _materialization_binding_frame( "household_id": np.arange(3, dtype="int64"), "esa_income": [10.0, 20.0, 0.0], "esa_contrib": [1.0, 2.0, 0.0], - "uc_is_child_limit_affected": [1.0, 0.0, 1.0], - "children_count": [2.0, 1.0, 3.0], + # Mapped to household, uc_is_child_limit_affected sums to the + # number of flagged children: both the affected flag and the + # affected-children count. + "uc_is_child_limit_affected": [2.0, 0.0, 3.0], } ) salary_sacrifice_metric = "hmrc/salary_sacrifice_it_relief_basic_rate" From aa2fd22a84da27648766904e07a71e5c500bafa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:05:41 +0200 Subject: [PATCH 13/26] Reduce person-level household conditions through person_household_id UKFrameTargetAdapter.household_condition built a group-membership column for every non-household entity, so a condition declaring entity "person" asked for "person_person_id" and raised. People sit directly in a household; only group entities need that lookup. Every ONS household-composition reference uses person-level conditions (is_child, age), so all ten were excluded from calibration. That left household structure unconstrained while population stayed targeted, and the solve satisfied population by inflating households rather than multiplying them: weighted benunits per household reached 1.476 against the incumbent's 1.132, from an identical unweighted 1.158. The downstream cost is Universal Credit. 75.5% of the candidate's single UC claimants end up in multi-benunit households (incumbent: 8.6%), where only one benunit claims housing costs, so the housing element reaches 47.2% of them against the incumbent's 73.9% -- despite the candidate having more renters (88.3% vs 79.2%) and higher rent, and despite beating the incumbent within both strata (92.6% vs 78.9% solo, 32.5% vs 20.4% multi). Simpson's paradox, driven entirely by composition. Median single UC award falls to GBP 8,059 against GBP 9,310, emptying the GBP 8-11k and GBP 14-19k award bands that carry 98% of the caseload shortfall. Shared UK runtime, so this cherry-picks into the spine lane. --- .../build/uk_runtime/ledger_targets.py | 5 ++ .../tests/test_uk_ledger_targets.py | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 62c70f9d10..5bbb0dbcbc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -184,6 +184,11 @@ def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: source = self.tables[entity] if entity == "household": household_ids = source["household_id"] + elif entity == "person": + # People sit directly in a household; only group entities need the + # membership lookup below, whose column would be the nonexistent + # "person_person_id" here. + household_ids = source["person_household_id"] else: people = self.tables["person"] entity_membership = f"person_{entity}_id" diff --git a/packages/microcosm-build/tests/test_uk_ledger_targets.py b/packages/microcosm-build/tests/test_uk_ledger_targets.py index c3257ed8da..85b08c0fa8 100644 --- a/packages/microcosm-build/tests/test_uk_ledger_targets.py +++ b/packages/microcosm-build/tests/test_uk_ledger_targets.py @@ -98,3 +98,75 @@ def test_materialize_uk_ledger_targets_with_stub_adapter(): assert adapter.tables["household"][ "dwp/uc/two_child_limit/children_affected" ].tolist() == [3.0, 0.0, 4.0] + + +def _composition_frame(): + """Two households: one lone parent with a child, one older couple.""" + + import pandas as pd + + from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + return Frame( + { + "person": pd.DataFrame( + { + "person_id": np.arange(5), + "person_benunit_id": [0, 0, 1, 2, 2], + "person_household_id": [0, 0, 0, 1, 1], + "is_child": [0.0, 1.0, 0.0, 0.0, 0.0], + "age": [40.0, 8.0, 30.0, 70.0, 72.0], + } + ), + "benunit": pd.DataFrame( + { + "benunit_id": np.arange(3), + "family_type": ["LONE_PARENT", "SINGLE", "COUPLE_NO_CHILDREN"], + } + ), + "household": pd.DataFrame({"household_id": np.arange(2)}), + }, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(np.array([10.0, 20.0]), WeightKind.DESIGN)}, + metadata={"time_period": "2025"}, + ) + + +def test_household_condition_reduces_person_level_conditions(): + # Person-level conditions collapse through person_household_id. Building + # the group-membership column here would ask for "person_person_id", + # which cannot exist — the failure that excluded every ONS + # household-composition reference from calibration. + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + adapter = UKFrameTargetAdapter(_composition_frame()) + + children = adapter.household_condition( + { + "variable": "is_child", + "entity": "person", + "reduce": "sum", + "operator": ">=", + "value": 1, + } + ) + + assert list(children) == [True, False] + + +def test_household_condition_still_reduces_group_entities(): + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + adapter = UKFrameTargetAdapter(_composition_frame()) + + single = adapter.household_condition( + { + "variable": "family_type", + "entity": "benunit", + "reduce": "any", + "operator": "==", + "value": "SINGLE", + } + ) + + assert list(single) == [True, False] From 53a66094990700601436b9384e5d0649733824a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:00:24 +0200 Subject: [PATCH 14/26] Pin-shape regression test for the armed run's Ledger role pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1a3274b6 crash fix landed without a test; this locks the contract it restored: the ledger_facts role pin carries exactly {sha256, size_bytes} (both feed layouts) and survives role_pins_digest, while the full provenance block is rejected — the exact shape that crashed the first armed run at input_pins_digest. Co-Authored-By: Claude Fable 5 --- .../tests/test_uk_national_build_driver.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/microcosm-build/tests/test_uk_national_build_driver.py b/packages/microcosm-build/tests/test_uk_national_build_driver.py index 4be972b64f..1f793c12ec 100644 --- a/packages/microcosm-build/tests/test_uk_national_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_national_build_driver.py @@ -13,6 +13,7 @@ from microcosm.build.gate_battery import GateBatteryBlockedError from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows +from microcosm.build.logbook_adoption import role_pins_digest from microcosm.build.uk_runtime.national_frame import ( UKStagingProvenance, _uk_source_file_fingerprint, @@ -1584,3 +1585,44 @@ def test_invalid_logbook_predecessor_refuses_before_sidecar_cleanup( for sidecar in sidecars: assert sidecar.read_text() == '{"stale": true}\n' assert not (tmp_path / "logbook-spool").exists() + + +@pytest.mark.parametrize("feed_layout", ["file", "directory"]) +def test_ledger_facts_role_pin_is_digestible(tmp_path, feed_layout) -> None: + """The armed run's Ledger role pin must survive role_pins_digest. + + First-armed-run finding (2026-08-23): _source_pins stored the full Ledger + provenance block under the 'ledger_facts' role, but role_pins_digest + accepts exactly {sha256, size_bytes}, so every armed build raised + ValueError at input_pins_digest before its first stage. Neither PR branch + fired it alone because only --ledger-facts reaches this path. + """ + + builder = _load_builder_module() + facts_body = b'{"fact": 1}\n' + if feed_layout == "file": + feed_path = tmp_path / "consumer_facts.jsonl" + feed_path.write_bytes(facts_body) + else: + feed_path = tmp_path / "feed" + feed_path.mkdir() + (feed_path / "consumer_facts.jsonl").write_bytes(facts_body) + artifact = SimpleNamespace( + path=feed_path, + facts_sha256="a" * 64, + provenance=lambda: { + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + "profile": "uk-national", + }, + ) + + pin = builder._ledger_facts_pin(artifact) + + assert set(pin) == {"sha256", "size_bytes"} + assert pin["sha256"] == "a" * 64 + assert pin["size_bytes"] == len(facts_body) + digest = role_pins_digest({"ledger_facts": pin}) + assert len(digest) == 64 + with pytest.raises(ValueError): + role_pins_digest({"ledger_facts": dict(artifact.provenance())}) From e7d392be6a0d30aea08cd7ef71aef4de5e796bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:04:21 +0200 Subject: [PATCH 15/26] Admit family_equal to the target-weight vocabulary and wire the doctrine rule into the solve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-cut of the build branch's 035ed63e under María's 2026-08-24 ruling: family_equal enters _ALLOWED_TARGET_WEIGHT_RULES with its armed-run receipts, and uk_national_target_loss_weights() derives the family-share vector — but the doctrine DEFAULT stays uniform. She passes family_equal as an explicit per-run setup while the weighting doctrine is measured; neither rule is adopted by default (run-9 receipts cut both ways). Also fixes the latent wiring defect the armed run exposed: the stage echoed target_weight_rule in its manifest but never passed a weight vector to calibrate(), so any declared rule silently solved uniform. The vector now travels explicitly; uniform maps to None, keeping the shipped identity byte-stable under the default. Co-Authored-By: Claude Fable 5 --- .../microcosm/build/uk_runtime/__init__.py | 4 ++ .../build/uk_runtime/national_calibration.py | 9 +++ .../build/uk_runtime/national_doctrine.py | 63 +++++++++++++++- .../tests/test_uk_national_calibration.py | 71 +++++++++++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index d79f1d17a1..80a2d0ea07 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -315,7 +315,9 @@ UK_NATIONAL_SOLVE_DOCTRINE, UK_NATIONAL_SOLVE_EPOCHS, UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + uk_national_target_loss_weights, ) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, @@ -663,6 +665,8 @@ "UK_NATIONAL_SOLVE_DOCTRINE", "UK_NATIONAL_SOLVE_EPOCHS", "UK_NATIONAL_TARGET_LOSS_CAP", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_national_target_loss_weights", "UKNationalStage", "UKRowwiseDatasetResult", "UKReleaseInputColumn", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index ffe754bb3c..afdae1615a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -17,6 +17,7 @@ from microcosm.build.uk_runtime.national_doctrine import ( UK_NATIONAL_SOLVE_DOCTRINE, UKNationalSolveDoctrine, + uk_national_target_loss_weights, ) from microcosm.calibrate import ( CalibrationResult, @@ -89,6 +90,13 @@ def __call__(self, frame: Frame) -> Frame: spec.family for spec in self.registry.specs ) mass_log_records_before_calibration = len(frame.mass_log) + # Target-set rows follow registry spec order, so the doctrine weight + # vector aligns positionally; under the default "uniform" rule this + # is None — the kernel's own equal weighting. + target_loss_weights = uk_national_target_loss_weights( + [spec.family for spec in self.registry.specs], + rule=self.doctrine.target_weight_rule, + ) result = calibrate( prepared, self.registry.to_target_set(), @@ -101,6 +109,7 @@ def __call__(self, frame: Frame) -> Frame: seed=self.doctrine.seed, l0_lambda=self.doctrine.l0_lambda, target_loss_cap=self.doctrine.target_loss_cap, + target_loss_weights=target_loss_weights, ) if result.skipped or len(result.problem.names) != declared: skipped = [item.name for item in result.skipped] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py index 40687602dc..3735a24d1a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py @@ -8,6 +8,8 @@ from __future__ import annotations +from collections import Counter +from collections.abc import Sequence from dataclasses import dataclass import numpy as np @@ -21,6 +23,8 @@ "UK_NATIONAL_SOLVE_DOCTRINE", "UK_NATIONAL_SOLVE_EPOCHS", "UK_NATIONAL_TARGET_LOSS_CAP", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_national_target_loss_weights", "UKNationalSolveDoctrine", ] @@ -31,9 +35,26 @@ UK_NATIONAL_TARGET_LOSS_CAP = 10.0 UK_NATIONAL_L0_LAMBDA = 0.0 UK_NATIONAL_MASS_RULE = "free" +UK_NATIONAL_TARGET_WEIGHT_RULE = "uniform" _ALLOWED_SCALE_RULES = ("default_target_loss_scales",) -_ALLOWED_TARGET_WEIGHT_RULES = ("uniform",) + +# "family_equal" gives every declared target family one equal share of the +# objective, split evenly within the family — the reviewed rule name the +# local doctrine reserved for exactly this ("a future family-level weighting +# would be a new reviewed rule name here — never a per-target vector"). +# +# Vocabulary admitted on 2026-08-23 from the first armed-run receipts: under +# "uniform" the hmrc SPI income-band family is 57% of the target surface and +# supplied 101 of the 102 references that start past the loss cap; pursuing +# them annihilated 24 population and OBR targets that were never past cap +# themselves (objective gain from annihilating a segment: +4.48 uniform vs +# +0.68 family_equal at cap 10, negative at cap 2). María's ruling +# (2026-08-24): the default stays "uniform" — family_equal is reachable only +# as an explicit, receipted per-run override while the weighting doctrine is +# still being measured (its own run-9 receipts show a 9-member family +# carrying 5.5x the per-reference weight of a 50-member one). +_ALLOWED_TARGET_WEIGHT_RULES = ("uniform", "family_equal") _ALLOWED_MASS_RULES = ("free",) @@ -47,7 +68,7 @@ class UKNationalSolveDoctrine: seed: int = UK_NATIONAL_SEED target_loss_cap: float = UK_NATIONAL_TARGET_LOSS_CAP scale_rule: str = "default_target_loss_scales" - target_weight_rule: str = "uniform" + target_weight_rule: str = UK_NATIONAL_TARGET_WEIGHT_RULE mass_rule: str = UK_NATIONAL_MASS_RULE l0_lambda: float = UK_NATIONAL_L0_LAMBDA @@ -127,3 +148,41 @@ def __post_init__(self) -> None: UK_NATIONAL_SOLVE_DOCTRINE = UKNationalSolveDoctrine() + + +def uk_national_target_loss_weights( + families: Sequence[str], + *, + rule: str = UK_NATIONAL_TARGET_WEIGHT_RULE, +) -> np.ndarray | None: + """The declared target-weight vector for one compiled register. + + ``uniform`` returns ``None`` — the kernel's own equal weighting, and the + identity this doctrine shipped with. ``family_equal`` gives each declared + family one equal share of the objective, divided evenly among its + members, so an over-supplied family cannot outvote the rest of the + surface by sheer count. + + The vector is derived from the compiled register's declared families and + nothing else: it is a rule, not a per-target vector, which is the + property the local doctrine requires of any weighting. + """ + + if rule not in _ALLOWED_TARGET_WEIGHT_RULES: + raise ValueError( + f"target_weight_rule must be one of {_ALLOWED_TARGET_WEIGHT_RULES}, " + f"got {rule!r}." + ) + if rule == "uniform": + return None + labels = [str(family) for family in families] + if not labels: + raise ValueError("family_equal weighting requires a non-empty register.") + if any(not label for label in labels): + raise ValueError("every target must declare a family for family_equal.") + counts = Counter(labels) + family_count = len(counts) + return np.asarray( + [1.0 / (family_count * counts[label]) for label in labels], + dtype=np.float64, + ) diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 79b1b33b1c..6f9bca424c 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -19,7 +19,9 @@ UK_NATIONAL_SOLVE_DOCTRINE, UK_NATIONAL_SOLVE_EPOCHS, UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + uk_national_target_loss_weights, ) from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation from microcosm.build.uk_runtime.national_build import write_uk_national_frame @@ -548,9 +550,34 @@ def test_national_doctrine_constants_are_the_declared_contract() -> None: assert UK_NATIONAL_TARGET_LOSS_CAP == 10.0 assert UK_NATIONAL_L0_LAMBDA == 0.0 assert UK_NATIONAL_MASS_RULE == "free" + # María's ruling (2026-08-24): family_equal is vocabulary, never the + # default — she passes it as an explicit per-run override. + assert UK_NATIONAL_TARGET_WEIGHT_RULE == "uniform" assert UK_NATIONAL_SOLVE_DOCTRINE == UKNationalSolveDoctrine() assert UK_NATIONAL_SOLVE_DOCTRINE.scale_rule == "default_target_loss_scales" assert UK_NATIONAL_SOLVE_DOCTRINE.target_weight_rule == "uniform" + assert UKNationalSolveDoctrine(target_weight_rule="family_equal") + + +def test_family_equal_gives_each_family_one_equal_share() -> None: + weights = uk_national_target_loss_weights( + ["hmrc"] * 3 + ["obr"], rule="family_equal" + ) + assert weights is not None + assert weights.sum() == pytest.approx(1.0) + # Three hmrc rows share half the objective; the single obr row holds the + # other half, so an over-supplied family cannot outvote by count. + assert weights[:3].sum() == pytest.approx(0.5) + assert weights[3] == pytest.approx(0.5) + + +def test_uniform_rule_defers_to_the_kernel_default() -> None: + assert uk_national_target_loss_weights(["hmrc", "obr"], rule="uniform") is None + + +def test_family_equal_refuses_an_undeclared_family() -> None: + with pytest.raises(ValueError, match="must declare a family"): + uk_national_target_loss_weights(["hmrc", ""], rule="family_equal") def test_national_doctrine_rejects_tampered_bounds() -> None: @@ -572,3 +599,47 @@ def test_national_doctrine_rejects_tampered_bounds() -> None: UKNationalSolveDoctrine(mass_rule="conserve") with pytest.raises(ValueError, match="l0_lambda"): UKNationalSolveDoctrine(l0_lambda=-0.1) + + +@pytest.mark.parametrize( + ("rule", "expected"), + [("uniform", None), ("family_equal", [1.0])], +) +def test_doctrine_target_weight_rule_reaches_the_solver( + monkeypatch, rule, expected +) -> None: + """The declared rule must reach calibrate(), not just the manifest echo. + + First-armed-run finding (2026-08-23): the doctrine declared a + target_weight_rule the stage never passed to the kernel, so the solve + silently ran uniform whatever the doctrine said. The doctrine vector now + travels explicitly; "uniform" maps to None — the kernel's own default — + so the shipped identity is unchanged under the default rule. + """ + + from microcosm.calibrate import calibrate as real_calibrate + + captured: dict[str, object] = {} + + def capturing_calibrate(*args, **kwargs): + captured["target_loss_weights"] = kwargs["target_loss_weights"] + return real_calibrate(*args, **kwargs) + + monkeypatch.setattr( + "microcosm.build.uk_runtime.national_calibration.calibrate", + capturing_calibrate, + ) + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5, target_weight_rule=rule), + ) + + stage(_frame()) + + weights = captured["target_loss_weights"] + if expected is None: + assert weights is None + else: + assert weights is not None + assert weights.tolist() == expected From 91074ddd27e11a8527bba4521d907f5e3074d57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:07:31 +0200 Subject: [PATCH 16/26] Extend the loader-guarantee test to the corrected crosstab binding shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-child-limit rebinding (760fe8b4) replaced the prose count_of labels with real value_variable columns; the chronicle loader-guarantee test still demanded count_of on every baseline_flag_crosstab binding. The guarantee now matches the provider: affected_flag_variable plus either counted-column spelling. Latent on the build branch too — this suite was never re-run there after the rebinding. Co-Authored-By: Claude Fable 5 --- .../microcosm-build/tests/test_uk_national_targets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_national_targets.py b/packages/microcosm-build/tests/test_uk_national_targets.py index 1dd1c672c4..133d4e4368 100644 --- a/packages/microcosm-build/tests/test_uk_national_targets.py +++ b/packages/microcosm-build/tests/test_uk_national_targets.py @@ -278,9 +278,11 @@ def test_uk_national_targets_declare_chronicle_loader_guarantees(): "gate_comparison", } <= set(binding), target["target_id"] elif kind == "baseline_flag_crosstab": - assert {"affected_flag_variable", "count_of"} <= set(binding), target[ - "target_id" - ] + assert "affected_flag_variable" in binding, target["target_id"] + # The two-child-limit rebinding counts a real value_variable per + # record; count_of survives as the legacy spelling the provider + # still accepts. Either key must name the counted column. + assert {"value_variable", "count_of"} & set(binding), target["target_id"] if "reduce" in binding: assert binding["reduce"] in BINDING_REDUCERS, target["target_id"] From 96e73047a631e45a8b93366101d256d68937cec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:37:07 +0200 Subject: [PATCH 17/26] Relocate the UK national H5 I/O into national_frame and add the shared measure-resolution loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two moves the calibration seam needs, both behaviour-neutral for the June path. The H5 reader/writer pair moves next to uk_national_frame and validate_uk_national_frame — its natural home — with national_build re-exporting every name, so the legacy driver and its suites are unchanged. The seam can then load and stage frames without importing the legacy build module at all. target_materialization grows the country-agnostic half of the assessment harness's resolution loop: probe-adapter rounds, once-per-round dedup of a shared missing key, and a fail-loud finish. The country-specific step is injected as a provider, so nothing about policyengine-uk enters the shared module. Where the harness excluded an unresolvable reference with a receipt, production raises: register pruning belongs to a reviewed exclusion register, never to a silent loop. assert_calibration_input_finite is the seam's NaN fence — it lists every offending column at once instead of failing on the first. Co-Authored-By: Claude Fable 5 --- .../build/frame_serializer_registry.py | 4 +- .../microcosm/build/target_materialization.py | 223 ++++++++++++- .../build/uk_runtime/national_build.py | 303 +++--------------- .../build/uk_runtime/national_frame.py | 230 +++++++++++++ .../tests/test_frame_serializer_registry.py | 2 +- .../tests/test_uk_national_build.py | 6 +- .../tests/test_uk_rowwise_weight_metadata.py | 6 +- 7 files changed, 498 insertions(+), 276 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py index b903cb5a21..4d83c2f549 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py @@ -77,7 +77,7 @@ class HdfWriteExclusion: FrameSerializerSpec( serializer_id="uk_single_year_h5", writer=HdfWriteSite( - "packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py", + "packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py", "_write_uk_single_year_tables", ), backend="pandas.HDFStore table", @@ -177,7 +177,7 @@ class HdfWriteExclusion: HdfWriteExclusion( exclusion_id="uk_weight_root_attrs", writer=HdfWriteSite( - "packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py", + "packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py", "_write_weight_metadata", ), reason="Adds weight-kind and mass-log root attributes only.", diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 3c3e85191c..802d12aea8 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -6,7 +6,8 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any +from types import MappingProxyType +from typing import Any, Protocol import numpy as np @@ -38,6 +39,21 @@ r"([\d,]+(?:\.\d+)?)\s*(?:or more|or over|and over|and above|\+)", re.IGNORECASE, ) +_MISSING_COLUMN = re.compile(r"^'(?:([a-z_0-9]+)\.)?([A-Za-z_0-9]+)'$") + + +class MeasureProvider(Protocol): + """Provider of adapter-level variables needed for target materialization. + + ``knows`` answers whether the provider can compute a variable for a given + entity. ``compute`` returns the values and a short receipted route. The + returned array is injected into fresh probe adapters only; it is never + written back to the source frame. + """ + + def knows(self, entity: str, variable: str) -> bool: ... + + def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: ... @dataclass(frozen=True) @@ -49,6 +65,22 @@ class MaterializationSkip: reason: str +@dataclass(frozen=True) +class MeasureResolution: + """Adapter-level variables resolved for target materialization.""" + + measure_inputs: Mapping[tuple[str, str], np.ndarray] + receipt: Mapping[str, Any] + + +class MeasureResolutionError(RuntimeError): + """Raised when simulated measure resolution cannot bind the registry.""" + + def __init__(self, message: str, *, receipt: Mapping[str, Any]): + super().__init__(message) + self.receipt = dict(receipt) + + @dataclass(frozen=True) class TargetMaterializationResult: """Prepared target columns and skipped-measure diagnostics.""" @@ -67,6 +99,195 @@ def report(self) -> dict[str, object]: } +def assert_calibration_input_finite(frame: Any) -> None: + """Refuse NaN-bearing float inputs before calibration materialization.""" + + failures: list[str] = [] + for entity in frame.entities: + table = frame.table(entity) + for column in table.columns: + series = table[column] + if getattr(series.dtype, "kind", None) != "f": + continue + count = int(series.isna().sum()) + if count: + failures.append(f"{entity}.{column}: {count}") + if failures: + raise ValueError( + "Calibration input contains NaN float values: " + "; ".join(failures) + ) + + +def resolve_target_measures( + adapter_factory: Callable[[], Any], + registry: TargetRegistry, + provider: MeasureProvider, + *, + period: int | str, + max_rounds: int = 8, + contract_targets: Mapping[str, Mapping[str, Any]] | None = None, +) -> MeasureResolution: + """Resolve provider-computed inputs until target materialization binds. + + ``adapter_factory`` must return a fresh adapter over the same frame each + round. Previously resolved values are injected into those probe adapters' + table copies only, then the shared target-binding interpreter is retried. + The source frame is never mutated. + """ + + contract = _measure_resolution_contract( + registry, provider, contract_targets=contract_targets + ) + measure_inputs: dict[tuple[str, str], np.ndarray] = {} + receipt: dict[str, Any] = { + "rounds": [], + "attached": {}, + "provider": _measure_provider_receipt(provider), + } + all_skips: list[dict[str, Any]] = [] + + for round_index in range(max_rounds): + probe = adapter_factory() + _inject_measure_inputs(probe, measure_inputs) + result = materialize_target_bindings( + probe, + registry, + contract, + period=period, + ) + skipped = tuple(result.skipped) + round_receipt = { + "round": round_index, + "provided_before": len(measure_inputs), + "skipped": [skip.__dict__ for skip in skipped], + } + receipt["rounds"].append(round_receipt) + all_skips.extend( + {**skip.__dict__, "round": round_index} for skip in skipped + ) + if not skipped: + return MeasureResolution( + measure_inputs=MappingProxyType(dict(measure_inputs)), + receipt=MappingProxyType(receipt), + ) + + provided_before = set(measure_inputs) + provided_this_round: set[tuple[str, str]] = set() + progressed = False + for skip in skipped: + reason = str(skip.reason) + if "counterfactual_delta" in reason or "counterfactual delta" in reason: + _raise_measure_resolution( + "counterfactual target measure cannot be resolved", + receipt, + all_skips, + ) + match = _MISSING_COLUMN.match(reason) + if match is None: + _raise_measure_resolution( + f"unparseable target materialization skip reason: {reason}", + receipt, + all_skips, + ) + entity, variable = match.group(1), match.group(2) + if entity is None: + entity = _provider_entity_for(provider, variable) + if entity is None: + _raise_measure_resolution( + f"target measure {variable!r} did not identify an entity", + receipt, + all_skips, + ) + key = (entity, variable) + if key in provided_this_round: + continue + if key in provided_before: + _raise_measure_resolution( + f"{entity}.{variable} remained unmaterializable after injection", + receipt, + all_skips, + ) + if not provider.knows(entity, variable): + _raise_measure_resolution( + f"provider does not know {entity}.{variable}", + receipt, + all_skips, + ) + try: + values, route = provider.compute(entity, variable) + except Exception as exc: # noqa: BLE001 - receipt preserves cause + _raise_measure_resolution( + f"provider failed computing {entity}.{variable}: {exc}", + receipt, + all_skips, + ) + measure_inputs[key] = np.asarray(values) + provided_this_round.add(key) + receipt["attached"][f"{entity}.{variable}"] = route + progressed = True + if not progressed: + _raise_measure_resolution( + f"measure materialization made no progress on round {round_index}", + receipt, + all_skips, + ) + + _raise_measure_resolution( + f"measure materialization did not converge after {max_rounds} rounds", + receipt, + all_skips, + ) + + +def _measure_resolution_contract( + registry: TargetRegistry, + provider: MeasureProvider, + *, + contract_targets: Mapping[str, Mapping[str, Any]] | None, +) -> Mapping[str, Mapping[str, Any]]: + if contract_targets is not None: + return contract_targets + for owner in (registry, provider): + value = getattr(owner, "contract_targets", None) + if value is not None: + return value + raise TypeError( + "resolve_target_measures requires contract_targets, or a registry/provider " + "with a contract_targets attribute." + ) + + +def _measure_provider_receipt(provider: MeasureProvider) -> Mapping[str, Any]: + receipt = getattr(provider, "receipt", None) + if callable(receipt): + return dict(receipt()) + return {"class": type(provider).__name__} + + +def _provider_entity_for(provider: MeasureProvider, variable: str) -> str | None: + entity_for = getattr(provider, "entity_for", None) + if callable(entity_for): + value = entity_for(variable) + return None if value is None else str(value) + return None + + +def _inject_measure_inputs( + adapter: Any, measure_inputs: Mapping[tuple[str, str], np.ndarray] +) -> None: + for (entity, variable), values in measure_inputs.items(): + adapter.tables[entity][variable] = values + + +def _raise_measure_resolution( + message: str, + receipt: dict[str, Any], + skips: list[dict[str, Any]], +) -> None: + receipt["skips"] = list(skips) + raise MeasureResolutionError(message, receipt=receipt) + + def _band_lower_edge( spec: Any, binding: Mapping[str, Any] ) -> float | None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index bc2c5a2da3..4808ead157 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -19,7 +19,6 @@ from __future__ import annotations import json -import uuid from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date @@ -27,8 +26,6 @@ from types import SimpleNamespace from typing import Any -import pandas as pd - import microcosm.build.uk_runtime.national_frame as _national_frame from microcosm.build.country_spec import load_country_spec from microcosm.build.frame_sampling import ( @@ -48,12 +45,47 @@ from microcosm.build.plan import StagePlan, StageRecord from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance, + UK_HOUSEHOLD_WEIGHT_KIND_ATTR as UK_HOUSEHOLD_WEIGHT_KIND_ATTR, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_MASS_LOG_ATTR as UK_MASS_LOG_ATTR, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_H5_TABLES as UK_NATIONAL_H5_TABLES, +) +from microcosm.build.uk_runtime.national_frame import ( + UKStagingProvenance as UKStagingProvenance, +) +from microcosm.build.uk_runtime.national_frame import ( + _mass_log_from_stored as _mass_log_from_stored, +) +from microcosm.build.uk_runtime.national_frame import ( + _read_uk_national_tables as _read_uk_national_tables, +) +from microcosm.build.uk_runtime.national_frame import ( + _read_weight_metadata as _read_weight_metadata, +) +from microcosm.build.uk_runtime.national_frame import ( + _weight_kind_from_stored as _weight_kind_from_stored, +) +from microcosm.build.uk_runtime.national_frame import ( + _write_uk_single_year_tables as _write_uk_single_year_tables, +) +from microcosm.build.uk_runtime.national_frame import ( + _write_weight_metadata as _write_weight_metadata, +) +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame as load_uk_national_frame, +) +from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.national_frame import ( + write_uk_national_frame as write_uk_national_frame, +) from microcosm.build.uk_runtime.national_sampling import ( UK_SAMPLE_SEED_DEFAULT, sample_uk_national_frame, @@ -68,14 +100,7 @@ exclusion_evaluation_date, ) from microcosm.calibrate import TargetRegistry -from microcosm.frame import ( - Frame, - MassChangeRecord, - WeightKind, - engine_tables, - put_frame_table, - read_frame_table, -) +from microcosm.frame import Frame __all__ = [ "UKNationalBuildResult", @@ -90,11 +115,6 @@ "write_uk_national_frame", ] -UK_NATIONAL_H5_TABLES = ("person", "benunit", "household", "time_period") -UK_HOUSEHOLD_WEIGHT_KIND_ATTR = "populace_household_weight_kind" -UK_MASS_LOG_ATTR = "populace_mass_log_json" - - # The fingerprint pair moved to national_frame with the Frame carrier; the # re-import keeps this module's existing consumers (hmrc_restoration binds # certified candidates to it) on their current import path. @@ -175,169 +195,6 @@ def input_coverage_path(self) -> Path: return self.terminal_gate_path -def _read_uk_national_tables( - path: str | Path, -) -> tuple[dict[str, Any], _UKSourceFileFingerprint, Path]: - """Read a compact UK single-year H5's payload with the race guard. - - Shared body of both loaders: suffix check, symlink resolution, and the - fingerprint-before/after guard binding the returned tables to one stable - set of bytes. - """ - - requested_path = Path(path).expanduser() - if requested_path.suffix != ".h5": - raise ValueError("UK national dataset path must end with '.h5'.") - # Hugging Face cache entries retain the requested ``.h5`` name as a - # symlink whose content-addressed blob target has no suffix. Validate the - # caller-facing artifact name before resolving it, while binding all - # provenance and stable-byte checks to the actual opened file. - input_path = requested_path.resolve() - if not input_path.is_file(): - raise FileNotFoundError(f"UK national dataset not found: {input_path}.") - - fingerprint_before = _uk_source_file_fingerprint(input_path) - stored_kind, stored_mass_log = _read_weight_metadata(input_path) - with pd.HDFStore(input_path, mode="r") as store: - keys = {key.lstrip("/") for key in store.keys()} - missing = sorted(set(UK_NATIONAL_H5_TABLES) - keys) - if missing: - raise ValueError(f"UK national dataset is missing table(s): {missing}.") - raw_period = store["time_period"] - if len(raw_period) != 1: - raise ValueError( - "UK national dataset time_period must contain exactly one value." - ) - payload = { - "person": read_frame_table(store, "person"), - "benunit": read_frame_table(store, "benunit"), - "household": read_frame_table(store, "household"), - "time_period": str(raw_period.iloc[0]), - "household_weight_kind": _weight_kind_from_stored(stored_kind), - "mass_log": _mass_log_from_stored(stored_mass_log), - } - fingerprint_after = _uk_source_file_fingerprint(input_path) - if fingerprint_after != fingerprint_before: - raise RuntimeError( - "UK national source H5 changed while it was being loaded; refusing " - "to bind mixed or stale bytes to build stages." - ) - return payload, fingerprint_after, input_path - - -def load_uk_national_frame( - path: str | Path, -) -> tuple[Frame, UKStagingProvenance]: - """Load a compact UK single-year H5 as a validated Frame plus provenance. - - Frame construction is where the structural invariants are enforced - (linkage in both directions, sorted group ids, column uniqueness, weight - health); :func:`validate_uk_national_frame` adds the UK residue. The - provenance record travels beside the frame — it is the same source-path - and fingerprint identity the shadow carrier smuggled in private fields. - """ - - payload, fingerprint, input_path = _read_uk_national_tables(path) - frame = uk_national_frame( - person=payload["person"], - benunit=payload["benunit"], - household=payload["household"], - time_period=payload["time_period"], - weight_kind=payload["household_weight_kind"], - mass_log=payload["mass_log"], - ) - validate_uk_national_frame(frame) - return frame, UKStagingProvenance(source_h5=input_path, fingerprint=fingerprint) - - -def _write_uk_single_year_tables( - *, - person: pd.DataFrame, - benunit: pd.DataFrame, - household: pd.DataFrame, - time_period: str, - weight_kind: WeightKind, - mass_log: tuple[MassChangeRecord, ...], - path: Path, -) -> Path: - """The one physical writer for every UK single-year H5. - - Tables and the weight-kind/mass-log attrs must land together: writing - them into a temporary file and renaming keeps a metadata failure from - leaving a complete-looking attr-less H5 that would silently default to - DESIGN semantics on the next read. - """ - - path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp.h5") - try: - with pd.HDFStore(temporary_path) as store: - put_frame_table( - store, - "person", - person, - preferred_format="table", - data_columns=True, - ) - put_frame_table( - store, - "benunit", - benunit, - preferred_format="table", - data_columns=True, - ) - put_frame_table( - store, - "household", - household, - preferred_format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series([time_period]), - format="table", - data_columns=True, - ) - _write_weight_metadata( - temporary_path, weight_kind=weight_kind, mass_log=mass_log - ) - temporary_path.replace(path) - finally: - temporary_path.unlink(missing_ok=True) - return path - - -def write_uk_national_frame(frame: Frame, path: str | Path) -> Path: - """Atomically write a validated UK national Frame as a staging H5. - - The engine-facing payload is materialized through the shared - :func:`microcosm.frame.engine_tables`, so the typed weights are - authoritative and the ``household_weight`` column is overwritten in - place (preserving its position, and therefore the artifact's column - order). - """ - - validate_uk_national_frame(frame) - output_path = Path(path) - if output_path.suffix != ".h5": - raise ValueError("UK national staging path must end with '.h5'.") - # The export contract is pinned, not inherited: household weights are the - # one materialized vector, so a frame that somehow reached this point - # with other typed weights cannot grow reserved columns the UK loader - # rejects (validate_uk_national_frame refuses such frames anyway). - tables = engine_tables(frame, weighted_entities=("household",)) - return _write_uk_single_year_tables( - person=tables["person"], - benunit=tables["benunit"], - household=tables["household"], - time_period=uk_time_period(frame), - weight_kind=uk_household_weight_kind(frame), - mass_log=frame.mass_log, - path=output_path, - ) - - def build_uk_national_dataset( *, input_h5: str | Path, @@ -940,89 +797,3 @@ def _write_input_coverage_diagnostic(path: Path, gate: GateResult) -> None: encoding="utf-8", ) temporary_path.replace(path) - - -def _weight_kind_from_stored(value: object) -> WeightKind: - if isinstance(value, bytes): - value = value.decode("utf-8") - try: - return WeightKind(str(value)) - except ValueError as exc: - raise ValueError(f"Unknown stored UK household weight kind {value!r}.") from exc - - -def _read_weight_metadata(path: Path) -> tuple[object, object]: - try: - import h5py - except ImportError as exc: # pragma: no cover - UK H5 runtime dependency - raise RuntimeError("h5py is required to read UK national metadata.") from exc - with h5py.File(path, mode="r") as file: - return ( - file.attrs.get( - UK_HOUSEHOLD_WEIGHT_KIND_ATTR, - WeightKind.DESIGN.value, - ), - file.attrs.get(UK_MASS_LOG_ATTR, "[]"), - ) - - -def _write_weight_metadata( - path: Path, - *, - weight_kind: WeightKind, - mass_log: tuple[MassChangeRecord, ...], -) -> None: - try: - import h5py - except ImportError as exc: # pragma: no cover - UK H5 runtime dependency - raise RuntimeError("h5py is required to write UK national metadata.") from exc - with h5py.File(path, mode="r+") as file: - file.attrs[UK_HOUSEHOLD_WEIGHT_KIND_ATTR] = weight_kind.value - file.attrs[UK_MASS_LOG_ATTR] = json.dumps( - [_mass_change_record_payload(record) for record in mass_log], - sort_keys=True, - ) - - -def _mass_log_from_stored(value: object) -> tuple[MassChangeRecord, ...]: - if isinstance(value, bytes): - value = value.decode("utf-8") - try: - payload = json.loads(str(value)) - except json.JSONDecodeError as exc: - raise ValueError("Stored UK microcosm mass log is not valid JSON.") from exc - if not isinstance(payload, list): - raise ValueError("Stored UK microcosm mass log must be a JSON list.") - records: list[MassChangeRecord] = [] - for entry in payload: - if not isinstance(entry, dict): - raise ValueError("Stored UK microcosm mass-log entries must be objects.") - try: - records.append( - MassChangeRecord( - entity=str(entry["entity"]), - old_total=float(entry["old_total"]), - new_total=float(entry["new_total"]), - declared_factor=( - None - if entry.get("declared_factor") is None - else float(entry["declared_factor"]) - ), - reason=str(entry["reason"]), - ) - ) - except (KeyError, TypeError, ValueError) as exc: - raise ValueError( - "Stored UK microcosm mass-log entry is malformed." - ) from exc - return tuple(records) - - -def _mass_change_record_payload(record: MassChangeRecord) -> dict[str, object]: - return { - "entity": record.entity, - "old_total": record.old_total, - "new_total": record.new_total, - "declared_factor": record.declared_factor, - "reason": record.reason, - } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 2690adfb93..3d3a0640b3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -24,9 +24,12 @@ from __future__ import annotations +import json +import uuid from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -38,23 +41,34 @@ WeightKind, Weights, engine_tables, + put_frame_table, + read_frame_table, ) __all__ = [ "UK_EXPORTED_WEIGHT_COLUMNS", + "UK_HOUSEHOLD_WEIGHT_KIND_ATTR", + "UK_MASS_LOG_ATTR", + "UK_NATIONAL_H5_TABLES", "UK_NATIONAL_SCHEMA", "UK_TIME_PERIOD_METADATA_KEY", "UKStagingProvenance", + "load_uk_national_frame", "uk_household_weight_kind", "uk_national_frame", "uk_time_period", "validate_uk_national_frame", + "write_uk_national_frame", ] UK_NATIONAL_SCHEMA = EntitySchema(group_entities=("benunit", "household")) UK_TIME_PERIOD_METADATA_KEY = "time_period" +UK_NATIONAL_H5_TABLES = ("person", "benunit", "household", "time_period") +UK_HOUSEHOLD_WEIGHT_KIND_ATTR = "populace_household_weight_kind" +UK_MASS_LOG_ATTR = "populace_mass_log_json" + # The persisted weight columns of the materialized export contract above: # real columns on the tables, but plumbing rather than input mass. Consumers # that total column mass (the input-mass parity surface, its reference @@ -115,6 +129,92 @@ def _uk_source_file_fingerprint(path: Path) -> _UKSourceFileFingerprint: ) +def _weight_kind_from_stored(value: object) -> WeightKind: + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return WeightKind(str(value)) + except ValueError as exc: + raise ValueError(f"Unknown stored UK household weight kind {value!r}.") from exc + + +def _read_weight_metadata(path: Path) -> tuple[object, object]: + try: + import h5py + except ImportError as exc: # pragma: no cover - UK H5 runtime dependency + raise RuntimeError("h5py is required to read UK national metadata.") from exc + with h5py.File(path, mode="r") as file: + return ( + file.attrs.get( + UK_HOUSEHOLD_WEIGHT_KIND_ATTR, + WeightKind.DESIGN.value, + ), + file.attrs.get(UK_MASS_LOG_ATTR, "[]"), + ) + + +def _write_weight_metadata( + path: Path, + *, + weight_kind: WeightKind, + mass_log: tuple[MassChangeRecord, ...], +) -> None: + try: + import h5py + except ImportError as exc: # pragma: no cover - UK H5 runtime dependency + raise RuntimeError("h5py is required to write UK national metadata.") from exc + with h5py.File(path, mode="r+") as file: + file.attrs[UK_HOUSEHOLD_WEIGHT_KIND_ATTR] = weight_kind.value + file.attrs[UK_MASS_LOG_ATTR] = json.dumps( + [_mass_change_record_payload(record) for record in mass_log], + sort_keys=True, + ) + + +def _mass_log_from_stored(value: object) -> tuple[MassChangeRecord, ...]: + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + payload = json.loads(str(value)) + except json.JSONDecodeError as exc: + raise ValueError("Stored UK microcosm mass log is not valid JSON.") from exc + if not isinstance(payload, list): + raise ValueError("Stored UK microcosm mass log must be a JSON list.") + records: list[MassChangeRecord] = [] + for entry in payload: + if not isinstance(entry, dict): + raise ValueError("Stored UK microcosm mass-log entries must be objects.") + try: + records.append( + MassChangeRecord( + entity=str(entry["entity"]), + old_total=float(entry["old_total"]), + new_total=float(entry["new_total"]), + declared_factor=( + None + if entry.get("declared_factor") is None + else float(entry["declared_factor"]) + ), + reason=str(entry["reason"]), + ) + ) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError( + "Stored UK microcosm mass-log entry is malformed." + ) from exc + return tuple(records) + + +def _mass_change_record_payload(record: MassChangeRecord) -> dict[str, object]: + return { + "entity": record.entity, + "old_total": record.old_total, + "new_total": record.new_total, + "declared_factor": record.declared_factor, + "reason": record.reason, + } + + @dataclass(frozen=True) class UKStagingProvenance: """Where a UK national frame's bytes came from, carried beside it. @@ -127,6 +227,65 @@ class UKStagingProvenance: fingerprint: _UKSourceFileFingerprint +def _read_uk_national_tables( + path: str | Path, +) -> tuple[dict[str, Any], _UKSourceFileFingerprint, Path]: + """Read a compact UK single-year H5's payload with the race guard.""" + + requested_path = Path(path).expanduser() + if requested_path.suffix != ".h5": + raise ValueError("UK national dataset path must end with '.h5'.") + input_path = requested_path.resolve() + if not input_path.is_file(): + raise FileNotFoundError(f"UK national dataset not found: {input_path}.") + + fingerprint_before = _uk_source_file_fingerprint(input_path) + stored_kind, stored_mass_log = _read_weight_metadata(input_path) + with pd.HDFStore(input_path, mode="r") as store: + keys = {key.lstrip("/") for key in store.keys()} + missing = sorted(set(UK_NATIONAL_H5_TABLES) - keys) + if missing: + raise ValueError(f"UK national dataset is missing table(s): {missing}.") + raw_period = store["time_period"] + if len(raw_period) != 1: + raise ValueError( + "UK national dataset time_period must contain exactly one value." + ) + payload = { + "person": read_frame_table(store, "person"), + "benunit": read_frame_table(store, "benunit"), + "household": read_frame_table(store, "household"), + "time_period": str(raw_period.iloc[0]), + "household_weight_kind": _weight_kind_from_stored(stored_kind), + "mass_log": _mass_log_from_stored(stored_mass_log), + } + fingerprint_after = _uk_source_file_fingerprint(input_path) + if fingerprint_after != fingerprint_before: + raise RuntimeError( + "UK national source H5 changed while it was being loaded; refusing " + "to bind mixed or stale bytes to build stages." + ) + return payload, fingerprint_after, input_path + + +def load_uk_national_frame( + path: str | Path, +) -> tuple[Frame, UKStagingProvenance]: + """Load a compact UK single-year H5 as a validated Frame plus provenance.""" + + payload, fingerprint, input_path = _read_uk_national_tables(path) + frame = uk_national_frame( + person=payload["person"], + benunit=payload["benunit"], + household=payload["household"], + time_period=payload["time_period"], + weight_kind=payload["household_weight_kind"], + mass_log=payload["mass_log"], + ) + validate_uk_national_frame(frame) + return frame, UKStagingProvenance(source_h5=input_path, fingerprint=fingerprint) + + def uk_national_frame( *, person: pd.DataFrame, @@ -194,6 +353,77 @@ def uk_household_weight_kind(frame: Frame) -> WeightKind: return frame.weights_for("household").kind +def _write_uk_single_year_tables( + *, + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + time_period: str, + weight_kind: WeightKind, + mass_log: tuple[MassChangeRecord, ...], + path: Path, +) -> Path: + """The one physical writer for every UK single-year H5.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp.h5") + try: + with pd.HDFStore(temporary_path) as store: + put_frame_table( + store, + "person", + person, + preferred_format="table", + data_columns=True, + ) + put_frame_table( + store, + "benunit", + benunit, + preferred_format="table", + data_columns=True, + ) + put_frame_table( + store, + "household", + household, + preferred_format="table", + data_columns=True, + ) + store.put( + "time_period", + pd.Series([time_period]), + format="table", + data_columns=True, + ) + _write_weight_metadata( + temporary_path, weight_kind=weight_kind, mass_log=mass_log + ) + temporary_path.replace(path) + finally: + temporary_path.unlink(missing_ok=True) + return path + + +def write_uk_national_frame(frame: Frame, path: str | Path) -> Path: + """Atomically write a validated UK national Frame as a staging H5.""" + + validate_uk_national_frame(frame) + output_path = Path(path) + if output_path.suffix != ".h5": + raise ValueError("UK national staging path must end with '.h5'.") + tables = engine_tables(frame, weighted_entities=("household",)) + return _write_uk_single_year_tables( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + path=output_path, + ) + + @dataclass(frozen=True) class _UKGateSurface: """The duck-attr evidence surface the legacy UK gate modules read. diff --git a/packages/microcosm-build/tests/test_frame_serializer_registry.py b/packages/microcosm-build/tests/test_frame_serializer_registry.py index 04f5df7da4..11edf2aeb7 100644 --- a/packages/microcosm-build/tests/test_frame_serializer_registry.py +++ b/packages/microcosm-build/tests/test_frame_serializer_registry.py @@ -20,7 +20,7 @@ HDF_WRITE_EXCLUSIONS, FrameSerializerSpec, ) -from microcosm.build.uk_runtime.national_build import ( +from microcosm.build.uk_runtime.national_frame import ( _read_uk_national_tables, _write_uk_single_year_tables, ) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index 53815df963..3e06e92638 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -1597,7 +1597,7 @@ def test_atomic_writer_cleans_temporary_h5_after_write_failure( monkeypatch, tmp_path ) -> None: pytest.importorskip("tables") - from microcosm.build.uk_runtime import national_build + from microcosm.build.uk_runtime import national_frame input_h5 = tmp_path / "base.h5" staging_h5 = tmp_path / "staging.h5" @@ -1609,10 +1609,10 @@ def fail_store(path, *_args, **_kwargs): Path(path).write_bytes(b"partial") raise OSError("simulated HDF write failure") - monkeypatch.setattr(national_build.pd, "HDFStore", fail_store) + monkeypatch.setattr(national_frame.pd, "HDFStore", fail_store) with pytest.raises(OSError, match="simulated HDF write failure"): - national_build.write_uk_national_frame(frame, staging_h5) + national_frame.write_uk_national_frame(frame, staging_h5) assert staging_h5.read_bytes() == b"previous-good-artifact" assert list(tmp_path.glob(".staging.h5.*.tmp.h5")) == [] diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index a7c83e7428..117a9d6b0c 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -346,9 +346,9 @@ def test_write_is_atomic_when_metadata_write_fails( monkeypatch: pytest.MonkeyPatch, tmp_path ) -> None: pytest.importorskip("tables") - # The rowwise writer routes through the national seam's shared physical - # writer, so the metadata-failure seam now lives there. - from microcosm.build.uk_runtime import national_build as module + # The rowwise writer routes through the national frame seam's shared + # physical writer, so the metadata-failure seam lives there. + from microcosm.build.uk_runtime import national_frame as module result = clone_uk_dataset_tables_with_rowwise_geography( person=person_frame(), From 971a9699c3c4d89e3f1fa6f4a3c1d26d74c07637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:37:21 +0200 Subject: [PATCH 18/26] Give the UK seam its simulated-measure provider and per-run doctrine overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 187 of the 388 compiled references bind model outputs — income tax, UC — that no production code ever computed onto a frame; the stage's adapter reads stored columns only, which is why the first armed run could not bind them. The provider computes them through policyengine-uk on four receipted routes and injects them at adapter level, where names are table-scoped, so the Frame's global column-uniqueness rule is never touched and the staged artifact carries no scratch column. The five salary-sacrifice counterfactual references cannot bind without a live counterfactual run, so they leave the register through a reviewed exclusion file with a written reason each, not through a silent skip. A stale entry that matches nothing is an error. Doctrine defaults stay the reviewed constants. uk_doctrine_with_overrides builds an effective doctrine through the frozen dataclass, so the closed vocabularies still apply, and returns the diff against v1 — recorded as an explicit deviation wherever the run is described. Only the four fields calibration work actually turns are overridable; seed, weight ratio, mass and scale rules stay reviewed constants and refuse. Co-Authored-By: Claude Fable 5 --- .../uk/calibration_measure_exclusions.json | 30 +++ .../microcosm/build/uk/country_package.json | 5 + .../microcosm/build/uk_runtime/__init__.py | 2 + .../build/uk_runtime/measure_simulation.py | 218 ++++++++++++++++ .../build/uk_runtime/national_calibration.py | 58 ++++- .../build/uk_runtime/national_doctrine.py | 44 +++- .../tests/test_country_spec.py | 2 + .../tests/test_spec_engine_country_bundles.py | 2 +- .../tests/test_target_materialization.py | 233 +++++++++++++++++- .../tests/test_uk_measure_simulation.py | 231 +++++++++++++++++ .../tests/test_uk_national_calibration.py | 120 +++++++++ 11 files changed, 936 insertions(+), 9 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py create mode 100644 packages/microcosm-build/tests/test_uk_measure_simulation.py diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json new file mode 100644 index 0000000000..087d6637a8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "exclusions": [ + { + "name": "hmrc.salary_sacrifice.it_relief_basic_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.it_relief_higher_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.it_relief_additional_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employee", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employer", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 9bc16e859b..7d0cb04ffa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -67,6 +67,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "calibration_measure_exclusions.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "hmrc_income_release_gate_report.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 80a2d0ea07..cab8e16dba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -317,6 +317,7 @@ UK_NATIONAL_TARGET_LOSS_CAP, UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + uk_doctrine_with_overrides, uk_national_target_loss_weights, ) from microcosm.build.uk_runtime.national_frame import ( @@ -666,6 +667,7 @@ "UK_NATIONAL_SOLVE_EPOCHS", "UK_NATIONAL_TARGET_LOSS_CAP", "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_doctrine_with_overrides", "uk_national_target_loss_weights", "UKNationalStage", "UKRowwiseDatasetResult", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py new file mode 100644 index 0000000000..4b545f7be9 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -0,0 +1,218 @@ +"""UK simulated-measure inputs for the national calibration seam.""" + +from __future__ import annotations + +import json +from importlib import resources as importlib_resources +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry + +_ENTITY_LINK = {"benunit": "person_benunit_id", "household": "person_household_id"} +_ENTITY_ID = {"person": "person_id", "benunit": "benunit_id", "household": "household_id"} + + +def compute_uk_measure_input( + frame: Any, simulation: Any, entity: str, variable: str, year: int +) -> tuple[np.ndarray, str]: + """Compute one policyengine-uk variable at the requested entity grain.""" + + definition = simulation.tax_benefit_system.variables.get(variable) + if definition is None: + raise KeyError(f"policyengine-uk has no variable {variable!r}") + native = definition.entity.key + table = frame.table(entity) + raw = simulation.calculate(variable, year) + + if native == entity: + values = _values(raw) + route = "native" + else: + native_values = _values(raw) + if native_values.dtype.kind in {"O", "U", "S", "b"}: + if entity == "person" and native in ("benunit", "household"): + native_table = frame.table(native) + lookup = pd.Series( + native_values, index=native_table[_ENTITY_ID[native]].to_numpy() + ) + keys = frame.table("person")[_ENTITY_LINK[native]].to_numpy() + values = lookup.loc[keys].to_numpy() + route = f"categorical_broadcast_{native}_to_person" + elif native == "person" and native_values.dtype.kind == "b": + person = frame.table("person") + collapsed = ( + pd.Series(native_values.astype(bool)) + .groupby(person[_ENTITY_LINK[entity]].to_numpy()) + .max() + ) + keys = table[_ENTITY_ID[entity]].to_numpy() + values = collapsed.loc[keys].to_numpy().astype(float) + route = f"bool_any_collapse_person_to_{entity}" + else: + raise KeyError( + f"no categorical mapping from {native} to {entity} " + f"for {variable!r}" + ) + else: + mapped = simulation.calculate(variable, year, map_to=entity) + values = _values(mapped) + route = f"map_to_{entity}" + + if len(values) != len(table): + raise ValueError( + f"{variable!r} produced {len(values)} values for {len(table)} " + f"{entity} rows." + ) + if values.dtype.kind not in {"O", "U", "S"}: + values = values.astype(float) + return values, route + + +class UKMeasureResolver: + """B2 measure provider backed by a policyengine-uk Microsimulation.""" + + def __init__( + self, + *, + simulation_source: Path | None, + scratch_dir: Path, + year: int, + frame: Any, + microsimulation_factory: Any | None = None, + ): + self.year = int(year) + policyengine_uk = _policyengine_uk_module() + factory = ( + microsimulation_factory + if microsimulation_factory is not None + else policyengine_uk.Microsimulation + ) + if simulation_source is None: + if frame is None: + raise ValueError("scratch-mode UKMeasureResolver requires a frame.") + scratch_dir.mkdir(parents=True, exist_ok=True) + source_path = scratch_dir / "simulation-input.h5" + write_uk_national_frame(frame, source_path) + mode = "scratch_frame_export" + else: + source_path = Path(simulation_source) + mode = "direct_h5" + if frame is None: + frame, _provenance = load_uk_national_frame(source_path) + self.frame = frame + self.simulation = factory(dataset=str(source_path)) + self._receipt = { + "mode": mode, + "source_path": str(source_path), + "policyengine_uk_version": str( + getattr(policyengine_uk, "__version__", "unknown") + ), + } + self.contract_targets = _uk_contract_targets() + + def knows(self, entity: str, variable: str) -> bool: + definition = self.simulation.tax_benefit_system.variables.get(variable) + if definition is None: + return False + native = definition.entity.key + return native == entity or entity in _ENTITY_ID + + def entity_for(self, variable: str) -> str | None: + definition = self.simulation.tax_benefit_system.variables.get(variable) + if definition is None: + return None + return str(definition.entity.key) + + def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: + return compute_uk_measure_input( + self.frame, self.simulation, entity, variable, self.year + ) + + def receipt(self) -> dict[str, str]: + return dict(self._receipt) + + +def load_uk_calibration_measure_exclusions( + path: Path | None = None, +) -> tuple[dict[str, str], ...]: + """Load and validate the reviewed UK calibration-measure exclusions.""" + + if path is None: + text = ( + importlib_resources.files("microcosm.build.uk") + .joinpath("calibration_measure_exclusions.json") + .read_text(encoding="utf-8") + ) + else: + text = Path(path).read_text(encoding="utf-8") + payload = json.loads(text) + allowed_top = {"schema_version", "exclusions"} + unknown = sorted(set(payload) - allowed_top) + if unknown: + raise ValueError(f"unknown top-level exclusion key(s): {unknown}") + if payload.get("schema_version") != 1: + raise ValueError("UK calibration measure exclusions schema_version must be 1.") + exclusions = payload.get("exclusions") + if not isinstance(exclusions, list): + raise ValueError("UK calibration measure exclusions must contain a list.") + seen: set[str] = set() + loaded: list[dict[str, str]] = [] + for entry in exclusions: + if not isinstance(entry, dict): + raise ValueError("UK calibration measure exclusion entries must be objects.") + name = str(entry.get("name", "")) + reason = str(entry.get("reason", "")) + tracking = str(entry.get("tracking", "")) + if name in seen: + raise ValueError(f"duplicate UK calibration measure exclusion {name!r}.") + if not reason.strip(): + raise ValueError(f"UK calibration measure exclusion {name!r} has empty reason.") + seen.add(name) + loaded.append({"name": name, "reason": reason, "tracking": tracking}) + return tuple(loaded) + + +def apply_uk_calibration_measure_exclusions( + registry: TargetRegistry, exclusions: tuple[dict[str, str], ...] +) -> tuple[TargetRegistry, dict[str, str]]: + """Remove reviewed excluded references from a UK target registry.""" + + reasons = {entry["name"]: entry["reason"] for entry in exclusions} + matched = {spec.name for spec in registry.specs if spec.name in reasons} + stale = sorted(set(reasons) - matched) + if stale: + raise ValueError( + "UK calibration measure exclusion matched zero registry specs: " + + ", ".join(stale) + ) + kept = [spec for spec in registry.specs if spec.name not in reasons] + receipt = {name: reasons[name] for name in sorted(matched)} + return TargetRegistry(kept, country=registry.country), receipt + + +def _values(raw: Any) -> np.ndarray: + return np.asarray(raw.values if hasattr(raw, "values") else raw) + + +def _policyengine_uk_module() -> Any: + import policyengine_uk + + return policyengine_uk + + +def _uk_contract_targets() -> dict[str, Any]: + payload = ( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + contract = json.loads(payload) + return {target["target_id"]: target for target in contract["targets"]} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index afdae1615a..15028a200c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -8,6 +8,10 @@ import numpy as np from microcosm.build.plan import Stage +from microcosm.build.target_materialization import ( + MeasureResolution, + resolve_target_measures, +) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.ledger_targets import ( UKFrameTargetAdapter, @@ -43,6 +47,7 @@ def __init__( *, period: int, doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, + measure_resolver: object | None = None, ) -> None: self.compilation = ( registry @@ -59,6 +64,7 @@ def __init__( ) self.period = period self.doctrine = doctrine + self.measure_resolver = measure_resolver self.manifest: dict[str, object] | None = None self.diagnostics: tuple[dict[str, object], ...] = () self.solve_result: CalibrationResult | None = None @@ -74,6 +80,12 @@ def __call__(self, frame: Frame) -> Frame: f"unsupported={self.compilation.unsupported!r}." ) adapter = _CalibrationFrameAdapter(frame) + original_columns = { + entity: set(table.columns) for entity, table in adapter.tables.items() + } + measure_resolution = self._resolve_measures(frame) + if measure_resolution is not None: + _inject_measure_inputs(adapter, measure_resolution.measure_inputs) materialized = materialize_uk_ledger_targets( adapter, self.registry, @@ -85,6 +97,12 @@ def __call__(self, frame: Frame) -> Frame: "UK national calibration could not materialize every activated " f"target reference: skipped={skipped}." ) + if measure_resolution is not None: + _drop_injected_measure_inputs( + adapter, + measure_resolution.measure_inputs, + original_columns, + ) prepared = adapter.prepared_frame() mass_reason = national_calibration_mass_reason( spec.family for spec in self.registry.specs @@ -139,7 +157,7 @@ def __call__(self, frame: Frame) -> Frame: new_total = float(calibration_record.new_total) before_kind = frame.weights_for("household").kind after_kind = clean_frame.weights_for("household").kind - self.manifest = { + manifest = { "activated_reference_count": declared, "resolved_reference_count": resolved, "matrix_target_count": len(result.problem.names), @@ -176,9 +194,29 @@ def __call__(self, frame: Frame) -> Frame: }, "parameters": {"doctrine": _doctrine_bounds(self.doctrine)}, } + if measure_resolution is not None: + manifest["measure_resolution"] = dict(measure_resolution.receipt) + self.manifest = manifest self.output_content_identity = uk_frame_content_identity(clean_frame) return clean_frame + def _resolve_measures(self, frame: Frame) -> MeasureResolution | None: + if self.measure_resolver is None: + return None + resolve = getattr(self.measure_resolver, "resolve", None) + if callable(resolve): + return resolve( + lambda: _CalibrationFrameAdapter(frame), + self.registry, + period=self.period, + ) + return resolve_target_measures( + lambda: _CalibrationFrameAdapter(frame), + self.registry, + self.measure_resolver, + period=self.period, + ) + def checkpoint_metadata(self) -> Mapping[str, object]: if self.manifest is None: raise RuntimeError("UK national calibration has not run.") @@ -292,6 +330,24 @@ def _doctrine_bounds(doctrine: UKNationalSolveDoctrine) -> dict[str, object]: } +def _inject_measure_inputs( + adapter: UKFrameTargetAdapter, + measure_inputs: Mapping[tuple[str, str], np.ndarray], +) -> None: + for (entity, variable), values in measure_inputs.items(): + adapter.tables[entity][variable] = values + + +def _drop_injected_measure_inputs( + adapter: UKFrameTargetAdapter, + measure_inputs: Mapping[tuple[str, str], np.ndarray], + original_columns: Mapping[str, set[str]], +) -> None: + for entity, variable in measure_inputs: + if variable not in original_columns[entity]: + adapter.tables[entity].drop(columns=[variable], inplace=True) + + class _CalibrationFrameAdapter(UKFrameTargetAdapter): """The shared UK adapter plus the prepared-frame/restore lifecycle. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py index 3735a24d1a..a17ac1928e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py @@ -1,16 +1,18 @@ """UK national calibration doctrine: reviewed solve constants for #623. -The national release path exposes no calibration knobs. Every solver option -that the first Ledger-backed national calibration currently inherits is -declared here so a future change must edit this module and its pinned tests, -forcing review before any armed run. +The national release path exposes calibration knobs only as receipted per-run +overrides on the calibration seam driver; defaults remain the reviewed +constants. Every solver option that the first Ledger-backed national +calibration currently inherits is declared here so a future default change +must edit this module and its pinned tests, forcing review before any armed +run. """ from __future__ import annotations from collections import Counter from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, fields, replace import numpy as np @@ -24,6 +26,7 @@ "UK_NATIONAL_SOLVE_EPOCHS", "UK_NATIONAL_TARGET_LOSS_CAP", "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_doctrine_with_overrides", "uk_national_target_loss_weights", "UKNationalSolveDoctrine", ] @@ -56,6 +59,9 @@ # carrying 5.5x the per-reference weight of a 50-member one). _ALLOWED_TARGET_WEIGHT_RULES = ("uniform", "family_equal") _ALLOWED_MASS_RULES = ("free",) +_OVERRIDABLE_FIELDS = frozenset( + {"epochs", "learning_rate", "target_weight_rule", "target_loss_cap"} +) @dataclass(frozen=True) @@ -150,6 +156,34 @@ def __post_init__(self) -> None: UK_NATIONAL_SOLVE_DOCTRINE = UKNationalSolveDoctrine() +def uk_doctrine_with_overrides( + **overrides: object, +) -> tuple[UKNationalSolveDoctrine, dict[str, dict[str, object]]]: + """Return the national doctrine plus a receipt of effective overrides.""" + + fields_by_name = {field.name for field in fields(UKNationalSolveDoctrine)} + unknown = sorted(set(overrides) - fields_by_name) + if unknown: + raise ValueError( + "unknown UK national doctrine field(s) " + f"{unknown}; frozen fields are {sorted(fields_by_name)}." + ) + frozen = sorted(set(overrides) - _OVERRIDABLE_FIELDS) + if frozen: + raise ValueError( + "UK national doctrine field(s) are reviewed constants, not knobs: " + f"{frozen}; overridable fields are {sorted(_OVERRIDABLE_FIELDS)}." + ) + doctrine = replace(UK_NATIONAL_SOLVE_DOCTRINE, **overrides) + receipt: dict[str, dict[str, object]] = {} + for name in sorted(_OVERRIDABLE_FIELDS): + default = getattr(UK_NATIONAL_SOLVE_DOCTRINE, name) + effective = getattr(doctrine, name) + if effective != default: + receipt[name] = {"default": default, "effective": effective} + return doctrine, receipt + + def uk_national_target_loss_weights( families: Sequence[str], *, diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index a7998c2352..390c08c7d3 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -284,6 +284,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "calibration_measure_exclusions.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", @@ -350,6 +351,7 @@ def test_uk_package_loads(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "calibration_measure_exclusions.json", "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 617eff5599..a2cd4d33e4 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e12a2cb87c0e096af0173bd51fbede4b7df5e7c118ffe07ef616bbb30640e4ea", + "0c8ea0de40107b63ab088edc5793f42db073e6bc688ecedc2d87ff290b552e56", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 22dc8e6edb..faf27327a6 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -1,6 +1,13 @@ import numpy as np - -from microcosm.build.target_materialization import materialize_target_bindings +import pandas as pd +import pytest + +from microcosm.build.target_materialization import ( + MeasureResolutionError, + assert_calibration_input_finite, + materialize_target_bindings, + resolve_target_measures, +) from microcosm.calibrate import TargetRegistry, TargetSpec @@ -41,6 +48,83 @@ def counterfactual_delta(self, binding, period): return np.array([0.0, -1.0, -2.0]) +class ResolutionAdapter: + def __init__(self, source): + self.source = source + self.tables = {entity: table.copy() for entity, table in source.items()} + + def column(self, entity, variable): + if variable not in self.tables[entity]: + raise KeyError(f"{entity}.{variable}") + return np.asarray(self.tables[entity][variable]) + + def set_column(self, entity, variable, values): + self.tables[entity][variable] = np.asarray(values) + + +class StubMeasureProvider: + contract_targets = { + "needs_input_a": { + "bindings": {"policyengine": {"value_variable": "input_a"}} + }, + "needs_input_a_and_b": { + "bindings": { + "policyengine": {"value_expression": "input_a + input_b"} + } + }, + "needs_input_a_again": { + "bindings": {"policyengine": {"value_variable": "input_a"}} + }, + } + + def __init__(self): + self.calls = [] + + def knows(self, entity, variable): + return (entity, variable) in { + ("person", "input_a"), + ("person", "input_b"), + } + + def compute(self, entity, variable): + self.calls.append((entity, variable)) + values = { + ("person", "input_a"): np.array([1.0, 2.0, 3.0]), + ("person", "input_b"): np.array([10.0, 20.0, 30.0]), + }[(entity, variable)] + return values, f"stub:{entity}.{variable}" + + def receipt(self): + return {"provider": "stub"} + + +def _resolution_source(): + return { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "base": [5.0, 6.0, 7.0], + } + ) + } + + +def _resolution_registry(*names): + specs = [] + for name in names: + specs.append( + TargetSpec( + name=name, + entity="person", + measure=f"{name}_measure", + value=1.0, + source="test", + metadata={"contract_target_id": name}, + ) + ) + return TargetRegistry(specs, country="uk") + + def test_prepared_column_path_materializes_filtered_values(): registry = TargetRegistry( [ @@ -185,6 +269,151 @@ def test_missing_materialization_inputs_are_reported_as_skips(): assert result.report()["skipped_count"] == 1 +def test_resolve_target_measures_converges_over_multiple_rounds_without_frame_mutation(): + source = _resolution_source() + before = {entity: table.copy(deep=True) for entity, table in source.items()} + provider = StubMeasureProvider() + + resolution = resolve_target_measures( + lambda: ResolutionAdapter(source), + _resolution_registry("needs_input_a_and_b"), + provider, + period=2025, + ) + + assert set(resolution.measure_inputs) == { + ("person", "input_a"), + ("person", "input_b"), + } + assert len(resolution.receipt["rounds"]) == 3 + assert resolution.receipt["attached"] == { + "person.input_a": "stub:person.input_a", + "person.input_b": "stub:person.input_b", + } + for entity, table in source.items(): + pd.testing.assert_frame_equal(table, before[entity]) + + +def test_resolve_target_measures_computes_shared_missing_key_once_per_round(): + source = _resolution_source() + provider = StubMeasureProvider() + + resolve_target_measures( + lambda: ResolutionAdapter(source), + _resolution_registry("needs_input_a", "needs_input_a_again"), + provider, + period=2025, + ) + + assert provider.calls == [("person", "input_a")] + + +def test_resolve_target_measures_raises_when_provided_key_still_fails(): + class IgnoringAdapter(ResolutionAdapter): + def column(self, entity, variable): + if variable == "input_a": + raise KeyError(f"{entity}.{variable}") + return super().column(entity, variable) + + provider = StubMeasureProvider() + + with pytest.raises( + MeasureResolutionError, match="remained unmaterializable" + ) as error: + resolve_target_measures( + lambda: IgnoringAdapter(_resolution_source()), + _resolution_registry("needs_input_a"), + provider, + period=2025, + ) + + assert error.value.receipt["attached"] == { + "person.input_a": "stub:person.input_a" + } + assert error.value.receipt["skips"][-1]["name"] == "needs_input_a" + + +def test_resolve_target_measures_raises_on_no_progress_and_max_rounds(): + provider = StubMeasureProvider() + + with pytest.raises(MeasureResolutionError, match="does not know"): + resolve_target_measures( + lambda: ResolutionAdapter(_resolution_source()), + TargetRegistry( + [ + TargetSpec( + name="unknown", + entity="person", + measure="unknown_measure", + value=1.0, + source="test", + metadata={"contract_target_id": "unknown"}, + ) + ], + country="uk", + ), + provider, + period=2025, + contract_targets={ + "unknown": { + "bindings": { + "policyengine": {"value_variable": "not_provider_known"} + } + } + }, + ) + + with pytest.raises(MeasureResolutionError, match="did not converge"): + resolve_target_measures( + lambda: ResolutionAdapter(_resolution_source()), + _resolution_registry("needs_input_a"), + provider, + period=2025, + max_rounds=0, + ) + + +def test_assert_calibration_input_finite_reports_every_nan_float_column(): + class FrameLike: + entities = ("person", "household") + + def __init__(self): + self._tables = { + "person": pd.DataFrame( + { + "ok": [1.0, 2.0], + "bad": [np.nan, 3.0], + "also_bad": [np.nan, np.nan], + "label": ["x", None], + } + ), + "household": pd.DataFrame({"bad": [np.nan]}), + } + + def table(self, entity): + return self._tables[entity] + + with pytest.raises(ValueError) as error: + assert_calibration_input_finite(FrameLike()) + + message = str(error.value) + assert "person.bad: 1" in message + assert "person.also_bad: 2" in message + assert "household.bad: 1" in message + assert "person.label" not in message + + +def test_assert_calibration_input_finite_noops_on_clean_frame(): + class CleanFrame: + entities = ("person",) + + def table(self, entity): + assert entity == "person" + return pd.DataFrame({"value": [1.0], "label": ["x"]}) + + assert_calibration_input_finite(CleanFrame()) + + def test_count_aliases_and_in_predicates_materialize_prepared_columns(): registry = TargetRegistry( [ diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py new file mode 100644 index 0000000000..a5ebee21a3 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import measure_simulation +from microcosm.build.uk_runtime.measure_simulation import ( + UKMeasureResolver, + apply_uk_calibration_measure_exclusions, + compute_uk_measure_input, + load_uk_calibration_measure_exclusions, +) +from microcosm.calibrate import TargetRegistry, TargetSpec + + +class FrameStub: + def __init__(self): + self._tables = { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_benunit_id": [10, 10, 20], + "person_household_id": [100, 100, 200], + } + ), + "benunit": pd.DataFrame({"benunit_id": [10, 20]}), + "household": pd.DataFrame({"household_id": [100, 200]}), + } + + def table(self, entity): + return self._tables[entity] + + +class SimulationStub: + def __init__(self, values): + self.values = values + variables = { + name: SimpleNamespace(entity=SimpleNamespace(key=entity)) + for name, (entity, _values) in values.items() + } + self.tax_benefit_system = SimpleNamespace(variables=variables) + self.calls = [] + + def calculate(self, variable, year, map_to=None): + self.calls.append((variable, year, map_to)) + key = (variable, map_to) + if key in self.values: + return self.values[key][1] + return self.values[variable][1] + + +def test_compute_uk_measure_input_native_route(): + sim = SimulationStub({"income_tax": ("person", np.array([1.0, 2.0, 3.0]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "person", "income_tax", 2025 + ) + + assert route == "native" + assert values.tolist() == [1.0, 2.0, 3.0] + + +def test_compute_uk_measure_input_categorical_broadcast_group_to_person(): + sim = SimulationStub({"family_type": ("benunit", np.array(["couple", "single"]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "person", "family_type", 2025 + ) + + assert route == "categorical_broadcast_benunit_to_person" + assert values.tolist() == ["couple", "couple", "single"] + + +def test_compute_uk_measure_input_bool_any_collapse_person_to_group(): + sim = SimulationStub({"is_disabled": ("person", np.array([False, True, False]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "household", "is_disabled", 2025 + ) + + assert route == "bool_any_collapse_person_to_household" + assert values.tolist() == [1.0, 0.0] + + +def test_compute_uk_measure_input_numeric_map_to_entity(): + sim = SimulationStub( + { + "benefit": ("person", np.array([1.0, 2.0, 3.0])), + ("benefit", "household"): ("household", np.array([3.0, 3.0])), + } + ) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "household", "benefit", 2025 + ) + + assert route == "map_to_household" + assert values.tolist() == [3.0, 3.0] + assert sim.calls[-1] == ("benefit", 2025, "household") + + +def test_compute_uk_measure_input_refuses_unknown_and_length_mismatch(): + with pytest.raises(KeyError, match="no variable"): + compute_uk_measure_input(FrameStub(), SimulationStub({}), "person", "missing", 2025) + + sim = SimulationStub({"income_tax": ("person", np.array([1.0]))}) + with pytest.raises(ValueError, match="produced 1 values"): + compute_uk_measure_input(FrameStub(), sim, "person", "income_tax", 2025) + + +def _write_exclusions(path: Path, payload: dict) -> Path: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_exclusion_loader_refusals(tmp_path: Path): + base = { + "schema_version": 1, + "exclusions": [ + {"name": "a", "reason": "reviewed reason", "tracking": "microcosm#623"} + ], + } + + with pytest.raises(ValueError, match="schema_version"): + load_uk_calibration_measure_exclusions( + _write_exclusions(tmp_path / "bad-version.json", {**base, "schema_version": 2}) + ) + with pytest.raises(ValueError, match="unknown top-level"): + load_uk_calibration_measure_exclusions( + _write_exclusions(tmp_path / "unknown.json", {**base, "extra": True}) + ) + with pytest.raises(ValueError, match="empty reason"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "empty.json", + {**base, "exclusions": [{"name": "a", "reason": "", "tracking": "x"}]}, + ) + ) + with pytest.raises(ValueError, match="duplicate"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "duplicate.json", + { + **base, + "exclusions": [ + {"name": "a", "reason": "one", "tracking": "x"}, + {"name": "a", "reason": "two", "tracking": "x"}, + ], + }, + ) + ) + + +def test_exclusion_applier_returns_pruned_registry_and_receipt(): + registry = TargetRegistry( + [ + TargetSpec(name="drop", entity="person", measure="drop", value=1.0, source="test"), + TargetSpec(name="keep", entity="person", measure="keep", value=1.0, source="test"), + ], + country="uk", + ) + + pruned, receipt = apply_uk_calibration_measure_exclusions( + registry, + ({"name": "drop", "reason": "reviewed", "tracking": "microcosm#623"},), + ) + + assert [spec.name for spec in pruned.specs] == ["keep"] + assert receipt == {"drop": "reviewed"} + with pytest.raises(ValueError, match="matched zero"): + apply_uk_calibration_measure_exclusions( + registry, + ({"name": "stale", "reason": "reviewed", "tracking": "microcosm#623"},), + ) + + +def test_packaged_exclusions_load(): + exclusions = load_uk_calibration_measure_exclusions() + assert {entry["name"] for entry in exclusions} == { + "hmrc.salary_sacrifice.it_relief_basic_rate", + "hmrc.salary_sacrifice.it_relief_higher_rate", + "hmrc.salary_sacrifice.it_relief_additional_rate", + "hmrc.salary_sacrifice.nics_relief_employee", + "hmrc.salary_sacrifice.nics_relief_employer", + } + + +def test_measure_resolver_direct_and_scratch_receipts(monkeypatch, tmp_path: Path): + created = [] + + class FakeMicrosimulation: + def __init__(self, *, dataset): + created.append(dataset) + self.tax_benefit_system = SimpleNamespace(variables={}) + + fake_policyengine_uk = SimpleNamespace( + __version__="9.9.9", Microsimulation=FakeMicrosimulation + ) + monkeypatch.setitem(sys.modules, "policyengine_uk", fake_policyengine_uk) + writes = [] + monkeypatch.setattr( + measure_simulation, + "write_uk_national_frame", + lambda frame, path: writes.append((frame, path)) or path, + ) + + frame = FrameStub() + direct = UKMeasureResolver( + simulation_source=tmp_path / "input.h5", + scratch_dir=tmp_path, + year=2025, + frame=frame, + ) + scratch = UKMeasureResolver( + simulation_source=None, + scratch_dir=tmp_path, + year=2025, + frame=frame, + ) + + assert direct.receipt()["mode"] == "direct_h5" + assert direct.receipt()["policyengine_uk_version"] == "9.9.9" + assert scratch.receipt()["mode"] == "scratch_frame_export" + assert writes == [(frame, tmp_path / "simulation-input.h5")] + assert created == [str(tmp_path / "input.h5"), str(tmp_path / "simulation-input.h5")] diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 6f9bca424c..787b093261 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -21,6 +21,7 @@ UK_NATIONAL_TARGET_LOSS_CAP, UK_NATIONAL_TARGET_WEIGHT_RULE, UKNationalSolveDoctrine, + uk_doctrine_with_overrides, uk_national_target_loss_weights, ) from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation @@ -116,6 +117,48 @@ def _frame() -> Frame: ) +def _frame_without_uc_column() -> Frame: + frame = _frame() + return Frame( + { + "person": frame.table("person"), + "benunit": frame.table("benunit").drop(columns=["universal_credit"]), + "household": frame.table("household"), + }, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +class StubMeasureResolver: + contract_targets = { + "dwp.uc.households": { + "bindings": { + "policyengine": { + "from_entity": "benunit", + "value_variable": "universal_credit", + } + } + } + } + + def __init__(self): + self.calls = [] + + def knows(self, entity, variable): + return (entity, variable) == ("benunit", "universal_credit") + + def compute(self, entity, variable): + self.calls.append((entity, variable)) + return np.array([1.0, 1.0, 0.0, 0.0]), "stub_uc" + + def receipt(self): + return {"provider": "stub_uc"} + + def _nested_frame() -> Frame: return Frame( { @@ -267,6 +310,42 @@ def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() validate_uk_national_frame(result) +def test_stage_measure_resolver_injects_columns_then_restores_pristine_output() -> None: + frame = _frame_without_uc_column() + resolver = StubMeasureResolver() + original_columns = {entity: set(frame.table(entity).columns) for entity in frame.entities} + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + measure_resolver=resolver, + ) + + result = stage(frame) + + assert resolver.calls == [("benunit", "universal_credit")] + assert stage.manifest["measure_resolution"]["provider"] == {"provider": "stub_uc"} + assert stage.manifest["measure_resolution"]["attached"] == { + "benunit.universal_credit": "stub_uc" + } + for entity in frame.entities: + assert set(result.table(entity).columns) == original_columns[entity] + assert "universal_credit" not in result.table("benunit") + validate_uk_national_frame(result) + + +def test_stage_manifest_omits_measure_resolution_without_resolver() -> None: + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + ) + + stage(_frame()) + + assert "measure_resolution" not in stage.manifest + + def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: stage = UKNationalCalibrationStage( UKLedgerTargetCompilation( @@ -559,6 +638,47 @@ def test_national_doctrine_constants_are_the_declared_contract() -> None: assert UKNationalSolveDoctrine(target_weight_rule="family_equal") +def test_uk_doctrine_with_overrides_receipts_effective_diffs_only() -> None: + doctrine, receipt = uk_doctrine_with_overrides() + assert doctrine == UK_NATIONAL_SOLVE_DOCTRINE + assert receipt == {} + + doctrine, receipt = uk_doctrine_with_overrides( + epochs=UK_NATIONAL_SOLVE_EPOCHS, + target_weight_rule=UK_NATIONAL_TARGET_WEIGHT_RULE, + ) + assert doctrine == UK_NATIONAL_SOLVE_DOCTRINE + assert receipt == {} + + doctrine, receipt = uk_doctrine_with_overrides( + epochs=128, + learning_rate=0.01, + target_weight_rule="family_equal", + target_loss_cap=5.0, + ) + assert doctrine.epochs == 128 + assert doctrine.learning_rate == 0.01 + assert doctrine.target_weight_rule == "family_equal" + assert doctrine.target_loss_cap == 5.0 + assert receipt == { + "epochs": {"default": 256, "effective": 128}, + "learning_rate": {"default": 0.02, "effective": 0.01}, + "target_loss_cap": {"default": 10.0, "effective": 5.0}, + "target_weight_rule": {"default": "uniform", "effective": "family_equal"}, + } + + +def test_uk_doctrine_with_overrides_refuses_invalid_or_frozen_fields() -> None: + with pytest.raises(ValueError, match="target_weight_rule"): + uk_doctrine_with_overrides(target_weight_rule="per_target") + with pytest.raises(ValueError, match="epochs"): + uk_doctrine_with_overrides(epochs=0) + with pytest.raises(ValueError, match="reviewed constants"): + uk_doctrine_with_overrides(seed=1) + with pytest.raises(ValueError, match="unknown"): + uk_doctrine_with_overrides(not_a_field=1) + + def test_family_equal_gives_each_family_one_equal_share() -> None: weights = uk_national_target_loss_weights( ["hmrc"] * 3 + ["obr"], rule="family_equal" From e54e546bf0e2865ddaad1d87843851a34d37ac82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:37:35 +0200 Subject: [PATCH 19/26] Add the calibration seam and its driver: calibrate whichever H5 you are given MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The June national driver rederives SPI income, redraws capital gains and replays support tables onto its input — work the WS-E spine now ships already derived, and which its own zero-weight precondition refuses to perform on a spine at all. This seam does none of it: it verifies the input pin, loads, fences NaNs, solves weights under doctrine, and writes the staged H5, US-format diagnostics, build record and signed gate report. Its defining invariant has a test: every data column of the staged artifact is identical to the input, and only the household weights, their kind and one appended mass record differ. Nothing here imports the legacy build module, so the June path can retire on its own schedule without this lane moving. The gate posture is calibration-scoped. Spine-construction gates cannot pass a build that constructs no spine, and pretending otherwise would be the invented pass this repo refuses; each out-of-scope entry is listed in the signed report with its reason, and a test asserts scope and exclusions together cover every declared gate, so a new gate cannot land unclassified. Aggregate-admin evidence follows the anchors' own convention — per-household means for the NEED anchors, a national total for NHS — and refuses when the frame cannot supply an anchor rather than reporting a zero. The build record states shippable: false with its reason. A publishable artifact needs the full battery, which is release-cut work. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/calibration_run.py | 502 ++++++++++++++++++ .../microcosm/build/uk_runtime/diagnostics.py | 29 + .../tests/test_uk_calibration_run.py | 293 ++++++++++ .../tests/test_uk_calibration_seam_driver.py | 165 ++++++ tools/calibrate_uk_national_dataset.py | 252 +++++++++ 5 files changed, 1241 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py create mode 100644 packages/microcosm-build/tests/test_uk_calibration_run.py create mode 100644 packages/microcosm-build/tests/test_uk_calibration_seam_driver.py create mode 100644 tools/calibrate_uk_national_dataset.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py new file mode 100644 index 0000000000..5ac51849fb --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -0,0 +1,502 @@ +"""Production UK national calibration seam orchestration.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import os +import time +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np + +from microcosm.build.country_spec import GatesManifest, load_country_spec +from microcosm.build.gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryRun, + gate_signing_key_env, +) +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.logbook_adoption import ( + AttemptState, + append_phase, + git_code_pin, + local_artifact_reference, + record_terminal_attempt, + resolve_predecessor, + role_pins_digest, +) +from microcosm.build.target_materialization import assert_calibration_input_finite +from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.diagnostics import ( + uk_target_geography_levels, + write_uk_calibration_diagnostics, +) +from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry +from microcosm.frame import Frame + +_REPOSITORY = Path(__file__).resolve().parents[6] +_PIPELINE = "uk-national-calibration" + + +@dataclass(frozen=True) +class UKCalibrationRunPaths: + input_h5: Path + staging_h5: Path + diagnostics_json: Path + build_record_json: Path + terminal_gate_json: Path + + +@dataclass(frozen=True) +class UKCalibrationRunResult: + frame: Frame + diagnostics_sha256: str + staging_sha256: str + build_record_sha256: str + terminal_gate_sha256: str + logbook_spool: Path + gate_report: Mapping[str, object] + build_record: Mapping[str, object] + + +UK_CALIBRATION_GATE_SCOPE = ( + "uk_target_fit", + "uk_weight_ratio", + "uk_weight_ess", + "uk_zero_weight_strata", + "uk_aggregate_admin", + "uk_calibration_reference_coverage", +) + + +def _scope_exclusions() -> dict[str, str]: + full = {entry.id for entry in load_country_spec("uk").gates.gates} + excluded = full - set(UK_CALIBRATION_GATE_SCOPE) + rationales: dict[str, str] = {} + for gate_id in sorted(excluded): + if "parity" in gate_id or gate_id in {"uk_export_surface", "uk_target_surface"}: + reason = "swap-acceptance evidence; produced by the swap lane, not the calibration seam." + elif gate_id in { + "uk_release_input_coverage_manifest_current", + "uk_release_family_build_stages", + "uk_release_input_coverage", + "uk_nonnegative_columns", + "uk_support", + "uk_take_up_signal", + "uk_brma_enum_domain", + "uk_degenerate_release_surface", + "uk_input_mass_parity", + "uk_qrf_tail_concentration", + "uk_weights_audit", + }: + reason = "spine-construction gate; owned by the spine build's own battery." + else: + reason = "outside the calibration seam's reviewed gate scope." + rationales[gate_id] = reason + if set(UK_CALIBRATION_GATE_SCOPE) | set(rationales) != full: + raise RuntimeError("UK calibration gate scope does not classify every gate id.") + return rationales + + +UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS = _scope_exclusions() + + +def run_uk_calibration( + *, + paths: UKCalibrationRunPaths, + input_sha256: str, + ledger_artifact: Any, + register_registry: TargetRegistry, + calibration_year: int, + exclusion_receipt: Mapping[str, str], + doctrine: Any, + doctrine_overrides: Mapping[str, Mapping[str, object]], + measure_resolver: object | None, + source_pins: Mapping[str, Mapping[str, object]], + run_config_extra: Mapping[str, object], + release_candidate: bool, + release_id: str, + logbook_prev_row_digest: str | None = None, +) -> UKCalibrationRunResult: + """Run the UK national calibration seam and write its sidecars.""" + + started_at = time.perf_counter() + started_ts = datetime.now(UTC) + code_pin = git_code_pin(_REPOSITORY) + run_config = { + "pipeline": _PIPELINE, + "register_sha256": register_registry.version, + "calibration_year": int(calibration_year), + "doctrine": _doctrine_payload(doctrine), + "doctrine_overrides": dict(doctrine_overrides), + **dict(run_config_extra), + } + state = AttemptState( + build_id=f"uk-national-calibration-{release_id}", + identity_digest=hashlib.sha256(canonical_json_bytes(run_config)).hexdigest(), + input_pins_digest=role_pins_digest(source_pins), + phases_reached=["attempt_started"], + gate_verdicts={}, + ) + + measured_input_sha = _sha256_file(paths.input_h5) + if measured_input_sha != input_sha256: + raise ValueError( + "input H5 sha mismatch: " + f"measured {measured_input_sha}, pinned {input_sha256}" + ) + append_phase(state, "input_sha_verified") + frame, _provenance = load_uk_national_frame(paths.input_h5) + append_phase(state, "input_loaded") + assert_calibration_input_finite(frame) + append_phase(state, "input_finite") + + stage = UKNationalCalibrationStage( + register_registry, + # The declared calibration year the register was compiled at — the + # stage validates it; there is deliberately no fallback to the input + # frame's base-year time_period or any ambient default. + period=calibration_year, + doctrine=doctrine, + measure_resolver=measure_resolver, + ) + calibrated = stage(frame) + append_phase(state, "national_calibration_solved") + + build_block = { + "build_id": state.build_id, + "code_pin": code_pin, + "source_pins": dict(source_pins), + "input_posture": { + "tier": "staging_candidate", + "sha256": measured_input_sha, + "size_bytes": paths.input_h5.stat().st_size, + }, + "doctrine": _doctrine_payload(doctrine), + "doctrine_overrides": dict(doctrine_overrides), + "measure_exclusions": dict(exclusion_receipt), + "measure_resolution": ( + stage.manifest.get("measure_resolution") + if isinstance(stage.manifest, Mapping) + else None + ), + "register": _register_census(register_registry, exclusion_receipt), + "score_vs_enhanced_frs": None, + } + write_uk_calibration_diagnostics( + stage.solve_result, + paths.diagnostics_json, + calibrated, + target_geography_levels=uk_target_geography_levels(stage.registry), + target_registry=stage.registry, + build=build_block, + ) + diagnostics_sha = _sha256_file(paths.diagnostics_json) + append_phase(state, "diagnostics_written") + + gate_report = _run_calibration_gate_battery( + calibrated, + stage, + paths.terminal_gate_json, + release_candidate=release_candidate, + release_id=release_id, + diagnostics_sha256=diagnostics_sha, + ) + append_phase(state, "calibration_gates_evaluated") + for gate_id, payload in gate_report["gates"].items(): + state.gate_verdicts[gate_id] = { + "verdict": payload["status"], + "receipt": f"local://{paths.terminal_gate_json.name}#/gates/{gate_id}", + } + + write_uk_national_frame(calibrated, paths.staging_h5) + staging_sha = _sha256_file(paths.staging_h5) + append_phase(state, "staging_h5_written") + + record = { + "schema_version": 1, + "pipeline": _PIPELINE, + "build_id": state.build_id, + "run_config": run_config, + "source_pins": dict(source_pins), + "role_pins_digest": role_pins_digest(source_pins), + "input_posture": build_block["input_posture"], + "register": build_block["register"], + "calibration": stage.manifest, + "gate_summary": _gate_summary(gate_report), + "shippable": False, + "shippable_reason": ( + "calibration-scoped battery; release certification is the " + "release-cut producer's job" + ), + "artifacts": { + "staging_h5": {"path": str(paths.staging_h5), "sha256": staging_sha}, + "diagnostics_json": { + "path": str(paths.diagnostics_json), + "sha256": diagnostics_sha, + }, + "terminal_gate_json": { + "path": str(paths.terminal_gate_json), + "sha256": _sha256_file(paths.terminal_gate_json), + }, + }, + } + _write_json(paths.build_record_json, record) + build_record_sha = _sha256_file(paths.build_record_json) + append_phase(state, "build_record_written") + state.artifact_location = local_artifact_reference( + paths.staging_h5, repository_hint=_REPOSITORY + ) + spool = record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung="f100", + seed=getattr(doctrine, "seed", None), + code_pin=code_pin, + disposition="iterating", + predecessor=resolve_predecessor(logbook_prev_row_digest), + spool_dir=paths.staging_h5.parent / "logbook-spool", + ) + return UKCalibrationRunResult( + frame=calibrated, + diagnostics_sha256=diagnostics_sha, + staging_sha256=staging_sha, + build_record_sha256=build_record_sha, + terminal_gate_sha256=_sha256_file(paths.terminal_gate_json), + logbook_spool=spool, + gate_report=gate_report, + build_record=record, + ) + + +def _run_calibration_gate_battery( + frame: Frame, + stage: UKNationalCalibrationStage, + path: Path, + *, + release_candidate: bool, + release_id: str, + diagnostics_sha256: str, +) -> dict[str, object]: + manifest = _calibration_gate_manifest() + admin_totals, admin_receipt = _aggregate_admin_totals(frame, manifest) + artifacts = { + "national_calibration": stage.manifest, + "parity_evidence": SimpleNamespace( + target_relative_errors={ + str(row["name"]): float(row["relative_error"]) + for row in stage.diagnostics + } + ), + "aggregate_admin": admin_totals, + } + battery = GateBatteryRun( + manifest, + release_id=release_id, + report_path=path, + release_candidate=release_candidate, + registry=UK_GATE_REGISTRY, + release_evidence={"calibration_diagnostics_sha256": diagnostics_sha256}, + ) + battery.run_phase("terminal", EvidenceContext(frame=frame, artifacts=artifacts)) + battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) + payload = battery.report_payload() + payload["posture"] = "calibration_seam" + payload["scope_exclusions"] = dict(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + payload["aggregate_admin_measurement"] = admin_receipt + _resign_gate_report(payload) + _write_json(path, payload) + return payload + + +def _calibration_gate_manifest() -> GatesManifest: + source = load_country_spec("uk").gates + entries = tuple( + entry for entry in source.gates if entry.id in UK_CALIBRATION_GATE_SCOPE + ) + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; calibration_seam_scope", + phases=("terminal",), + gates=entries, + ) + + +def _aggregate_admin_totals( + frame: Frame, manifest: GatesManifest +) -> tuple[dict[str, float], list[dict[str, object]]]: + """Measure every declared admin anchor, fail-loud on absent evidence. + + The anchor value's magnitude tells its statistic — NEED per-household + means are hundreds of pounds, the NHS anchor is a national total — the + same reviewed convention the first armed-run receipts used. Non-household + entities carry their household's weight through the person linkage. + """ + + anchors = [] + for entry in manifest.gates: + if entry.id == "uk_aggregate_admin": + anchors = list(entry.parameters.get("anchors", ())) + break + household_weights = np.asarray( + frame.weights_for("household").values, dtype=float + ) + totals: dict[str, float] = {} + receipt: list[dict[str, object]] = [] + for anchor in anchors: + entity = str(anchor.get("entity", "household")) + name = str(anchor.get("name", anchor.get("measure"))) + measure = str(anchor.get("measure", anchor.get("name"))) + table = frame.table(entity) + if measure not in table: + raise ValueError( + f"aggregate_admin anchor {name!r} needs {entity}.{measure}, " + "which the calibrated frame does not carry; refusing to " + "fabricate a measured value." + ) + if entity == "household": + weights = household_weights + elif entity == "person": + person = frame.table("person") + lookup = dict( + zip( + frame.table("household")["household_id"].to_numpy(), + household_weights, + strict=True, + ) + ) + weights = np.asarray( + [lookup[key] for key in person["person_household_id"].to_numpy()], + dtype=float, + ) + else: + raise ValueError( + f"aggregate_admin anchor {name!r} declares entity {entity!r}; " + "the calibration seam measures household and person anchors " + "only." + ) + values = table[measure].to_numpy(dtype=float) + total = float(np.dot(values, weights)) + carriers = values != 0 + carrier_weight = float(weights[carriers].sum()) + mean_carriers = ( + float(np.dot(values[carriers], weights[carriers]) / carrier_weight) + if carrier_weight + else float("nan") + ) + declared = float(anchor["value"]) + measured = mean_carriers if abs(declared) < 1e6 else total + totals[name] = measured + receipt.append( + { + "anchor": name, + "entity": entity, + "measure": measure, + "measured": measured, + "weighted_total": total, + "weighted_mean_carriers": mean_carriers, + "statistic_convention": "assessed_by_anchor_magnitude", + } + ) + return totals, receipt + + +def _register_census( + registry: TargetRegistry, exclusions: Mapping[str, str] +) -> dict[str, object]: + return { + "country": registry.country, + "version": registry.version, + "compiled_count": len(registry.specs) + len(exclusions), + "excluded_count": len(exclusions), + "calibrated_count": len(registry.specs), + } + + +def _doctrine_payload(doctrine: Any) -> dict[str, object]: + return { + key: getattr(doctrine, key) + for key in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "seed", + "target_loss_cap", + "scale_rule", + "target_weight_rule", + "mass_rule", + "l0_lambda", + ) + if hasattr(doctrine, key) + } + + +def _gate_summary(report: Mapping[str, object]) -> dict[str, object]: + gates = report.get("gates", {}) + if not isinstance(gates, Mapping): + return {} + return { + gate_id: payload.get("status") + for gate_id, payload in gates.items() + if isinstance(payload, Mapping) + } + + +def _resign_gate_report(payload: dict[str, object]) -> None: + attestation = payload.get("attestation") + if not isinstance(attestation, dict): + raise RuntimeError("gate report has no attestation block.") + env = gate_signing_key_env("uk") + encoded = os.environ.get(env) + if not encoded: + raise RuntimeError( + f"{env} must be set; unsigned full-scale calibration runs refuse to stage." + ) + try: + key = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + raise RuntimeError(f"{env} must be valid base64.") from exc + if len(key) != 32: + raise RuntimeError(f"{env} must decode to exactly 32 bytes.") + attestation.pop("signing_error", None) + attestation["signing_key_sha256"] = hashlib.sha256(key).hexdigest() + attestation["signature"] = None + attestation["signature"] = hmac.new( + key, canonical_json_bytes(payload), hashlib.sha256 + ).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _write_json(path: Path, payload: Mapping[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index 732a523356..edd620a448 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -19,6 +19,7 @@ import math import uuid from collections.abc import Mapping, Sequence +from importlib import resources as importlib_resources from pathlib import Path from typing import Any @@ -37,6 +38,7 @@ "UK_DIAGNOSTICS_SCHEMA_VERSION", "UK_TARGET_GEOGRAPHY_LEVELS", "uk_calibration_diagnostics_payload", + "uk_target_geography_levels", "uk_weight_summary", "uk_zero_weight_strata", "write_uk_calibration_diagnostics", @@ -259,6 +261,33 @@ def _normalize_geography_level(value: object) -> str: return level +def uk_target_geography_levels(registry: TargetRegistry) -> dict[str, str]: + """Map declared UK target row names to their contract geography level.""" + + payload = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + targets = {row["target_id"]: row for row in payload["targets"]} + levels: dict[str, str] = {} + for spec in registry.specs: + target_id = spec.metadata.get("contract_target_id") + target = targets.get(str(target_id)) + if target is None: + raise ValueError( + f"UK calibration target {spec.name!r} references unknown " + f"contract target {target_id!r}." + ) + geography_levels = tuple(target.get("geography_levels") or ()) + if not geography_levels: + raise ValueError( + f"UK calibration target {spec.name!r} has no geography level." + ) + levels[spec.to_target().row_name] = str(geography_levels[0]) + return levels + + def _geography_mapping( target_names: Sequence[str], values: Mapping[str, object], diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py new file mode 100644 index 0000000000..5997b049fb --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.uk_runtime import calibration_run +from microcosm.build.uk_runtime.calibration_run import ( + UK_CALIBRATION_GATE_SCOPE, + UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, + UKCalibrationRunPaths, + run_uk_calibration, +) +from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind + +SIGNING_KEY = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" + + +@pytest.fixture(autouse=True) +def _signing_key(monkeypatch): + monkeypatch.setenv("MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY", SIGNING_KEY) + + +def _frame(): + ids = np.arange(4, dtype="int64") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "nhs_spending": [50.0, 50.0, 50.0, 50.0], + } + ), + benunit=pd.DataFrame( + {"benunit_id": ids, "universal_credit": [1.0, 1.0, 0.0, 0.0]} + ), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": [10.0, 10.0, 10.0, 10.0], + "household_is_spi_synthetic": [False, False, False, False], + "household_is_capital_gains_clone": [False, False, False, False], + "electricity_consumption": [1.0, 1.0, 1.0, 1.0], + "gas_consumption": [1.0, 1.0, 1.0, 1.0], + } + ), + time_period="2023", + weight_kind=WeightKind.DESIGN, + ) + + +def _registry(): + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=20.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _admin_anchor_values(): + values = {} + for entry in load_country_spec("uk").gates.gates: + if entry.id == "uk_aggregate_admin": + for anchor in entry.parameters["anchors"]: + values[str(anchor["name"])] = float(anchor["value"]) + return values + + +def test_gate_scope_classifies_every_uk_gate(): + all_ids = {entry.id for entry in load_country_spec("uk").gates.gates} + assert set(UK_CALIBRATION_GATE_SCOPE) | set(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) == all_ids + assert set(UK_CALIBRATION_GATE_SCOPE).isdisjoint(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + assert all(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS.values()) + + +def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): + source = Path(calibration_run.__file__).read_text(encoding="utf-8") + assert "microcosm.build.uk_runtime.national_build" not in source + assert "from microcosm.build.uk_runtime.national_build" not in source + + +def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + source_pins = { + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size}, + "ledger_facts": {"sha256": "a" * 64, "size_bytes": 1}, + } + + result = run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins=source_pins, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="test-run", + ) + + assert paths.staging_h5.exists() + assert paths.diagnostics_json.exists() + assert paths.build_record_json.exists() + assert paths.terminal_gate_json.exists() + assert result.build_record["artifacts"]["staging_h5"]["sha256"] == _sha(paths.staging_h5) + assert result.build_record["artifacts"]["diagnostics_json"]["sha256"] == _sha(paths.diagnostics_json) + assert result.build_record["artifacts"]["terminal_gate_json"]["sha256"] == _sha(paths.terminal_gate_json) + staged, _ = load_uk_national_frame(paths.staging_h5) + assert staged.weights_for("household").kind is WeightKind.CALIBRATED + report = json.loads(paths.terminal_gate_json.read_text()) + assert report["posture"] == "calibration_seam" + assert set(report["scope_exclusions"]) == set(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + attestation = report["attestation"] + signature = attestation["signature"] + attestation["signature"] = None + key = b"0123456789abcdef0123456789abcdef" + assert hmac.new(key, canonical_json_bytes(report), hashlib.sha256).hexdigest() == signature + assert result.logbook_spool.exists() + + +def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + with pytest.raises(ValueError, match="sha mismatch"): + run_uk_calibration( + paths=paths, + input_sha256="0" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="bad-sha", + ) + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + + +def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): + """The seam's defining invariant: weights move, data never does. + + Every data column of every entity table in the staged H5 must be + byte-identical to the input; only the household weights, the weight + kind, and exactly one appended mass record may differ. + """ + + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + # Target 30 against an initial weighted UC count of 20, so the solve + # genuinely has to move weights while the data stays untouched. + pulling_registry = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=30.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=pulling_registry, + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=50), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={}, + release_candidate=False, + release_id="invariant-run", + ) + + source, _ = load_uk_national_frame(input_h5) + staged, _ = load_uk_national_frame(paths.staging_h5) + for entity in ("person", "benunit", "household"): + left = source.table(entity) + right = staged.table(entity) + assert list(left.columns) == list(right.columns), entity + for column in left.columns: + pd.testing.assert_series_equal(left[column], right[column]) + assert staged.weights_for("household").kind is WeightKind.CALIBRATED + assert not np.allclose( + staged.weights_for("household").values, + source.weights_for("household").values, + ) + assert len(staged.mass_log) == len(source.mass_log) + 1 + + +def test_aggregate_admin_measurement_convention_and_refusals(): + frame = _frame() + manifest = calibration_run._calibration_gate_manifest() + + totals, receipt = calibration_run._aggregate_admin_totals(frame, manifest) + + # Small anchors (NEED means) measure as the weighted mean over carriers; + # the NHS total measures as the person total under mapped household + # weights: 4 persons x 50.0 x weight 10.0. + assert totals["need_electricity_mean_spending"] == pytest.approx(1.0) + assert totals["need_gas_mean_spending"] == pytest.approx(1.0) + assert totals["nhs_spending_total"] == pytest.approx(2000.0) + by_anchor = {row["anchor"]: row for row in receipt} + assert by_anchor["nhs_spending_total"]["entity"] == "person" + assert ( + by_anchor["need_electricity_mean_spending"]["statistic_convention"] + == "assessed_by_anchor_magnitude" + ) + + stripped = _frame() + stripped.table("household").drop(columns=["electricity_consumption"], inplace=True) + with pytest.raises(ValueError, match="household.electricity_consumption"): + calibration_run._aggregate_admin_totals(stripped, manifest) diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py new file mode 100644 index 0000000000..6c9b254619 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation +from microcosm.calibrate import TargetRegistry, TargetSpec + + +def _load_driver_module(): + root = Path(__file__).resolve().parents[3] + path = root / "tools" / "calibrate_uk_national_dataset.py" + spec = importlib.util.spec_from_file_location("calibrate_uk_national_dataset", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _registry(): + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=1.0, + source="test", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + +def _args(tmp_path: Path) -> list[str]: + paths = { + "input": tmp_path / "input.h5", + "ledger": tmp_path / "ledger", + "staging": tmp_path / "staging.h5", + "diagnostics": tmp_path / "diagnostics.json", + "record": tmp_path / "record.json", + } + paths["ledger"].mkdir(exist_ok=True) + for key, path in paths.items(): + if key != "ledger": + path.write_bytes(key.encode()) + return [ + "--input-h5", + str(paths["input"]), + "--input-sha256", + "a" * 64, + "--ledger-facts", + str(paths["ledger"]), + "--ledger-facts-sha256", + "b" * 64, + "--ledger-manifest-sha256", + "c" * 64, + "--staging-h5", + str(paths["staging"]), + "--diagnostics-json", + str(paths["diagnostics"]), + "--build-record-json", + str(paths["record"]), + "--release-id", + "dev-calibration", + ] + + +def test_driver_refuses_release_candidate_with_each_override(tmp_path: Path): + driver = _load_driver_module() + override_flags = [ + ["--epochs", "128"], + ["--target-weight-rule", "family_equal"], + ["--learning-rate", "0.01"], + ["--target-loss-cap", "5"], + ] + for flag in override_flags: + with pytest.raises(SystemExit): + driver._parse_args(_args(tmp_path) + ["--release-candidate", *flag]) + + +def test_driver_refuses_bad_sha_and_path_alias(tmp_path: Path): + driver = _load_driver_module() + base_args = _args(tmp_path) + with pytest.raises(SystemExit): + driver._parse_args([*base_args[:3], "not-a-sha", *base_args[4:]]) + + args = _args(tmp_path) + staging_index = args.index("--staging-h5") + 1 + args[staging_index] = args[args.index("--input-h5") + 1] + with pytest.raises(SystemExit, match="distinct paths"): + driver._parse_args(args) + + +def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, tmp_path, capsys): + driver = _load_driver_module() + calls = [] + registry = _registry() + artifact = SimpleNamespace( + path=tmp_path / "ledger", + facts=({"fact": 1},), + facts_sha256="b" * 64, + manifest_sha256="c" * 64, + ) + artifact.path.mkdir() + (artifact.path / "consumer_facts.jsonl").write_text("{}", encoding="utf-8") + monkeypatch.setattr(driver, "load_ledger_consumer_artifact", lambda *a, **k: artifact) + monkeypatch.setattr( + driver, + "compile_uk_target_registry", + lambda facts, target_period: UKLedgerTargetCompilation(registry, ()), + ) + monkeypatch.setattr( + driver, + "load_uk_frs_release", + lambda: SimpleNamespace(calibration_year=2025), + ) + monkeypatch.setattr(driver, "load_uk_calibration_measure_exclusions", lambda path: ()) + monkeypatch.setattr( + driver, + "apply_uk_calibration_measure_exclusions", + lambda reg, exclusions: (reg, {"excluded": "reviewed"}), + ) + + class FakeResolver: + def __init__(self, **kwargs): + self.kwargs = kwargs + + monkeypatch.setattr(driver, "UKMeasureResolver", FakeResolver) + + def fake_run(**kwargs): + calls.append(kwargs) + return SimpleNamespace( + staging_sha256="1" * 64, + diagnostics_sha256="2" * 64, + terminal_gate_sha256="3" * 64, + build_record_sha256="4" * 64, + build_record={"gate_summary": {"uk_target_fit": "passed"}}, + ) + + monkeypatch.setattr(driver, "run_uk_calibration", fake_run) + + argv = _args(tmp_path) + # The driver verifies the input pin before the resolver reads the H5, so + # the threading test must pin the fixture file's real digest. + argv[argv.index("--input-sha256") + 1] = driver._sha256_file( + Path(argv[argv.index("--input-h5") + 1]) + ) + result = driver.main(argv + ["--epochs", "128"]) + + assert result == 0 + call = calls[0] + assert call["register_registry"] is registry + assert call["calibration_year"] == 2025 + assert call["exclusion_receipt"] == {"excluded": "reviewed"} + assert call["doctrine"].epochs == 128 + assert call["doctrine_overrides"] == {"epochs": {"default": 256, "effective": 128}} + assert isinstance(call["measure_resolver"], FakeResolver) + assert call["measure_resolver"].kwargs["simulation_source"] == call["paths"].input_h5 + assert call["source_pins"]["ledger_facts"] == {"sha256": "b" * 64, "size_bytes": 2} + assert "uk_target_fit" in capsys.readouterr().out diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py new file mode 100644 index 0000000000..8c31b26cf1 --- /dev/null +++ b/tools/calibrate_uk_national_dataset.py @@ -0,0 +1,252 @@ +"""Calibrate an existing UK national spine H5 without replaying source stages. + +This driver is intentionally thin: it verifies pinned inputs, compiles the +Ledger target registry, applies the reviewed measure-exclusion register, records +any explicit doctrine overrides, and delegates the calibration/gate/logbook +work to :func:`microcosm.build.uk_runtime.calibration_run.run_uk_calibration`. + +Signed deviation for v1: no sampling rungs and no checkpointing. This seam runs +full-scale only; scale ladders and resumable source-stage checkpoints belong to +the spine build lane. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from itertools import combinations +from pathlib import Path +from typing import Any + +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.uk_runtime.calibration_run import ( + UKCalibrationRunPaths, + run_uk_calibration, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry +from microcosm.build.uk_runtime.measure_simulation import ( + UKMeasureResolver, + apply_uk_calibration_measure_exclusions, + load_uk_calibration_measure_exclusions, +) +from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_CANONICAL_UK_RELEASE_ID = re.compile(r"populace-uk-[1-9][0-9]*-[a-z0-9_]+-k[1-9][0-9]*") +_UK_JUNE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + artifact = load_ledger_consumer_artifact( + args.ledger_facts, + expected_facts_sha256=args.ledger_facts_sha256, + expected_manifest_sha256=args.ledger_manifest_sha256, + ) + calibration_year = load_uk_frs_release().calibration_year + compilation = compile_uk_target_registry( + artifact.facts, target_period=calibration_year + ) + if compilation.unsupported: + raise SystemExit( + f"{len(compilation.unsupported)} target references failed to compile" + ) + _compare_frozen_register(args.register_json, compilation.registry) + exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) + registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( + compilation.registry, exclusions + ) + overrides = { + key: value + for key, value in { + "epochs": args.epochs, + "target_weight_rule": args.target_weight_rule, + "learning_rate": args.learning_rate, + "target_loss_cap": args.target_loss_cap, + }.items() + if value is not None + } + doctrine, doctrine_overrides = uk_doctrine_with_overrides(**overrides) + paths = UKCalibrationRunPaths( + input_h5=args.input_h5, + staging_h5=args.staging_h5, + diagnostics_json=args.diagnostics_json, + build_record_json=args.build_record_json, + terminal_gate_json=args.terminal_gate_json, + ) + # The resolver reads the input H5 to build its simulation, so the pin is + # verified here first — no bytes are consumed before they match the CLI sha. + measured_input_sha = _sha256_file(args.input_h5) + if measured_input_sha != args.input_sha256: + raise SystemExit( + "error: --input-h5 sha mismatch: " + f"measured {measured_input_sha}, pinned {args.input_sha256}" + ) + resolver = UKMeasureResolver( + simulation_source=args.input_h5, + scratch_dir=args.staging_h5.parent, + year=calibration_year, + frame=None, + ) + result = run_uk_calibration( + paths=paths, + input_sha256=args.input_sha256, + ledger_artifact=artifact, + register_registry=registry, + calibration_year=calibration_year, + exclusion_receipt=exclusion_receipt, + doctrine=doctrine, + doctrine_overrides=doctrine_overrides, + measure_resolver=resolver, + source_pins={ + "input_h5": { + "sha256": args.input_sha256, + "size_bytes": args.input_h5.stat().st_size, + }, + "ledger_facts": _ledger_facts_pin(artifact), + }, + run_config_extra={"calibration_year": calibration_year}, + release_candidate=args.release_candidate, + release_id=args.release_id, + logbook_prev_row_digest=args.logbook_prev_row_digest, + ) + summary = { + "staging_h5_sha256": result.staging_sha256, + "diagnostics_sha256": result.diagnostics_sha256, + "terminal_gate_sha256": result.terminal_gate_sha256, + "build_record_sha256": result.build_record_sha256, + "gate_verdicts": result.build_record["gate_summary"], + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-h5", required=True, type=Path) + parser.add_argument("--input-sha256", required=True, type=_sha256) + parser.add_argument("--ledger-facts", required=True, type=Path) + parser.add_argument("--ledger-facts-sha256", required=True, type=_sha256) + parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) + parser.add_argument("--staging-h5", required=True, type=Path) + parser.add_argument("--diagnostics-json", required=True, type=Path) + parser.add_argument("--build-record-json", required=True, type=Path) + parser.add_argument("--release-id", required=True) + parser.add_argument("--terminal-gate-json", type=Path) + parser.add_argument("--register-json", type=Path) + parser.add_argument("--measure-exclusions", type=Path) + parser.add_argument("--release-candidate", action="store_true") + parser.add_argument("--logbook-prev-row-digest", type=_sha256) + parser.add_argument("--epochs", type=int) + parser.add_argument("--target-weight-rule") + parser.add_argument("--learning-rate", type=float) + parser.add_argument("--target-loss-cap", type=float) + args = parser.parse_args(argv) + args.terminal_gate_json = args.terminal_gate_json or args.staging_h5.with_suffix( + ".terminal_gates.json" + ) + override_flags = { + "--epochs": args.epochs, + "--target-weight-rule": args.target_weight_rule, + "--learning-rate": args.learning_rate, + "--target-loss-cap": args.target_loss_cap, + } + passed_overrides = [flag for flag, value in override_flags.items() if value is not None] + if args.release_candidate and passed_overrides: + parser.error( + "--release-candidate is refused with doctrine override flag(s): " + + ", ".join(passed_overrides) + ) + if ( + not args.release_candidate + and (_CANONICAL_UK_RELEASE_ID.fullmatch(args.release_id) or args.release_id == _UK_JUNE_RELEASE_ID) + ): + parser.error("canonical UK release ids require --release-candidate") + _validate_distinct_paths( + { + "--input-h5": args.input_h5, + "--ledger-facts": args.ledger_facts, + "--staging-h5": args.staging_h5, + "--diagnostics-json": args.diagnostics_json, + "--build-record-json": args.build_record_json, + "--terminal-gate-json": args.terminal_gate_json, + **({"--register-json": args.register_json} if args.register_json else {}), + **( + {"--measure-exclusions": args.measure_exclusions} + if args.measure_exclusions + else {} + ), + } + ) + return args + + +def _sha256(value: str) -> str: + if not _SHA256.fullmatch(value): + raise argparse.ArgumentTypeError("expected a lowercase SHA-256 digest") + return value + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _validate_distinct_paths(paths: dict[str, Path]) -> None: + resolved = {label: path.expanduser().resolve() for label, path in paths.items()} + for (left_label, left), (right_label, right) in combinations(resolved.items(), 2): + if _paths_alias(left, right): + raise SystemExit( + f"error: {left_label} and {right_label} must be distinct paths " + f"({left} aliases {right})" + ) + + +def _paths_alias(left: Path, right: Path) -> bool: + if str(left).casefold() == str(right).casefold(): + return True + try: + left_stat = left.stat() + right_stat = right.stat() + except FileNotFoundError: + return False + return (left_stat.st_dev, left_stat.st_ino) == (right_stat.st_dev, right_stat.st_ino) + + +def _compare_frozen_register(path: Path | None, registry: Any) -> None: + if path is None: + return + payload = { + "country": registry.country, + "version": registry.version, + "specs": [ + {field: value for field, value in spec.__dict__.items() if value is not None} + for spec in registry.specs + ], + } + derived = hashlib.sha256( + json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") + ).hexdigest() + frozen = hashlib.sha256(path.read_bytes()).hexdigest() + if derived != frozen: + raise SystemExit( + "re-derived register differs from the frozen scoring register: " + f"{derived} vs {frozen}" + ) + + +def _ledger_facts_pin(artifact: Any) -> dict[str, object]: + facts_path = ( + artifact.path / "consumer_facts.jsonl" if artifact.path.is_dir() else artifact.path + ) + return {"sha256": artifact.facts_sha256, "size_bytes": facts_path.stat().st_size} + + +if __name__ == "__main__": + raise SystemExit(main()) From 38bb5f743eeb52ce086f713be2c29f97c7b2345e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:37:48 +0200 Subject: [PATCH 20/26] Address Vahid's review: absent holdout losses, drop the duplicated vintage line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 2 (confirmed). The score block copied final_loss into candidate_holdout_loss and incumbent_holdout_loss while declaring holdout_basis none_declared. The June fixture this block mirrors carries a genuinely different holdout value — 0.1239 against a 0.0159 train loss, the eightfold degradation that made its holdout the discriminating comparison — so repeating the fitted loss there reads downstream as perfect generalization from a measurement never made. The keys now report absence, and declaring a basis without computing its split raises rather than falling back. The scorer also now takes the H5 loader from its new home instead of the legacy build module. Finding 4 (confirmed). source_vintages["frs"] was assigned twice, so load_uk_frs_release() ran twice; main carries one line, which makes this a branch-local artifact of the same merge that duplicated the spine driver's failure handler. A sweep of tools/ and uk_runtime/ for the same defect class found no others. Finding 1 (refuted, comment only). The staging-tier early return skips the certified-pin equality, which is what the tier declares, but the bytes are bound before it: the verification token is a module-private sentinel only the two verifiers stamp, and the staging verifier hashes the file against a mandatory declared sha with a mid-read race guard. The comment now says so, since the shape invites the reading twice now. Finding 3 was already fixed on this branch before the review was posted. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/hmrc_restoration.py | 11 ++++++++ .../tests/test_score_uk_national_candidate.py | 16 +++++++++-- tools/build_uk_national_dataset.py | 5 ++-- tools/score_uk_national_candidate.py | 28 ++++++++++++++++--- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py index 1f02c66554..520bb36693 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py @@ -849,6 +849,17 @@ def _validate_certified_candidate_identity( identity.size_bytes, ) if identity.tier == STAGING_CANDIDATE_TIER: + # A declared staging-candidate input is deliberately not the certified + # artifact, so the certified-pin equality below cannot apply. The bytes + # are still bound, by the two checks above rather than by this branch: + # the verification token is a module-private sentinel that only + # verify_certified_uk_candidate and verify_staging_candidate_uk_input + # stamp, and the latter hashes the file against a mandatory declared + # sha256 (re-reading the fingerprint to catch a mid-read swap) and + # refuses on mismatch; the source-file fingerprint is required of both + # tiers. What this tier declares is "the file the operator named", and + # the build record labels it non_certified_staging_candidate while + # --staging-candidate-input-sha256 is refused for release candidates. if identity.revision != STAGING_CANDIDATE_REVISION: raise ValueError( "UK staging-candidate input identity has an invalid revision " diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py index 36961df295..9cd79ca8d2 100644 --- a/packages/microcosm-build/tests/test_score_uk_national_candidate.py +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -85,8 +85,12 @@ def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: assert score["candidate_full_loss"] == pytest.approx(0.25) assert score["incumbent_full_loss"] == pytest.approx(0.1) assert score["candidate_train_loss"] == score["candidate_full_loss"] - assert score["candidate_holdout_loss"] == score["candidate_full_loss"] + # With no declared split, the holdout keys report absence rather than + # repeating the fitted loss under a name that means generalization. + assert score["candidate_holdout_loss"] is None + assert score["incumbent_holdout_loss"] is None assert score["holdout_basis"] == "none_declared" + assert score["loss"]["train_equals_full"] is True assert score["candidate_target_wins"] == 1 assert score["incumbent_target_wins"] == 1 assert score["target_wins_by_family"] == { @@ -140,5 +144,13 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: ) payload = json.loads(output_json.read_text(encoding="utf-8")) - assert payload["score_vs_enhanced_frs"]["holdout_basis"] == "none_declared" + score = payload["score_vs_enhanced_frs"] + assert score["holdout_basis"] == "none_declared" + # An undeclared holdout reports absence, never the fitted loss wearing a + # holdout name: June's fixture holds a genuinely different holdout value, + # so a copied one would read as generalization that was never measured. + assert score["candidate_holdout_loss"] is None + assert score["incumbent_holdout_loss"] is None + assert isinstance(score["candidate_train_loss"], float) + assert isinstance(score["incumbent_full_loss"], float) assert _load_registry(registry_json).version == _registry().version diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index 00c4fdc4d7..a600bf05dd 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -463,10 +463,10 @@ def _ledger_facts_pin(ledger_artifact: object) -> dict[str, object]: over a multi-hundred-megabyte file. """ - path = Path(getattr(ledger_artifact, "path")) + path = Path(ledger_artifact.path) facts_path = path / "consumer_facts.jsonl" if path.is_dir() else path return { - "sha256": str(getattr(ledger_artifact, "facts_sha256")), + "sha256": str(ledger_artifact.facts_sha256), "size_bytes": int(facts_path.stat().st_size), } @@ -1220,7 +1220,6 @@ def _aggregate_build_record( source_vintages["frs"] = load_uk_frs_release().vintage if ledger_artifact_provenance is not None: source_vintages["ledger_facts"] = ledger_artifact_provenance - source_vintages["frs"] = load_uk_frs_release().vintage return { "schema_version": 3, "build_kind": "uk_national_staging_dataset", diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index 3f98959fba..620a13e17c 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -7,10 +7,30 @@ from pathlib import Path from typing import Any -from microcosm.build.uk_runtime.national_build import load_uk_national_frame +from microcosm.build.uk_runtime.national_frame import load_uk_national_frame from microcosm.calibrate import TargetRegistry, TargetSpec, score_targets UK_SCORE_LOSS_CAP = 10.0 +UK_SCORE_HOLDOUT_BASIS = "none_declared" + + +def _holdout_loss(basis: str) -> float | None: + """The holdout loss for a declared basis, or None when none exists. + + June's frozen fixture carries a genuinely different holdout value + (0.1239 against a 0.0159 train loss) because it held rows out. This + register declares no split, so the holdout keys report absence rather + than the fitted loss wearing a holdout name — a copied value reads + downstream as perfect generalization from a measurement never made. + Declaring a basis without computing its split fails loudly here rather + than falling back to the fitted loss. + """ + + if basis == "none_declared": + return None + raise NotImplementedError( + f"holdout basis {basis!r} declares a split this scorer does not compute." + ) def score_uk_national_candidate( @@ -49,14 +69,14 @@ def score_uk_national_candidate( wins = _target_wins(candidate_errors, incumbent_errors) return { "candidate_train_loss": float(candidate.final_loss), - "candidate_holdout_loss": float(candidate.final_loss), + "candidate_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), "candidate_full_loss": float(candidate.final_loss), "incumbent_train_loss": float(incumbent.final_loss), - "incumbent_holdout_loss": float(incumbent.final_loss), + "incumbent_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), "incumbent_full_loss": float(incumbent.final_loss), "candidate_target_wins": wins["candidate"], "incumbent_target_wins": wins["incumbent"], - "holdout_basis": "none_declared", + "holdout_basis": UK_SCORE_HOLDOUT_BASIS, "loss": { "objective": "relative_error_loss", "target_loss_cap": UK_SCORE_LOSS_CAP, From 74b4d768f7f7c83eb0593464ceb0e2a7c81ec154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:16:30 +0200 Subject: [PATCH 21/26] Skip the seam's H5 round-trip tests when pytables is absent The wheels gate installs the base wheels without the optional pandas HDF backend, so the three calibration-run tests that write and re-read a real staged H5 failed there with an ImportError while passing everywhere else. They now carry the repo's existing guard, matching every other test that touches the HDF backend. The two other new seam test files monkeypatch the writer rather than performing real H5 I/O, so they need no guard and already passed the gate. Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/tests/test_uk_calibration_run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 5997b049fb..1c6a5abfff 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -108,6 +108,7 @@ def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend monkeypatch.setattr( calibration_run, "_aggregate_admin_totals", @@ -164,6 +165,7 @@ def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: P def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend input_h5 = tmp_path / "input.h5" write_uk_national_frame(_frame(), input_h5) paths = UKCalibrationRunPaths( @@ -203,6 +205,8 @@ def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): kind, and exactly one appended mass record may differ. """ + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( calibration_run, "_aggregate_admin_totals", From e4bee5303545a27413c87b0f6d775a30a6addeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:12:05 +0200 Subject: [PATCH 22/26] Count children as counts, and stop guessing which filter carries the band MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings on the calibration seam's measure surface, each one a wrong number that no error would have surfaced. The two-child-limit contract declares eight rows whose value is a count of children over a boolean member variable. The crosstab read that boolean at the target's own grain, so the resolver's person-to-household route collapsed it with `any` and published a household indicator against a child-count target. Counts now declare `value_reduction`, the adapter grows the numeric sibling of `household_condition` to sum members, and an adapter that cannot reduce refuses instead of falling back to the same-grain read. The stub and fixture frames that encoded the old assumption in a comment now carry real person rows, so the distinction is exercised rather than asserted. Band edges were taken from the alphabetically-first band-like Ledger filter with no tie to the binding's groupby variable, so a spec carrying two banded dimensions could slice on the wrong one — distinct, plausible, wrong numbers, the same class as the unsliced-measure defect one layer up. The edge is now attributed to the declared banding dimension, `band_filter_dimension` names it where the publisher's and the model's spellings differ, and an unresolvable tie raises. `UKMeasureResolver.knows` was vacuously true for every entity, so the resolution loop's refusal could never fire and an unmappable categorical surfaced as a wrapped provider exception. It now answers for the routes `compute_uk_measure_input` actually takes. Reported by @vahid-ahmadi (2, 3) and @MaxGhenis (1). Co-Authored-By: Claude Opus 5 --- .../microcosm/build/target_materialization.py | 81 +++++++++++++++-- .../build/uk/uk_national_targets.json | 40 +++++++++ .../build/uk_runtime/ledger_targets.py | 70 +++++++++++---- .../build/uk_runtime/measure_simulation.py | 20 ++++- .../tests/test_target_materialization.py | 89 +++++++++++++++++++ .../tests/test_uk_ledger_targets.py | 54 +++++++++-- .../tests/test_uk_measure_simulation.py | 64 ++++++++++++- .../tests/test_uk_national_calibration.py | 57 +++++++++--- .../tests/test_uk_national_targets.py | 17 ++++ 9 files changed, 447 insertions(+), 45 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index 802d12aea8..c9042df5c2 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -304,20 +304,64 @@ def _band_lower_edge( metadata = getattr(spec, "metadata", None) or {} factor = float(binding.get("band_period_factor", 1) or 1) + numeric: list[tuple[str, float]] = [] + labelled: list[tuple[str, float]] = [] for key, value in sorted(metadata.items()): if not key.startswith(_LEDGER_FILTER_PREFIX): continue if key.endswith(_BAND_LOWER_BOUND_SUFFIX): - return float(str(value).replace(",", "")) * factor - for key, value in sorted(metadata.items()): - if not key.startswith(_LEDGER_FILTER_PREFIX): + numeric.append((key, float(str(value).replace(",", "")))) continue match = _RANGE_LABEL.search(str(value)) or _OPEN_LABEL.search(str(value)) if match is not None: - return float(match.group(1).replace(",", "")) * factor + labelled.append((key, float(match.group(1).replace(",", "")))) + for candidates in (numeric, labelled): + if not candidates: + continue + return _select_band_edge(spec, binding, candidates) * factor return None +def _select_band_edge( + spec: Any, + binding: Mapping[str, Any], + candidates: Sequence[tuple[str, float]], +) -> float: + """Pick the band edge belonging to this binding's groupby variable. + + A spec can carry several Ledger filters, and more than one of them can + look like a band — an age filter alongside an income band, say. Slicing + on the wrong variable's edges produces plausible-looking but wrong + subpopulation totals, so the edge is bound to the declared banding + dimension and an unresolvable tie refuses rather than picking the + alphabetically-first key. + """ + + if len(candidates) == 1: + return candidates[0][1] + declared = binding.get("band_filter_dimension") + wanted = { + f"{_LEDGER_FILTER_PREFIX}{name}" + for name in (declared, binding.get("groupby_variable")) + if name + } + matched = [ + value + for key, value in candidates + if any(key == name or key.startswith(f"{name}_") for name in wanted) + ] + if len(matched) == 1: + return matched[0] + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} carries " + f"{len(candidates)} band-like Ledger filters " + f"({', '.join(key for key, _ in candidates)}) and " + f"{'none' if not matched else 'several'} of them belong to " + f"groupby_variable {binding.get('groupby_variable')!r}; declare " + "band_filter_dimension on the binding to name the banding dimension." + ) + + def _band_edges_by_group( registry: TargetRegistry, contract_targets: Mapping[str, Mapping[str, Any]], @@ -482,7 +526,15 @@ def baseline_flag_crosstab( flag = _column(adapter, entity, binding["affected_flag_variable"]).astype(bool) value_variable = str(binding.get("value_variable") or "") count_of = str(binding.get("count_of") or "") - if value_variable and value_variable not in _COUNT_VALUE_VARIABLES: + reduction = binding.get("value_reduction") + if reduction: + # A count declared over a member-level variable — the number of + # children in the household, not whether it has any. Reading such a + # boolean at the crosstab's own grain would collapse it to an + # indicator and publish that against a count target, so the reduction + # is declared and performed member-wise instead. + values = _entity_reduction(adapter, reduction) + elif value_variable and value_variable not in _COUNT_VALUE_VARIABLES: values = _column(adapter, entity, value_variable) elif count_of and count_of not in {"household", "person", "household_count"}: values = _column(adapter, entity, count_of) @@ -532,6 +584,25 @@ def _prepared_column_values( return np.where(mask, values, 0.0) +def _entity_reduction(adapter: Any, reduction: Mapping[str, Any]) -> np.ndarray: + """Reduce a member-level variable to the target entity's grain. + + Declared as ``value_reduction`` on a binding whose value is a count over + members. The adapter owns the linkage, the same way it owns + ``household_condition``; an adapter that cannot reduce refuses rather than + falling back to a same-grain read, which would silently substitute an + indicator for a count. + """ + + if not hasattr(adapter, "entity_reduction"): + raise ValueError( + "binding declares value_reduction " + f"{dict(reduction)!r} but the adapter cannot reduce member " + "variables; refusing to substitute a same-grain read." + ) + return np.asarray(adapter.entity_reduction(reduction), dtype=float) + + def _predicate_mask( adapter: Any, default_entity: str, diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json index 600162b7bf..4cff07befa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json @@ -3653,6 +3653,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, @@ -3686,6 +3691,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: children_in_affected_households." }, @@ -3770,6 +3780,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, @@ -3854,6 +3869,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, @@ -3938,6 +3958,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, @@ -4022,6 +4047,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, @@ -4106,6 +4136,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, @@ -4190,6 +4225,11 @@ "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 5bbb0dbcbc..50482bfcee 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -179,29 +179,65 @@ def counterfactual_delta( f"{metric_name!r}" ) - def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: - entity = str(condition.get("entity") or "household") + def _household_ids_for(self, entity: str) -> Any: + """The household each row of ``entity`` belongs to, row-aligned.""" + source = self.tables[entity] if entity == "household": - household_ids = source["household_id"] - elif entity == "person": + return source["household_id"] + if entity == "person": # People sit directly in a household; only group entities need the # membership lookup below, whose column would be the nonexistent # "person_person_id" here. - household_ids = source["person_household_id"] + return source["person_household_id"] + people = self.tables["person"] + entity_membership = f"person_{entity}_id" + if entity_membership not in people: + raise KeyError(entity_membership) + group_to_household = ( + people[[entity_membership, "person_household_id"]] + .drop_duplicates() + .set_index(entity_membership)["person_household_id"] + ) + if group_to_household.index.has_duplicates: + raise ValueError(f"UK {entity} groups span multiple households.") + return source[f"{entity}_id"].map(group_to_household) + + def entity_reduction(self, reduction: Mapping[str, Any]) -> np.ndarray: + """Reduce a member-level variable to a household-grain value. + + The numeric sibling of :meth:`household_condition`: where that answers + "does this household satisfy the predicate", this answers "what does + this household's members sum to". A count target declared over a + boolean member variable — the number of children, not whether the + household has any — is only honest through this path; collapsing the + boolean with ``any`` would publish an indicator against a count. + """ + + entity = str(reduction.get("entity") or "person") + source = self.tables[entity] + variable = str(reduction["variable"]) + if variable not in source: + raise KeyError(f"{entity}.{variable}") + household_ids = self._household_ids_for(entity) + reduce = str(reduction.get("reduce") or "sum") + values = source[variable] + if reduce == "sum": + aggregate = values.astype(float).groupby(household_ids).sum() + elif reduce == "count": + aggregate = values.groupby(household_ids).count() else: - people = self.tables["person"] - entity_membership = f"person_{entity}_id" - if entity_membership not in people: - raise KeyError(entity_membership) - group_to_household = ( - people[[entity_membership, "person_household_id"]] - .drop_duplicates() - .set_index(entity_membership)["person_household_id"] - ) - if group_to_household.index.has_duplicates: - raise ValueError(f"UK {entity} groups span multiple households.") - household_ids = source[f"{entity}_id"].map(group_to_household) + raise ValueError(f"Unsupported UK entity reduction {reduce!r}.") + households = self.tables["household"] + return np.asarray( + households["household_id"].map(aggregate).fillna(0.0), + dtype=float, + ) + + def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: + entity = str(condition.get("entity") or "household") + source = self.tables[entity] + household_ids = self._household_ids_for(entity) reduce = str(condition["reduce"]) variable = str(condition["variable"]) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index 4b545f7be9..38629f5767 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -119,11 +119,27 @@ def __init__( self.contract_targets = _uk_contract_targets() def knows(self, entity: str, variable: str) -> bool: + """Whether a route in :func:`compute_uk_measure_input` reaches here. + + The resolution loop turns a False into a named refusal; anything this + answers True to must actually compute, or the operator gets a wrapped + provider exception instead of the fence's message. + """ + definition = self.simulation.tax_benefit_system.variables.get(variable) - if definition is None: + if definition is None or entity not in _ENTITY_ID: return False native = definition.entity.key - return native == entity or entity in _ENTITY_ID + if native == entity: + return True + if "person" in (native, entity): + # Person-to-group and group-to-person are covered by the + # broadcast, any-collapse, and map_to routes alike. + return True + # Group to group has only the numeric map_to route: a categorical or + # boolean variable has no mapping between benunit and household and + # refuses here rather than deep inside the provider. + return getattr(definition, "value_type", None) in (int, float) def entity_for(self, variable: str) -> str | None: definition = self.simulation.tax_benefit_system.variables.get(variable) diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index faf27327a6..b4af50fb48 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -684,3 +684,92 @@ def test_crosstab_counts_a_real_value_variable_per_record(): # children is [3, 2, 4], masked by affected [1, 0, 1]. assert list(adapter.tables["household"]["affected_children"]) == [3.0, 0.0, 4.0] + + +def _two_filter_registry(*, matching: bool) -> TargetRegistry: + """Bands whose specs carry a second, competing band-like filter. + + ``matching`` decides whether the income filter's key relates to the + binding's ``groupby_variable``; when it does not, neither candidate can be + attributed and the edge must refuse rather than be guessed. + """ + + income_key = ( + "ledger_filter_income_lower_bound" + if matching + else "ledger_filter_total_income_lower_bound" + ) + specs = [] + for lower, age_lower, label in ((0, 16, "0"), (20, 30, "20")): + specs.append( + TargetSpec( + name=f"band_{label}", + entity="person", + measure=f"income_band_{label}", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + income_key: str(lower), + "ledger_filter_age_lower_bound": str(age_lower), + }, + ) + ) + return TargetRegistry(specs, country="uk") + + +def test_band_edges_follow_the_groupby_variable_not_alphabetical_order(): + # "age" sorts before "income", so the alphabetically-first key would have + # sliced the income measure on age edges: distinct, plausible, and wrong. + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, _two_filter_registry(matching=True), _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + # income is [10, 20, 30]; income bands are [0,20) and [20,inf). + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] + + +def test_unattributable_band_filters_refuse_rather_than_guess(): + adapter = StubAdapter() + + with pytest.raises(ValueError) as error: + materialize_target_bindings( + adapter, + _two_filter_registry(matching=False), + _BANDED_CONTRACT, + period=2025, + ) + + message = str(error.value) + assert "band-like Ledger filters" in message + assert "ledger_filter_age_lower_bound" in message + assert "band_filter_dimension" in message + + +def test_declared_band_filter_dimension_breaks_the_tie(): + adapter = StubAdapter() + contract = { + "spi.income_by_band": { + "bindings": { + "policyengine": { + **_BANDED_CONTRACT["spi.income_by_band"]["bindings"][ + "policyengine" + ], + "band_filter_dimension": "total_income", + } + } + } + } + + result = materialize_target_bindings( + adapter, _two_filter_registry(matching=False), contract, period=2025 + ) + + assert result.skipped == () + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] diff --git a/packages/microcosm-build/tests/test_uk_ledger_targets.py b/packages/microcosm-build/tests/test_uk_ledger_targets.py index 85b08c0fa8..793222f25d 100644 --- a/packages/microcosm-build/tests/test_uk_ledger_targets.py +++ b/packages/microcosm-build/tests/test_uk_ledger_targets.py @@ -15,25 +15,52 @@ class StubUKAdapter: + """Seven people across three households, flagged child by flagged child. + + The flag and the count are deliberately distinct here: household 0 holds + three flagged children and household 2 holds two, so a household-grain + read of the boolean (1/0/1) can never be mistaken for the child counts + (3/0/2) the contract's count rows declare. + """ + def __init__(self): + person_household = np.array([0, 0, 0, 0, 1, 2, 2]) + child_flags = np.array([True, True, True, False, False, True, True]) self.tables = { "person": { - "capital_gains": np.array([0.0, 5_000.0, 20_000.0]), + "capital_gains": np.array( + [0.0, 0.0, 0.0, 5_000.0, 0.0, 0.0, 20_000.0] + ), + "person_household_id": person_household, + "uc_is_child_limit_affected": child_flags, }, "household": { - # Mapped to household, uc_is_child_limit_affected sums to the - # number of flagged children, so it is both the affected flag - # and the affected-children count. - "uc_is_child_limit_affected": np.array([3.0, 0.0, 4.0]), + "household_id": np.array([0, 1, 2]), + # The household-grain flag is the any-collapse: "this + # household contains at least one flagged child". + "uc_is_child_limit_affected": np.array([1.0, 0.0, 1.0]), }, } - def column(self, entity, variable): return self.tables[entity][variable] def set_column(self, entity, variable, values): self.tables.setdefault(entity, {})[variable] = np.asarray(values) + def entity_reduction(self, reduction): + assert reduction["reduce"] == "sum" + assert reduction["entity"] == "person" + variable = reduction["variable"] + values = self.tables["person"][variable].astype(float) + households = self.tables["person"]["person_household_id"] + return np.asarray( + [ + values[households == household_id].sum() + for household_id in self.tables["household"]["household_id"] + ], + dtype=float, + ) + def parameter(self, name, period): assert name == "gov.hmrc.cgt.annual_exempt_amount" assert period == 2025 @@ -94,10 +121,21 @@ def test_materialize_uk_ledger_targets_with_stub_adapter(): result = materialize_uk_ledger_targets(adapter, registry, period=2025) assert result.skipped == () - assert adapter.tables["person"]["hmrc/cgt_taxpayers"].tolist() == [0.0, 0.0, 1.0] + assert adapter.tables["person"]["hmrc/cgt_taxpayers"].tolist() == [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + ] + # Child counts, not household indicators: the declared value_reduction + # sums the flag over each household's people. A boolean any-collapse + # would have published [1.0, 0.0, 1.0] against a count target. assert adapter.tables["household"][ "dwp/uc/two_child_limit/children_affected" - ].tolist() == [3.0, 0.0, 4.0] + ].tolist() == [3.0, 0.0, 2.0] def _composition_frame(): diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index a5ebee21a3..c80273efcc 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -37,12 +37,23 @@ def table(self, entity): return self._tables[entity] +def _stub_value_type(values): + """The policyengine ``value_type`` a real variable definition would carry.""" + + return {"f": float, "i": int, "b": bool}.get( + np.asarray(values).dtype.kind, str + ) + + class SimulationStub: def __init__(self, values): self.values = values variables = { - name: SimpleNamespace(entity=SimpleNamespace(key=entity)) - for name, (entity, _values) in values.items() + name: SimpleNamespace( + entity=SimpleNamespace(key=entity), + value_type=_stub_value_type(stub_values), + ) + for name, (entity, stub_values) in values.items() } self.tax_benefit_system = SimpleNamespace(variables=variables) self.calls = [] @@ -229,3 +240,52 @@ def __init__(self, *, dataset): assert scratch.receipt()["mode"] == "scratch_frame_export" assert writes == [(frame, tmp_path / "simulation-input.h5")] assert created == [str(tmp_path / "input.h5"), str(tmp_path / "simulation-input.h5")] + + +def _resolver_over(sim, monkeypatch, tmp_path: Path) -> UKMeasureResolver: + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace(__version__="9.9.9", Microsimulation=lambda *, dataset: sim), + ) + return UKMeasureResolver( + simulation_source=tmp_path / "input.h5", + scratch_dir=tmp_path, + year=2025, + frame=FrameStub(), + ) + + +def test_knows_answers_only_for_routes_compute_can_take(monkeypatch, tmp_path: Path): + sim = SimulationStub( + { + "income_tax": ("person", np.array([1.0, 2.0, 3.0])), + "is_disabled": ("person", np.array([False, True, False])), + "family_type": ("benunit", np.array(["couple", "single"])), + "benunit_income": ("benunit", np.array([1.0, 2.0])), + } + ) + resolver = _resolver_over(sim, monkeypatch, tmp_path) + + assert resolver.knows("person", "income_tax") + assert resolver.knows("household", "income_tax") + assert resolver.knows("household", "is_disabled") + assert resolver.knows("person", "family_type") + # Numeric group-to-group rides the map_to route… + assert resolver.knows("household", "benunit_income") + # …but a categorical one has no benunit-to-household mapping at all, and + # the fence must say so rather than let compute raise mid-resolution. + assert not resolver.knows("household", "family_type") + assert not resolver.knows("person", "no_such_variable") + assert not resolver.knows("firm", "income_tax") + + +def test_unknown_categorical_mapping_refuses_through_the_fence( + monkeypatch, tmp_path: Path +): + sim = SimulationStub({"family_type": ("benunit", np.array(["couple", "single"]))}) + resolver = _resolver_over(sim, monkeypatch, tmp_path) + + assert not resolver.knows("household", "family_type") + with pytest.raises(KeyError, match="no categorical mapping"): + resolver.compute("household", "family_type") diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 52b02c0849..4275bbf320 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -24,7 +24,10 @@ uk_doctrine_with_overrides, uk_national_target_loss_weights, ) -from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation +from microcosm.build.uk_runtime.ledger_targets import ( + UKLedgerTargetCompilation, + _uk_contract_targets, +) from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, @@ -159,6 +162,29 @@ def receipt(self): return {"provider": "stub_uc"} +class StubCrosstabResolver: + """Supplies the household-grain affected flag, as production does. + + ``uc_is_child_limit_affected`` is person-native in policyengine-uk, and + the Frame's global column-uniqueness rule forbids the same name on two + entity tables — so the household-grain flag can only ever arrive as a + table-scoped adapter injection. That is the production route, and it is + the route exercised here. + """ + + contract_targets = None + + def knows(self, entity, variable): + return (entity, variable) == ("household", "uc_is_child_limit_affected") + + def compute(self, entity, variable): + assert (entity, variable) == ("household", "uc_is_child_limit_affected") + return np.array([1.0, 0.0, 1.0]), "stub_any_collapse_person_to_household" + + def receipt(self): + return {"provider": "stub_crosstab_flag"} + + def _nested_frame() -> Frame: return Frame( { @@ -228,21 +254,19 @@ def _materialization_binding_frame( "household_id": np.arange(3, dtype="int64"), "esa_income": [10.0, 20.0, 0.0], "esa_contrib": [1.0, 2.0, 0.0], - # Mapped to household, uc_is_child_limit_affected sums to the - # number of flagged children: both the affected flag and the - # affected-children count. - "uc_is_child_limit_affected": [2.0, 0.0, 3.0], } ) salary_sacrifice_metric = "hmrc/salary_sacrifice_it_relief_basic_rate" person_columns = { - "person_id": np.arange(4, dtype="int64"), - "person_benunit_id": [0, 0, 1, 2], - "person_household_id": [0, 0, 1, 2], - "capital_gains": [0.0, 7_000.0, 12_000.0, 500.0], + "person_id": np.arange(6, dtype="int64"), + "person_benunit_id": [0, 0, 1, 2, 2, 2], + "person_household_id": [0, 0, 1, 2, 2, 2], + "capital_gains": [0.0, 7_000.0, 12_000.0, 500.0, 0.0, 0.0], + # Two flagged children in household 0, none in 1, three in 2. + "uc_is_child_limit_affected": [True, True, False, True, True, True], } if include_counterfactual_delta: - person_columns[salary_sacrifice_metric] = [1.0, 2.0, 0.0, 0.0] + person_columns[salary_sacrifice_metric] = [1.0, 2.0, 0.0, 0.0, 0.0, 0.0] return Frame( { "person": pd.DataFrame(person_columns), @@ -433,10 +457,13 @@ def test_packaged_binding_classes_materialize_through_national_stage() -> None: ) registry = compile_ledger_target_references(facts, references, country="uk") + resolver = StubCrosstabResolver() + resolver.contract_targets = _uk_contract_targets() stage = UKNationalCalibrationStage( registry, period=2025, doctrine=UKNationalSolveDoctrine(epochs=1, learning_rate=0.01), + measure_resolver=resolver, ) input_frame = _materialization_binding_frame() @@ -458,17 +485,25 @@ def test_packaged_binding_classes_materialize_through_national_stage() -> None: assert set(result.table(entity).columns) == original_columns[entity] # The binding classes produce the right prepared values on the adapter… adapter = UKFrameTargetAdapter(_materialization_binding_frame()) + # The same table-scoped injection the resolution loop performs. + adapter.tables["household"]["uc_is_child_limit_affected"] = np.array( + [1.0, 0.0, 1.0] + ) materialize_uk_ledger_targets(adapter, registry, period=2025) materialized = { ("benunit", "dwp/uc/households"): [1.0, 0.0, 1.0], ("household", "obr/esa"): [11.0, 22.0, 0.0], - ("person", "hmrc/cgt_taxpayers"): [0.0, 1.0, 1.0, 0.0], + ("person", "hmrc/cgt_taxpayers"): [0.0, 1.0, 1.0, 0.0, 0.0, 0.0], + # Counts of flagged children, reduced from the person rows — not the + # household indicator [1.0, 0.0, 1.0] a same-grain boolean read gives. ("household", "dwp/uc/two_child_limit/children_affected"): [2.0, 0.0, 3.0], ("person", "hmrc/salary_sacrifice_it_relief_basic_rate"): [ 1.0, 2.0, 0.0, 0.0, + 0.0, + 0.0, ], } for (entity, measure), expected in materialized.items(): diff --git a/packages/microcosm-build/tests/test_uk_national_targets.py b/packages/microcosm-build/tests/test_uk_national_targets.py index 133d4e4368..02ac5e428f 100644 --- a/packages/microcosm-build/tests/test_uk_national_targets.py +++ b/packages/microcosm-build/tests/test_uk_national_targets.py @@ -47,6 +47,7 @@ POLICYENGINE_BINDING_KEYS = { "affected_flag_variable", "band", + "band_filter_dimension", "band_period_factor", "count_of", "filters", @@ -58,6 +59,7 @@ "groupby_variable", "household_conditions", "kind", + "value_reduction", "map_to", "metric_name", "notes", @@ -284,6 +286,21 @@ def test_uk_national_targets_declare_chronicle_loader_guarantees(): # still accepts. Either key must name the counted column. assert {"value_variable", "count_of"} & set(binding), target["target_id"] + reduction = binding.get("value_reduction") + if reduction is not None: + # A count over a member-level variable declares its reduction, so + # the materializer sums members instead of reading the variable at + # the target's own grain — a boolean read there would collapse to + # a household indicator and be published against a count. + assert set(reduction) == {"variable", "entity", "reduce"}, ( + target["target_id"] + ) + assert reduction["reduce"] in PREDICATE_REDUCERS, target["target_id"] + assert reduction["entity"] in PREDICATE_ENTITIES, target["target_id"] + assert reduction["variable"] == binding.get("value_variable"), ( + target["target_id"] + ) + if "reduce" in binding: assert binding["reduce"] in BINDING_REDUCERS, target["target_id"] for field in ("filters", "household_conditions"): From a9db438700dbd79c72755ac6fa17ac8b1003beaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:22:54 +0200 Subject: [PATCH 23/26] Put the calibration seam's attempts on the ratified chain, and seal what it verified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam declared pipeline `uk-national-calibration`, which derives the scope `uk/national` — not in the ratified vocabulary. The FRS line's spine, staging, imputation and calibration stages share one chain by design (the dataset token names the base data, not the build mechanism), so calibration is `uk-frs-calibration` and lands on `uk/frs`. Three more lifecycle gaps went with it. The predecessor digest was resolved at the very end, so a disagreeing chain refused only after a staged H5, diagnostics and a signed gate report already existed; it is now validated before anything is written. Only successful runs recorded a row, against the binding rule that successful, failed, refused and discarded attempts all produce one; the attempt is now wrapped, and a refusal writes an error receipt and a `failed` row while staging nothing. And the build id was deterministic per release id, which both the local chain and the store reject on a re-run; attempts now carry unique ids and the release id travels in run_config. `run_uk_calibration` also accepted `ledger_artifact` and never used it, so the facts and manifest digests the caller had just verified reached neither the identity digest, the build record, nor the Logbook row. The verified identity is now sealed into run_config; a bare feed's missing manifest is recorded as absent rather than invented. Release candidates no longer accept `--measure-exclusions`: the register prunes the target surface before the solve and the calibration-scoped battery carries no target-surface gate, so an operator file could narrow what a candidate was measured against unnoticed. The applied receipt now keeps each exclusion's tracking reference, and the loader requires one. Reported by @MaxGhenis (findings 4, 5, 8). Co-Authored-By: Claude Opus 5 --- .../build/uk_runtime/calibration_run.py | 163 +++++++++++++++- .../build/uk_runtime/measure_simulation.py | 30 ++- .../tests/test_uk_calibration_run.py | 177 ++++++++++++++++++ .../tests/test_uk_calibration_seam_driver.py | 21 +++ .../tests/test_uk_measure_simulation.py | 21 ++- tools/calibrate_uk_national_dataset.py | 11 ++ 6 files changed, 409 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 5ac51849fb..3f124bb1d4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -9,6 +9,7 @@ import json import os import time +import uuid from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -29,11 +30,14 @@ from microcosm.build.logbook_adoption import ( AttemptState, append_phase, + apply_error_verdict, + error_receipt_path, git_code_pin, local_artifact_reference, record_terminal_attempt, resolve_predecessor, role_pins_digest, + write_error_receipt, ) from microcosm.build.target_materialization import assert_calibration_input_finite from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY @@ -50,7 +54,10 @@ from microcosm.frame import Frame _REPOSITORY = Path(__file__).resolve().parents[6] -_PIPELINE = "uk-national-calibration" +# The FRS line's spine, staging, imputation and calibration stages share one +# hash chain (logbook/README.md): the dataset token names the base data, not +# the build mechanism, so calibration derives the ratified `uk/frs` scope. +_PIPELINE = "uk-frs-calibration" @dataclass(frozen=True) @@ -123,7 +130,7 @@ def run_uk_calibration( ledger_artifact: Any, register_registry: TargetRegistry, calibration_year: int, - exclusion_receipt: Mapping[str, str], + exclusion_receipt: Mapping[str, Mapping[str, str]], doctrine: Any, doctrine_overrides: Mapping[str, Mapping[str, object]], measure_resolver: object | None, @@ -138,22 +145,143 @@ def run_uk_calibration( started_at = time.perf_counter() started_ts = datetime.now(UTC) code_pin = git_code_pin(_REPOSITORY) + # Predecessor configuration is validated before anything is written: a + # disagreeing chain must refuse with no artifact on disk, not after a + # staged H5, diagnostics and a signed gate report already exist. + predecessor = resolve_predecessor(logbook_prev_row_digest) run_config = { "pipeline": _PIPELINE, + "release_id": release_id, "register_sha256": register_registry.version, "calibration_year": int(calibration_year), "doctrine": _doctrine_payload(doctrine), "doctrine_overrides": dict(doctrine_overrides), + # The caller verifies the feed's facts and manifest digests; sealing + # the verified identity into run_config carries it through the + # identity digest, the build record and the Logbook row, so the run + # says which Ledger artifact it was measured against. + "ledger": _ledger_provenance(ledger_artifact), **dict(run_config_extra), } state = AttemptState( - build_id=f"uk-national-calibration-{release_id}", + # Attempts are distinct rows even when they re-run one release: both + # the local chain and the store refuse a repeated build id. + build_id=_new_calibration_attempt_id(timestamp=started_ts), identity_digest=hashlib.sha256(canonical_json_bytes(run_config)).hexdigest(), input_pins_digest=role_pins_digest(source_pins), phases_reached=["attempt_started"], gate_verdicts={}, ) + spool_dir = paths.staging_h5.parent / "logbook-spool" + try: + return _run_uk_calibration_attempt( + paths=paths, + input_sha256=input_sha256, + ledger_artifact=ledger_artifact, + register_registry=register_registry, + calibration_year=calibration_year, + exclusion_receipt=exclusion_receipt, + doctrine=doctrine, + doctrine_overrides=doctrine_overrides, + measure_resolver=measure_resolver, + source_pins=source_pins, + release_candidate=release_candidate, + release_id=release_id, + state=state, + run_config=run_config, + code_pin=code_pin, + started_at=started_at, + started_ts=started_ts, + predecessor=predecessor, + spool_dir=spool_dir, + ) + except BaseException as error: + # Every terminal disposition records a row — successful, failed, or + # refused (logbook/README.md). A refusal that left no row would be a + # silent gap in the chain the run is supposed to evidence. + _record_failed_attempt( + error=error, + state=state, + started_at=started_at, + started_ts=started_ts, + seed=getattr(doctrine, "seed", None), + code_pin=code_pin, + predecessor=predecessor, + receipt_base_dir=paths.staging_h5.parent, + spool_dir=spool_dir, + ) + raise + + +def _new_calibration_attempt_id(*, timestamp: datetime) -> str: + instant = timestamp.astimezone(UTC) + return ( + "uk-frs-calibration-attempt-" + f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" + ) + + +def _record_failed_attempt( + *, + error: BaseException, + state: AttemptState, + started_at: float, + started_ts: datetime, + seed: int | None, + code_pin: str, + predecessor: str | None, + receipt_base_dir: Path, + spool_dir: Path, +) -> None: + if state.spool_path is not None: + return + error_path = write_error_receipt( + error_receipt_path(receipt_base_dir, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}" + "#/error_type", + ) + record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung="f100", + seed=seed, + code_pin=code_pin, + disposition="failed", + predecessor=predecessor, + spool_dir=spool_dir, + ) + +def _run_uk_calibration_attempt( + *, + paths: UKCalibrationRunPaths, + input_sha256: str, + ledger_artifact: Any, + register_registry: TargetRegistry, + calibration_year: int, + exclusion_receipt: Mapping[str, Mapping[str, str]], + doctrine: Any, + doctrine_overrides: Mapping[str, Mapping[str, object]], + measure_resolver: object | None, + source_pins: Mapping[str, Mapping[str, object]], + release_candidate: bool, + release_id: str, + state: AttemptState, + run_config: Mapping[str, object], + code_pin: str, + started_at: float, + started_ts: datetime, + predecessor: str | None, + spool_dir: Path, +) -> UKCalibrationRunResult: measured_input_sha = _sha256_file(paths.input_h5) if measured_input_sha != input_sha256: raise ValueError( @@ -182,6 +310,7 @@ def run_uk_calibration( "build_id": state.build_id, "code_pin": code_pin, "source_pins": dict(source_pins), + "ledger": run_config["ledger"], "input_posture": { "tier": "staging_candidate", "sha256": measured_input_sha, @@ -271,8 +400,8 @@ def run_uk_calibration( seed=getattr(doctrine, "seed", None), code_pin=code_pin, disposition="iterating", - predecessor=resolve_predecessor(logbook_prev_row_digest), - spool_dir=paths.staging_h5.parent / "logbook-spool", + predecessor=predecessor, + spool_dir=spool_dir, ) return UKCalibrationRunResult( frame=calibrated, @@ -419,8 +548,30 @@ def _aggregate_admin_totals( return totals, receipt +def _ledger_provenance(artifact: Any) -> dict[str, object]: + """The verified identity of the Ledger consumer feed this run compiled. + + A bare ``consumer_facts.jsonl`` feed carries no manifest, so its + Ledger-side provenance is recorded as absent rather than invented. + """ + + provenance: dict[str, object] = { + "facts_sha256": getattr(artifact, "facts_sha256", None), + "fact_row_count": getattr(artifact, "fact_row_count", None), + "manifest_sha256": getattr(artifact, "manifest_sha256", None), + } + manifest = getattr(artifact, "manifest", None) + if isinstance(manifest, Mapping): + provenance["manifest"] = { + key: manifest.get(key) + for key in ("artifact_id", "profile", "schema_version", "generated_at") + if manifest.get(key) is not None + } + return provenance + + def _register_census( - registry: TargetRegistry, exclusions: Mapping[str, str] + registry: TargetRegistry, exclusions: Mapping[str, Mapping[str, str]] ) -> dict[str, object]: return { "country": registry.country, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index 38629f5767..b28beee1f6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -191,6 +191,11 @@ def load_uk_calibration_measure_exclusions( raise ValueError(f"duplicate UK calibration measure exclusion {name!r}.") if not reason.strip(): raise ValueError(f"UK calibration measure exclusion {name!r} has empty reason.") + if not tracking.strip(): + raise ValueError( + f"UK calibration measure exclusion {name!r} has empty tracking; " + "a narrowed target surface must name where it is being resolved." + ) seen.add(name) loaded.append({"name": name, "reason": reason, "tracking": tracking}) return tuple(loaded) @@ -198,19 +203,30 @@ def load_uk_calibration_measure_exclusions( def apply_uk_calibration_measure_exclusions( registry: TargetRegistry, exclusions: tuple[dict[str, str], ...] -) -> tuple[TargetRegistry, dict[str, str]]: - """Remove reviewed excluded references from a UK target registry.""" +) -> tuple[TargetRegistry, dict[str, dict[str, str]]]: + """Remove reviewed excluded references from a UK target registry. - reasons = {entry["name"]: entry["reason"] for entry in exclusions} - matched = {spec.name for spec in registry.specs if spec.name in reasons} - stale = sorted(set(reasons) - matched) + The receipt carries every field the register declares, tracking included: + an exclusion narrows the calibrated target surface, so the run evidence + must say where each narrowing is being resolved, not only why. + """ + + declared = {entry["name"]: entry for entry in exclusions} + matched = {spec.name for spec in registry.specs if spec.name in declared} + stale = sorted(set(declared) - matched) if stale: raise ValueError( "UK calibration measure exclusion matched zero registry specs: " + ", ".join(stale) ) - kept = [spec for spec in registry.specs if spec.name not in reasons] - receipt = {name: reasons[name] for name in sorted(matched)} + kept = [spec for spec in registry.specs if spec.name not in declared] + receipt = { + name: { + "reason": declared[name]["reason"], + "tracking": declared[name]["tracking"], + } + for name in sorted(matched) + } return TargetRegistry(kept, country=registry.country), receipt diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py index 1c6a5abfff..e77037d938 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_run.py +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -4,6 +4,7 @@ import hmac import json from pathlib import Path +from types import SimpleNamespace import numpy as np import pandas as pd @@ -295,3 +296,179 @@ def test_aggregate_admin_measurement_convention_and_refusals(): stripped.table("household").drop(columns=["electricity_consumption"], inplace=True) with pytest.raises(ValueError, match="household.electricity_consumption"): calibration_run._aggregate_admin_totals(stripped, manifest) + + +def test_seam_pipeline_derives_a_ratified_logbook_scope(): + """The seam appends to the FRS line's chain, not a new unratified one.""" + + logbook_tool = _load_logbook_tool() + + scope = logbook_tool._chain_scope(calibration_run._PIPELINE) + + assert scope == "uk/frs" + assert scope in logbook_tool.DECLARED_SCOPES + + +def _load_logbook_tool(): + import importlib.util + + path = Path(__file__).resolve().parents[3] / "tools" / "logbook.py" + spec = importlib.util.spec_from_file_location("_logbook_tool", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_refusal_records_a_failed_attempt_and_stages_nothing(tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + with pytest.raises(ValueError, match="sha mismatch"): + run_uk_calibration( + paths=paths, + input_sha256="0" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="refused-run", + ) + + # Every terminal disposition is a row; a refusal that left the chain + # silent would hide the attempt entirely. + spooled = sorted((tmp_path / "logbook-spool").rglob("*.json")) + assert spooled, "refusal recorded no Logbook row" + rows = [json.loads(path.read_text()) for path in spooled] + assert [row["disposition"] for row in rows] == ["failed"] + assert rows[0]["pipeline"] == calibration_run._PIPELINE + # The refusal is explained on disk, not only in the row. + receipts = sorted((tmp_path / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.terminal_gate_json.exists() + + +def test_attempt_ids_are_unique_across_reruns_of_one_release( + monkeypatch, tmp_path: Path +): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + source_pins = { + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + } + build_ids = [] + for attempt in ("a", "b"): + run_dir = tmp_path / attempt + run_dir.mkdir() + result = run_uk_calibration( + paths=UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=run_dir / "staged.h5", + diagnostics_json=run_dir / "diagnostics.json", + build_record_json=run_dir / "build_record.json", + terminal_gate_json=run_dir / "terminal_gates.json", + ), + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins=source_pins, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="one-release-id", + ) + build_ids.append(result.build_record["build_id"]) + + # Both the local chain and the store reject a duplicate build id, so one + # release re-run twice must not collide. + assert build_ids[0] != build_ids[1] + assert all(value.startswith("uk-frs-calibration-attempt-") for value in build_ids) + + +def test_verified_ledger_identity_reaches_the_run_evidence(monkeypatch, tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + artifact = SimpleNamespace( + facts_sha256="d" * 64, + fact_row_count=107_550, + manifest_sha256="e" * 64, + manifest={ + "artifact_id": "chronicle-uk-artifact-1cab809", + "profile": "uk-national", + "schema_version": 1, + "unrelated": "not carried", + }, + ) + + result = run_uk_calibration( + paths=UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ), + input_sha256=_sha(input_h5), + ledger_artifact=artifact, + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="ledger-identity", + ) + + ledger = result.build_record["run_config"]["ledger"] + assert ledger["facts_sha256"] == "d" * 64 + assert ledger["manifest_sha256"] == "e" * 64 + assert ledger["fact_row_count"] == 107_550 + assert ledger["manifest"] == { + "artifact_id": "chronicle-uk-artifact-1cab809", + "profile": "uk-national", + "schema_version": 1, + } + # A bare feed carries no manifest, and that absence is recorded rather + # than invented. + assert calibration_run._ledger_provenance(object())["manifest_sha256"] is None diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py index 6c9b254619..78deb7789d 100644 --- a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -83,6 +83,27 @@ def test_driver_refuses_release_candidate_with_each_override(tmp_path: Path): driver._parse_args(_args(tmp_path) + ["--release-candidate", *flag]) +def test_driver_refuses_release_candidate_with_operator_exclusions(tmp_path: Path): + # The scoped battery carries no target-surface gate, so an operator + # register could narrow what a release candidate was measured against + # without anything noticing. + driver = _load_driver_module() + exclusions = tmp_path / "operator.json" + exclusions.write_text("{}", encoding="utf-8") + + with pytest.raises(SystemExit): + driver._parse_args( + _args(tmp_path) + + ["--release-candidate", "--measure-exclusions", str(exclusions)] + ) + + # Without the release-candidate posture the same register is accepted. + parsed = driver._parse_args( + _args(tmp_path) + ["--measure-exclusions", str(exclusions)] + ) + assert parsed.measure_exclusions == exclusions + + def test_driver_refuses_bad_sha_and_path_alias(tmp_path: Path): driver = _load_driver_module() base_args = _args(tmp_path) diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index c80273efcc..ae7bb44b60 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -183,7 +183,9 @@ def test_exclusion_applier_returns_pruned_registry_and_receipt(): ) assert [spec.name for spec in pruned.specs] == ["keep"] - assert receipt == {"drop": "reviewed"} + assert receipt == { + "drop": {"reason": "reviewed", "tracking": "microcosm#623"} + } with pytest.raises(ValueError, match="matched zero"): apply_uk_calibration_measure_exclusions( registry, @@ -289,3 +291,20 @@ def test_unknown_categorical_mapping_refuses_through_the_fence( assert not resolver.knows("household", "family_type") with pytest.raises(KeyError, match="no categorical mapping"): resolver.compute("household", "family_type") + + +def test_exclusion_loader_requires_tracking(tmp_path: Path): + # An exclusion narrows the calibrated target surface; the register has to + # say where each narrowing is being resolved, not only why. + with pytest.raises(ValueError, match="empty tracking"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "untracked.json", + { + "schema_version": 1, + "exclusions": [ + {"name": "a", "reason": "reviewed reason", "tracking": ""} + ], + }, + ) + ) diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py index 8c31b26cf1..c093e3fd50 100644 --- a/tools/calibrate_uk_national_dataset.py +++ b/tools/calibrate_uk_national_dataset.py @@ -160,6 +160,17 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: "--release-candidate is refused with doctrine override flag(s): " + ", ".join(passed_overrides) ) + if args.release_candidate and args.measure_exclusions is not None: + # An exclusion register prunes the compiled target surface before the + # solve, and the calibration-scoped battery does not carry the + # target-surface gate — so an operator-supplied register would be an + # unreviewed narrowing of what a release candidate was measured + # against. Release candidates use the committed register only. + parser.error( + "--release-candidate is refused with --measure-exclusions: a " + "release candidate is measured against the committed target " + "surface, not an operator-supplied one" + ) if ( not args.release_candidate and (_CANONICAL_UK_RELEASE_ID.fullmatch(args.release_id) or args.release_id == _UK_JUNE_RELEASE_ID) From fa858785de6f379cba682646b04ae3344df4feea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:32:47 +0200 Subject: [PATCH 24/26] Make the rule-1 scorer able to score, and say what the parity trio proves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scorer loaded both H5s and called `score_targets` directly. Every packaged UK reference binds a slash-named prepared measure, and calibration deliberately strips those before export — so on a production register it bound nothing and aborted, while its tests stayed green on plain `measure_a` columns no register uses. Both sides now go through the same resolve-inject-materialize route the calibration seam takes, via `prepare_uk_target_frame`, and a skipped target refuses rather than quietly shrinking the surface the two sides are compared on. The receipt was also unbound to what it scored: two arbitrary files were accepted and reported under the default candidate and incumbent labels, and the registry loader bypassed `TargetRegistry.from_json`'s format and content-hash checks. Both artifacts are now verified against required digests before a byte is read and recorded with those digests, the register loads through its validating loader, and the per-target drift table the score block claimed to offer is actually emitted. The seam driver's frozen-register check now compares the same content hash through the same loader, so one artifact serves both tools. `uk_target_surface` claimed candidate and incumbent target surfaces agree exactly. They are not independently sourced: the frozen parity instrument carries input-column shares and no target surface, so the reference side is the declared register and the candidate side the solve's realized diagnostics. That is a real invariant — every declared target bound at its declared period — but it is not the incumbent comparison the note promised, and the note now says so. Gate manifest and spec-fingerprint pins recomputed for the edit. The runbook pointed operators at the June builder, which constructs the calibration stage with neither the measure resolver nor the exclusion register and aborts on the first unmaterializable reference. It now documents the seam driver, whose diagnostics digest is measured from the written bytes rather than declared on the command line, and the scorer invocation that follows it. Two findings reviewed and not reproduced, with the reasons recorded where a reader would look: the person-anchor weight lookup cannot drop a duplicate household id (the kernel validates group-table ids unique, and the CGT clone offsets its clones' ids), and measure injection cannot mutate the source frame (`UKFrameTargetAdapter` copies each table at construction) — the latter now held by a test on the frame itself rather than on the restored output. Reported by @MaxGhenis (findings 1, 2, 6, 7) and @vahid-ahmadi (1, 4). Co-Authored-By: Claude Opus 5 --- docs/uk-national-calibration-runbook-623.md | 76 ++++++-- .../microcosm/build/target_materialization.py | 7 +- .../src/microcosm/build/uk/gates.json | 2 +- .../build/uk_runtime/calibration_run.py | 4 + .../build/uk_runtime/national_build.py | 25 ++- .../build/uk_runtime/national_calibration.py | 39 ++++ .../tests/test_score_uk_national_candidate.py | 125 +++++++++++-- .../tests/test_uk_national_calibration.py | 63 +++++++ .../src/microcosm/data/contract.py | 4 +- tools/calibrate_uk_national_dataset.py | 31 ++-- tools/score_uk_national_candidate.py | 170 +++++++++++++++--- 11 files changed, 473 insertions(+), 73 deletions(-) diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md index 23adfa15c8..d7907f1c7c 100644 --- a/docs/uk-national-calibration-runbook-623.md +++ b/docs/uk-national-calibration-runbook-623.md @@ -11,31 +11,68 @@ Run only after one of these is true: - Maria explicitly names a different base spine for this run. The input posture is a non-certified staging candidate at pre-clone grain, -matching the incumbent published national surface. The staging input must be -SHA-pinned with `--staging-candidate-input-sha256`; release-candidate posture -continues to require the certified input path. +matching the incumbent published national surface. The seam records that +posture itself and marks its artifact `shippable: false`: release certification +is the release-cut producer's job, not calibration's. ## Command shape +Calibration runs through `tools/calibrate_uk_national_dataset.py`. That driver +is the only one that builds the measure resolver and applies the committed +measure-exclusion register, and 187 of the activated references bind model +outputs that no frame carries — so it is the only path on which this target +surface materializes. The June builder +(`tools/build_uk_national_dataset.py`) constructs the calibration stage without +either and aborts on the first unmaterializable reference; it also rebuilds SPI +income onto its input, which a spine artifact already carries. + ```bash -uv run --no-sync python tools/build_uk_national_dataset.py \ +uv run --no-sync python tools/calibrate_uk_national_dataset.py \ --input-h5 data/ukds/acceptance/623-first-calibrated-candidate/input-spine.h5 \ - --staging-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ - --staging-candidate-input-sha256 \ - --release-id populace-uk-2023-frs-k535080 \ - --calibration-diagnostics-sha256 \ - --national-calibration-diagnostics-json data/ukds/acceptance/623-first-calibrated-candidate/calibration_diagnostics.json \ - --frs-raw-dir \ - --spi-tab \ - --hmrc-ods \ - --cgt-ods \ + --input-sha256 \ --ledger-facts \ --ledger-facts-sha256 \ --ledger-manifest-sha256 \ - --terminal-gates-json data/ukds/acceptance/623-first-calibrated-candidate/terminal_gates.json \ - --build-record-json data/ukds/acceptance/623-first-calibrated-candidate/build_record.json + --staging-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ + --diagnostics-json data/ukds/acceptance/623-first-calibrated-candidate/calibration_diagnostics.json \ + --build-record-json data/ukds/acceptance/623-first-calibrated-candidate/build_record.json \ + --terminal-gate-json data/ukds/acceptance/623-first-calibrated-candidate/terminal_gates.json \ + --release-id dev-623-first-calibrated-candidate ``` +The diagnostics digest is measured, not declared: the seam writes the +diagnostics file, hashes its actual bytes, and only then constructs and signs +the terminal gate evidence. There is no `--calibration-diagnostics-sha256` to +supply, and no way for the receipt to claim an identity the file does not have. + +Solve parameters are per-run overrides, not defaults. The campaign settings are +`--epochs 1500 --target-weight-rule family_equal`; each override is validated +through the doctrine dataclass and echoed as an explicit deviation in the +manifest, diagnostics and build record. `--release-candidate` refuses every +override flag, and refuses `--measure-exclusions`, so a release candidate is +always solved under declared doctrine against the committed target surface. + +Signing the terminal gate report needs +`MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` in the environment. + +## Scoring + +Rule 1 is decided by a separate tool run against the staged artifact, so run +identity never depends on the incumbent's bytes: + +```bash +uv run --no-sync python tools/score_uk_national_candidate.py \ + --candidate-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ + --candidate-sha256 \ + --incumbent-h5 \ + --incumbent-sha256 \ + --registry-json data/ukds/acceptance/623-first-calibrated-candidate/frozen-register.json \ + --output-json data/ukds/acceptance/623-first-calibrated-candidate/score_vs_enhanced_frs.json +``` + +Both artifacts are verified against the supplied digests before they are read, +and both sides are scored on the same frozen register. + ## Evidence directory `data/ukds/acceptance/623-first-calibrated-candidate/` should contain: @@ -43,10 +80,15 @@ uv run --no-sync python tools/build_uk_national_dataset.py \ - `build_record.json` - `terminal_gates.json` - `calibration_diagnostics.json` -- calibration diagnostics SHA-256 receipt - `score_vs_enhanced_frs.json` -- twin-build payload identity receipt +- `logbook-spool/` (one row for the attempt, whatever its disposition) Acceptance follows #578: the candidate must not regress incumbent battery observables, and the score block decides rule 1. A rule-1 loss is evidence for #686/#736, not a threshold-edit instruction. + +The calibration battery is scoped to the calibration-relevant gates; the +spine-construction and imputation gates are out of scope here and are listed in +the report as scope exclusions with their rationale. A publishable +certification combines this with the spine build's own battery, which is +release-cut work (#757). diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index c9042df5c2..79fdfb228c 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -132,7 +132,12 @@ def resolve_target_measures( ``adapter_factory`` must return a fresh adapter over the same frame each round. Previously resolved values are injected into those probe adapters' table copies only, then the shared target-binding interpreter is retried. - The source frame is never mutated. + + The source frame is never mutated — the guarantee rests on the adapter + copying each entity table at construction (``UKFrameTargetAdapter`` + does), not on the later ``restore``. Any adapter passed here must copy; + ``test_uk_national_calibration.py`` holds that property directly, on the + source frame rather than on the restored output. """ contract = _measure_resolution_contract( diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index e9a1101bbf..51d48e864f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -304,7 +304,7 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": {}, - "notes": "Candidate and incumbent calibration target surfaces must agree exactly; the gate takes no tunable parameters." + "notes": "The solve's realized target surface must equal the declared register exactly, at name-and-period grain; the gate takes no tunable parameters. The reference side is the declared register, not an incumbent-derived surface \u2014 the frozen parity instrument carries input-column shares only \u2014 so this proves that every declared target was bound at its declared period, not agreement with the incumbent." }, { "id": "uk_target_fit", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 3f124bb1d4..745a245655 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -504,6 +504,10 @@ def _aggregate_admin_totals( if entity == "household": weights = household_weights elif entity == "person": + # `household_id` is a group-table id column, which the Frame + # kernel validates unique before any frame exists, and the CGT + # clone stage offsets its clones' ids for exactly that reason — + # so this lookup cannot silently drop a duplicate key. person = frame.table("person") lookup = dict( zip( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index 52eb261c0d..0c479d29cd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -710,12 +710,25 @@ def _stage_parity_evidence( frame: Frame, parity_reference: EfrsParityReference | None, ) -> object | None: - """Build the parity-trio evidence from independently sourced sides. - - The candidate side comes from the staged frame and the solve diagnostics; - the reference side comes from the frozen parity instrument and the - declared registry. The two sides must never alias each other — a copied - reference would make the trio pass by construction. + """Build the parity-trio evidence, side by side, never aliased. + + Two different comparisons travel in one object, and they are not equally + strong: + + * The **column** surfaces are independently sourced — the candidate's from + the staged frame, the reference's from the frozen parity instrument's + declared input entities. + * The **target** surfaces are not. The parity instrument carries source + identity and incumbent input-column shares only; it holds no incumbent + target surface. So the reference side is the declared registry and the + candidate side is the solve's realized diagnostics — which proves that + the solve bound every declared target at the declared period, and does + **not** prove agreement with any incumbent-derived surface. + + Neither side is ever copied from the other; a copied reference would make + the trio pass by construction. Sourcing the target side from an + incumbent-bound instrument needs an instrument that carries one, which is + release-cut work (#757). """ for stage in stages: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index 15028a200c..4ff938bb09 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -394,3 +394,42 @@ def restore(self, calibrated: Frame) -> Frame: mass_log=calibrated.mass_log, metadata=calibrated.metadata, ) + + +def prepare_uk_target_frame( + frame: Frame, + registry: TargetRegistry, + *, + period: int | str, + measure_resolver: object | None, +) -> tuple[Frame, Mapping[str, Any] | None]: + """Materialize a registry's measures onto a frame, for scoring. + + The same resolve-inject-materialize route the calibration stage takes, + without the solve: scoring a UK register against a raw exported H5 cannot + work, because every packaged reference binds a slash-named prepared + measure that calibration deliberately strips before export. A skipped + target refuses rather than quietly shrinking the surface both sides are + compared on. + """ + + resolution = None + if measure_resolver is not None: + resolution = resolve_target_measures( + lambda: _CalibrationFrameAdapter(frame), + registry, + measure_resolver, + period=period, + ) + adapter = _CalibrationFrameAdapter(frame) + if resolution is not None: + _inject_measure_inputs(adapter, resolution.measure_inputs) + materialized = materialize_uk_ledger_targets(adapter, registry, period=period) + if materialized.skipped: + raise RuntimeError( + "target measures did not materialize for scoring: " + f"{[skip.__dict__ for skip in materialized.skipped]}." + ) + return adapter.prepared_frame(), ( + None if resolution is None else dict(resolution.receipt) + ) diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py index 9cd79ca8d2..657667506e 100644 --- a/packages/microcosm-build/tests/test_score_uk_national_candidate.py +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -12,6 +12,7 @@ from microcosm.frame import WeightKind from tools.score_uk_national_candidate import ( _load_registry, + _sha256_file, main, score_uk_national_candidate, ) @@ -79,9 +80,30 @@ def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: score = score_uk_national_candidate( candidate_h5=candidate, incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), target_registry=_registry(), + calibration_year=2025, ) + assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) + assert score["artifacts"]["incumbent"]["sha256"] == _sha256_file(incumbent) + assert score["target_drift"] == [ + { + "target": "target_a@0", + "family": "family_a", + "candidate_relative_error": pytest.approx(0.0), + "incumbent_relative_error": pytest.approx(-0.2), + "winner": "candidate", + }, + { + "target": "target_b@0", + "family": "family_b", + "candidate_relative_error": pytest.approx(-0.5), + "incumbent_relative_error": pytest.approx(0.0), + "winner": "incumbent", + }, + ] assert score["candidate_full_loss"] == pytest.approx(0.25) assert score["incumbent_full_loss"] == pytest.approx(0.1) assert score["candidate_train_loss"] == score["candidate_full_loss"] @@ -115,29 +137,26 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: output_json = tmp_path / "score.json" _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) - registry_json.write_text( - json.dumps( - { - "country": "uk", - "specs": [ - asdict for asdict in (_spec.__dict__ for _spec in _registry().specs) - ], - } - ), - encoding="utf-8", - ) + _registry().to_json(registry_json) assert ( main( [ "--candidate-h5", str(candidate), + "--candidate-sha256", + _sha256_file(candidate), "--incumbent-h5", str(incumbent), - "--target-registry-json", + "--incumbent-sha256", + _sha256_file(incumbent), + "--registry-json", str(registry_json), "--output-json", str(output_json), + "--calibration-year", + "2025", + "--no-measure-resolution", ] ) == 0 @@ -154,3 +173,85 @@ def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: assert isinstance(score["candidate_train_loss"], float) assert isinstance(score["incumbent_full_loss"], float) assert _load_registry(registry_json).version == _registry().version + + +def test_scorer_refuses_artifacts_that_do_not_match_their_pins(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + + with pytest.raises(ValueError, match="candidate artifact sha mismatch"): + score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256="0" * 64, + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + ) + + +def test_scorer_refuses_a_register_that_is_not_a_validated_artifact(tmp_path) -> None: + # The loose loader accepted any object with a specs list, so a + # hand-edited surface could decide rule 1 unnoticed. + loose = tmp_path / "loose.json" + loose.write_text( + json.dumps( + { + "country": "uk", + "specs": [spec.__dict__ for spec in _registry().specs], + } + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="not a microcosm target registry artifact"): + _load_registry(loose) + + tampered = tmp_path / "tampered.json" + _registry().to_json(tampered) + payload = json.loads(tampered.read_text(encoding="utf-8")) + payload["specs"][0]["value"] = 999.0 + tampered.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="corrupt"): + _load_registry(tampered) + + +def test_scorer_refuses_prepared_measures_it_cannot_materialize(tmp_path) -> None: + """A production-shaped register cannot be scored off raw exported columns. + + Every packaged UK reference binds a slash-named prepared measure, and + calibration strips those before export — so scoring without measure + resolution has to refuse rather than report on whatever happens to bind. + """ + + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + production_shaped = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=20.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + with pytest.raises(RuntimeError, match="did not materialize for scoring"): + score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=production_shaped, + calibration_year=2025, + ) diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index 4275bbf320..bd547a8e13 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -798,3 +798,66 @@ def capturing_calibrate(*args, **kwargs): else: assert weights is not None assert weights.tolist() == expected + + +def test_measure_resolution_never_touches_the_source_frame() -> None: + """Injection lands on adapter table copies, not on the caller's frame. + + A test that inspects the frame after ``restore`` would pass either way, + so this one holds the property where it actually lives: the source frame + gains no column while the resolution loop probes round after round. + """ + + from microcosm.build.target_materialization import resolve_target_measures + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + frame = _materialization_binding_frame() + before = { + entity: list(frame.table(entity).columns) for entity in frame.entities + } + + class ProbeResolver: + contract_targets = { + "probe.target": { + "bindings": { + "policyengine": { + "from_entity": "household", + "value_variable": "probe_measure", + } + } + } + } + + def knows(self, entity, variable): + return (entity, variable) == ("household", "probe_measure") + + def compute(self, entity, variable): + return np.array([1.0, 2.0, 3.0]), "probe" + + def receipt(self): + return {"provider": "probe"} + + registry = TargetRegistry( + [ + TargetSpec( + name="probe.target", + entity="household", + measure="probe/measure", + value=6.0, + source="test", + metadata={"contract_target_id": "probe.target"}, + ) + ], + country="uk", + ) + + resolution = resolve_target_measures( + lambda: UKFrameTargetAdapter(frame), + registry, + ProbeResolver(), + period=2025, + ) + + assert ("household", "probe_measure") in resolution.measure_inputs + for entity in frame.entities: + assert list(frame.table(entity).columns) == before[entity] diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index ad3ec35066..379d4eda47 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -376,10 +376,10 @@ "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" + "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" + "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py index c093e3fd50..815c8f3f25 100644 --- a/tools/calibrate_uk_national_dataset.py +++ b/tools/calibrate_uk_national_dataset.py @@ -33,6 +33,7 @@ load_uk_calibration_measure_exclusions, ) from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides +from microcosm.calibrate import TargetRegistry _SHA256 = re.compile(r"[0-9a-f]{64}") _CANONICAL_UK_RELEASE_ID = re.compile(r"populace-uk-[1-9][0-9]*-[a-z0-9_]+-k[1-9][0-9]*") @@ -231,24 +232,26 @@ def _paths_alias(left: Path, right: Path) -> bool: def _compare_frozen_register(path: Path | None, registry: Any) -> None: + """Refuse when the compiled register is not the frozen scoring surface. + + Both sides are compared on ``TargetRegistry.version`` — the registry's own + content hash — through the validating loader, so the same artifact serves + this check and ``tools/score_uk_national_candidate.py``, and incidental + formatting cannot make two identical registers look different. + """ + if path is None: return - payload = { - "country": registry.country, - "version": registry.version, - "specs": [ - {field: value for field, value in spec.__dict__.items() if value is not None} - for spec in registry.specs - ], - } - derived = hashlib.sha256( - json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") - ).hexdigest() - frozen = hashlib.sha256(path.read_bytes()).hexdigest() - if derived != frozen: + try: + frozen = TargetRegistry.from_json(path) + except ValueError as error: + raise SystemExit( + f"error: frozen scoring register is unusable: {error}" + ) from error + if frozen.version != registry.version: raise SystemExit( "re-derived register differs from the frozen scoring register: " - f"{derived} vs {frozen}" + f"{registry.version} vs {frozen.version}" ) diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py index 620a13e17c..451b4af233 100644 --- a/tools/score_uk_national_candidate.py +++ b/tools/score_uk_national_candidate.py @@ -3,12 +3,15 @@ from __future__ import annotations import argparse +import hashlib import json +from collections.abc import Callable from pathlib import Path from typing import Any +from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame from microcosm.build.uk_runtime.national_frame import load_uk_national_frame -from microcosm.calibrate import TargetRegistry, TargetSpec, score_targets +from microcosm.calibrate import TargetRegistry, score_targets UK_SCORE_LOSS_CAP = 10.0 UK_SCORE_HOLDOUT_BASIS = "none_declared" @@ -37,16 +40,32 @@ def score_uk_national_candidate( *, candidate_h5: str | Path, incumbent_h5: str | Path, + candidate_sha256: str, + incumbent_sha256: str, target_registry: TargetRegistry, + calibration_year: int, + measure_resolver_factory: Callable[[Path, Any], Any] | None = None, candidate_label: str = "populace_uk_2023", incumbent_label: str = "enhanced_frs_2024_25", ) -> dict[str, Any]: - """Return the #578 rule-1 score block on a shared target registry.""" + """Return the #578 rule-1 score block on a shared target registry. + + Both artifacts are verified against their declared digests before a byte + is read, and both sides are materialized through the same route the + calibration seam uses, so the score names the artifacts it actually + measured rather than two labels supplied on the command line. + """ if target_registry.country != "uk" or not target_registry.specs: raise ValueError("UK candidate scoring requires a non-empty UK registry.") - candidate_frame, _candidate_provenance = load_uk_national_frame(candidate_h5) - incumbent_frame, _incumbent_provenance = load_uk_national_frame(incumbent_h5) + candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) + incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) + candidate_frame, candidate_resolution = _scored_frame( + candidate_h5, target_registry, calibration_year, measure_resolver_factory + ) + incumbent_frame, incumbent_resolution = _scored_frame( + incumbent_h5, target_registry, calibration_year, measure_resolver_factory + ) candidate = score_targets( candidate_frame, target_registry.to_target_set(), @@ -88,9 +107,16 @@ def score_uk_national_candidate( "n_specs": len(target_registry), }, "artifacts": { - "candidate": candidate_label, - "incumbent": incumbent_label, + "candidate": {"label": candidate_label, **candidate_pin}, + "incumbent": {"label": incumbent_label, **incumbent_pin}, + }, + "measure_resolution": { + "candidate": candidate_resolution, + "incumbent": incumbent_resolution, }, + "target_drift": _target_drift( + target_registry, candidate_errors, incumbent_errors + ), "signed_asymmetries": [ { "id": "incumbent_own_registry", @@ -162,21 +188,96 @@ def _target_wins_by_family( def _load_registry(path: str | Path) -> TargetRegistry: - payload = json.loads(Path(path).read_text(encoding="utf-8")) - if isinstance(payload, dict): - specs_payload = payload.get("specs") - country = str(payload.get("country", "uk")) - else: - specs_payload = payload - country = "uk" - if not isinstance(specs_payload, list): - raise ValueError("target registry JSON must contain a specs list.") - return TargetRegistry( - (TargetSpec(**item) for item in specs_payload), - country=country, + """Load the frozen register through its own validating loader. + + ``TargetRegistry.from_json`` checks the format revision and re-derives the + content hash, so a hand-edited or drifted register refuses here instead of + silently deciding rule 1 on a surface nobody reviewed. + """ + + return TargetRegistry.from_json(path) + + +def _sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_artifact(role: str, path: str | Path, expected: str) -> dict[str, object]: + measured = _sha256_file(path) + if measured != expected: + raise ValueError( + f"{role} artifact sha mismatch: measured {measured}, pinned {expected}" + ) + return { + "path": str(path), + "sha256": measured, + "size_bytes": Path(path).stat().st_size, + } + + +def _scored_frame( + h5_path: str | Path, + registry: TargetRegistry, + calibration_year: int, + factory: Callable[[Path, Any], Any] | None, +) -> tuple[Any, Any]: + frame, _provenance = load_uk_national_frame(h5_path) + resolver = None if factory is None else factory(Path(h5_path), frame) + return prepare_uk_target_frame( + frame, registry, period=calibration_year, measure_resolver=resolver ) +def _default_measure_resolver_factory(scratch_dir: Path, year: int): + def build(h5_path: Path, frame: Any): + from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver + + return UKMeasureResolver( + simulation_source=h5_path, + scratch_dir=scratch_dir, + year=year, + frame=frame, + ) + + return build + + +def _target_drift( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> list[dict[str, object]]: + """Per-target relative errors, the auditable half of the score block.""" + + families = {spec.to_target().row_name: spec.family for spec in registry.specs} + rows = [] + for name in sorted(candidate_errors): + candidate = candidate_errors[name] + incumbent = incumbent_errors[name] + # Relative errors are signed; the win is decided on magnitude, the + # same rule the aggregate counts use. + rows.append( + { + "target": name, + "family": families.get(name), + "candidate_relative_error": candidate, + "incumbent_relative_error": incumbent, + "winner": ( + "candidate" + if abs(candidate) < abs(incumbent) + else "incumbent" + if abs(incumbent) < abs(candidate) + else "tie" + ), + } + ) + return rows + + def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: output = Path(path) output.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") @@ -186,14 +287,43 @@ def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--candidate-sha256", required=True) parser.add_argument("--incumbent-h5", required=True, type=Path) - parser.add_argument("--target-registry-json", required=True, type=Path) + parser.add_argument("--incumbent-sha256", required=True) + parser.add_argument("--registry-json", required=True, type=Path) parser.add_argument("--output-json", required=True, type=Path) + parser.add_argument("--calibration-year", type=int) + parser.add_argument( + "--no-measure-resolution", + action="store_true", + help=( + "Score frames whose measures are already columns. Every packaged " + "UK reference binds a prepared measure, so this is for fixtures " + "only; a production register refuses on the first skipped target." + ), + ) args = parser.parse_args(argv) + registry = _load_registry(args.registry_json) + calibration_year = args.calibration_year + if calibration_year is None: + from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + + calibration_year = load_uk_frs_release().calibration_year + factory = ( + None + if args.no_measure_resolution + else _default_measure_resolver_factory( + args.output_json.parent, int(calibration_year) + ) + ) score = score_uk_national_candidate( candidate_h5=args.candidate_h5, incumbent_h5=args.incumbent_h5, - target_registry=_load_registry(args.target_registry_json), + candidate_sha256=args.candidate_sha256, + incumbent_sha256=args.incumbent_sha256, + target_registry=registry, + calibration_year=int(calibration_year), + measure_resolver_factory=factory, ) _write_json(args.output_json, {"score_vs_enhanced_frs": score}) return 0 From 46630e714e703c8578f67956d446c1ffb0f95558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:01:57 +0200 Subject: [PATCH 25/26] Mirror the recomputed gate pins in the contract test's own copy The contract test declares its pins independently and a lockstep test holds the two equal, so a re-cut moves both. Caught by running the data shard's UK slice rather than only the build shard's pin suite. Co-Authored-By: Claude Opus 5 --- changelog.d/623-uk-calibration-review-fixes.fixed.md | 7 +++++++ changelog.d/623-uk-target-surface-gate-note.changed.md | 4 ++++ .../src/microcosm/build/uk_runtime/calibration_run.py | 6 +++++- packages/microcosm-data/tests/test_contract.py | 4 ++-- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 changelog.d/623-uk-calibration-review-fixes.fixed.md create mode 100644 changelog.d/623-uk-target-surface-gate-note.changed.md diff --git a/changelog.d/623-uk-calibration-review-fixes.fixed.md b/changelog.d/623-uk-calibration-review-fixes.fixed.md new file mode 100644 index 0000000000..bc8d55abec --- /dev/null +++ b/changelog.d/623-uk-calibration-review-fixes.fixed.md @@ -0,0 +1,7 @@ +Count two-child-limit child targets as counts rather than household +indicators, attribute band edges to their own groupby dimension, make the +measure-provider fence answer only for routes it can take, put calibration +attempts on the ratified `uk/frs` Logbook chain with failure rows and unique +attempt ids, seal the verified Ledger identity into the run receipts, refuse +operator-supplied measure exclusions on release candidates, and let the rule-1 +scorer materialize and authenticate what it scores. diff --git a/changelog.d/623-uk-target-surface-gate-note.changed.md b/changelog.d/623-uk-target-surface-gate-note.changed.md new file mode 100644 index 0000000000..0460eb56db --- /dev/null +++ b/changelog.d/623-uk-target-surface-gate-note.changed.md @@ -0,0 +1,4 @@ +State what the UK `uk_target_surface` gate proves — the solve's realized +target surface equals the declared register at name-and-period grain — rather +than claiming an incumbent comparison the frozen parity instrument cannot +source. Gate manifest and spec-fingerprint pins recomputed. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 745a245655..6e5e947b04 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -254,7 +254,11 @@ def _record_failed_attempt( rung="f100", seed=seed, code_pin=code_pin, - disposition="failed", + # An operator interrupt is a discarded attempt, not a failed one; the + # row says which so the chain reads honestly. + disposition=( + "discarded" if isinstance(error, KeyboardInterrupt) else "failed" + ), predecessor=predecessor, spool_dir=spool_dir, ) diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index a418a586cf..065832b827 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -151,10 +151,10 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" + "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" + "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" From 45d6583b4df6188e81304980fdabc5e3b62a6d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:17:35 +0200 Subject: [PATCH 26/26] Teach the reference generator the two new binding keys The generator validates every contract binding against its own closed vocabulary, so declaring `value_reduction` and `band_filter_dimension` in the contract without widening it there refuses regeneration outright. Co-Authored-By: Claude Opus 5 --- tools/generate_uk_target_references.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/generate_uk_target_references.py b/tools/generate_uk_target_references.py index 84b91fe0e3..1de3b4a7db 100644 --- a/tools/generate_uk_target_references.py +++ b/tools/generate_uk_target_references.py @@ -34,6 +34,7 @@ { "affected_flag_variable", "band", + "band_filter_dimension", "band_period_factor", "count_of", "filters", @@ -54,6 +55,7 @@ "source_lines", "threshold_price_base_year", "value_expression", + "value_reduction", "value_variable", "zeroed_input", }