From 5c1908df42bc039b308f43f009be171a611ab574 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 14 Oct 2026 15:39:17 +0100 Subject: [PATCH] Package UK area lists and certify UK regional releases Ship the 650 constituencies (2024) and 361 local authorities (April 2023, with Ards and North Down and real hexes for the 2023 unitaries) with the package. Area regions and geography output labels read them instead of GCS; the opt-in registry previously came back empty because its GCS helper no longer exists in policyengine-core. Extend --regional-manifest-uri to the UK: a Microcosm local-area release routes constituency and local-authority regions to its file, and --regional-release-kind area_coded_national routes them to the enhanced FRS with its weight matrices at their own revision, for a national-first switch. Certification records regional_release_kind, checks the regional release's own model compatibility, HEAD-checks carried and Microcosm UK artifacts, refuses re-certification that would drop regional routing, and names a build by its cut tag. Area regions take the routed dataset and join the default registry only for a local-area release. Add the Universal Credit coverage floor check from #491. Refs #553 Co-Authored-By: Claude Opus 5.5 --- changelog.d/553.added.md | 1 + changelog.d/553.fixed.md | 1 + docs/engineering/skills/data-certification.md | 92 ++- docs/regions.md | 60 +- docs/release-bundles.md | 7 + scripts/bundle.py | 23 +- scripts/certify_data_release.py | 31 +- scripts/check_uk_uc_floor.py | 57 ++ src/policyengine/countries/uk/regions.py | 131 ++-- .../data/qualification/uk_uc_floor.json | 32 + src/policyengine/data/uk_geography/SOURCES.md | 38 + .../data/uk_geography/constituencies_2024.csv | 651 ++++++++++++++++++ .../uk_geography/local_authorities_2023.csv | 362 ++++++++++ src/policyengine/data/uk_geography_assets.py | 24 + .../outputs/constituency_impact.py | 11 +- .../outputs/local_authority_impact.py | 12 +- .../outputs/uk_geography_impact.py | 14 +- src/policyengine/provenance/certification.py | 414 +++++++++-- src/policyengine/provenance/manifest.py | 37 +- src/policyengine/provenance/uk_uc_floor.py | 159 +++++ tests/test_certify_data_release.py | 495 ++++++++++++- tests/test_constituency_impact.py | 31 +- tests/test_local_authority_impact.py | 37 +- tests/test_release_manifests.py | 35 + tests/test_uk_area_lists.py | 114 +++ tests/test_uk_regions.py | 141 ++++ tests/test_uk_uc_floor.py | 84 +++ 27 files changed, 2881 insertions(+), 213 deletions(-) create mode 100644 changelog.d/553.added.md create mode 100644 changelog.d/553.fixed.md create mode 100644 scripts/check_uk_uc_floor.py create mode 100644 src/policyengine/data/qualification/uk_uc_floor.json create mode 100644 src/policyengine/data/uk_geography/SOURCES.md create mode 100644 src/policyengine/data/uk_geography/constituencies_2024.csv create mode 100644 src/policyengine/data/uk_geography/local_authorities_2023.csv create mode 100644 src/policyengine/provenance/uk_uc_floor.py create mode 100644 tests/test_uk_area_lists.py create mode 100644 tests/test_uk_uc_floor.py diff --git a/changelog.d/553.added.md b/changelog.d/553.added.md new file mode 100644 index 00000000..d2e90df7 --- /dev/null +++ b/changelog.d/553.added.md @@ -0,0 +1 @@ +Ship the UK constituency (2024, 650 including Northern Ireland) and local-authority (April 2023, 361 including Ards and North Down) lists with the package, so area regions and geography output labels no longer depend on GCS. Certify a UK regional release with `certify-data --regional-manifest-uri`: a Microcosm local-area release, or, for a national-first switch, the enhanced FRS with its area weight matrices (`--regional-release-kind area_coded_national`). Constituency and local-authority regions load the routed dataset and join the default registry when it is a local-area release. Add the Universal Credit coverage floor check (`scripts/check_uk_uc_floor.py`) for UK default switches. diff --git a/changelog.d/553.fixed.md b/changelog.d/553.fixed.md new file mode 100644 index 00000000..64cac112 --- /dev/null +++ b/changelog.d/553.fixed.md @@ -0,0 +1 @@ +Constituency and local-authority regions were empty when requested, because their GCS download helper no longer exists in policyengine-core. Certification now names a build by its cut tag when the release has one, refuses a regional release that does not serve the pinned model version, and refuses a re-certification that would silently drop a bundle's regional routing (`--drop-regional-release` removes it on purpose). diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 94211889..24559358 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -23,9 +23,75 @@ the checkout is dirty). ```bash python scripts/bundle.py certify-data --country uk --data-producer populace \ - --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" ``` +Pin the cut tag (`@`), never `main`: each cut of a Microcosm release +line rewrites that line's `releases//` directory, for example +`releases/microcosm-uk-2024-25-national/`. Microcosm records the line name as +`build.build_id` on every cut and the cut in `build.cut_tag`; certification +uses the cut tag as the build id (`data_releases..version`, the +certification's `data_build_id` and the changelog fragment name), so two cuts +of one line never look identical. + +### UK constituency and local-authority regions + +UK constituency and local-authority regions row-filter the dataset a regional +release routes them to (`region_datasets.constituency` and +`region_datasets.local_authority`), and the national dataset when nothing is +routed. policyengine-sim-api accepts a UK default without area codes +(`constituency_code_oa`, `la_code_oa`) only when both are routed. Certify a +Microcosm national release with one of: + +- **A Microcosm local-area release (ship together).** The release declares + `dataset_role: non_default_local_area`, as for the US below. + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" \ + --model-version "" + ``` + + This records `regional_release_kind: local_area`, and the default UK region + registry then includes all 650 constituencies and 361 local authorities. + +- **The enhanced FRS (national first).** When the national release is ready + before a local one, route the area regions to a certified policyengine-uk-data + release whose households carry area codes: + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "hf://model/policyengine/policyengine-uk-data-private@/releases//release_manifest.json" \ + --regional-release-kind area_coded_national \ + --model-version "" + ``` + + Certification carries that release's national dataset + (`enhanced_frs_2024_25`) and its `weights` artifacts (the constituency and + local-authority matrices and `local_geography_weights`) at their own repo + and revision, records `regional_release_kind: area_coded_national`, and + leaves area regions out of the default registry: the API keeps serving them + from the matrices. The enhanced FRS release must itself claim the pinned + policyengine-uk version (built with it or listed in + `compatible_model_packages`); otherwise policyengine-uk-data must publish a + claim first. Its `frs_2024_25` is not carried, so keep it loadable with a + `dataset_overlays.uk` entry. + +Either way, certification checks the regional release's own model +compatibility and HEAD-checks every artifact it carries. + +### Re-certifying a routed bundle + +Certification rewrites `data_releases.` from the manifests it is +given. While the bundle routes regions (`regional_source_manifest_uri` is set), +a run without `--regional-manifest-uri` is refused; pass the current regional +release again (or its replacement), or `--drop-regional-release` to remove the +routing on purpose. + +### US + For US Populace certification, certify the Populace release manifest directly: ```bash @@ -109,8 +175,10 @@ Hard failures (certification refuses): missing national default dataset, default dataset absent from artifacts, any artifact without a revision pin, unreachable certified dataset, missing required supplemental release files (for example Populace-US `us_source_coverage.json`), a malformed shared -local-area release or malformed legacy per-state artifacts when -`--regional-manifest-uri` is used, unknown country. +local-area release, area-coded national release or legacy per-state artifacts +when `--regional-manifest-uri` is used, a regional release that does not serve +the pinned model version, an unreachable regional artifact, a re-certification +that would silently drop regional routing, unknown country. Certification gate: the model version must either exactly match the build-time model (`compatibility_basis: built_with_model_package`) or be @@ -123,6 +191,24 @@ refused: a new data build or a published compatibility claim is required. Warnings (recorded, not blocking): artifacts without sha256, and the publisher-claim basis above. +## Acceptance evidence for a UK default switch + +A PR that changes the UK default dataset carries, besides the UK candidate +qualification gates, the Universal Credit coverage floor from #491: benefit +units with Universal Credit and Universal Credit spend in Great Britain within +10% of DWP's and the OBR's figures. + +```bash +python scripts/check_uk_uc_floor.py --dataset \ + --year 2025 --output uc_floor.json +``` + +The benchmarks and their sources are in +`src/policyengine/data/qualification/uk_uc_floor.json`. The receipt holds +weighted national totals only. Before quoting any other figure from UK data +in a PR, apply the UKDS output rules: no unit-record values, no cells under +10 unweighted cases, no differencing. + ## Release runbooks Concrete, fill-in-the-id runbooks that replay a specific certification live diff --git a/docs/regions.md b/docs/regions.md index 443c3cc7..de432fe9 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -91,13 +91,44 @@ england = pe.uk.model.get_region("country/england") Pass the registry entry's `scoping_strategy` to `Simulation` when using a national UK dataset. +## UK constituency and local-authority regions + +The package ships the area lists: the 650 parliamentary constituencies (2024 +boundaries, including Northern Ireland) and the 361 local authority districts +(April 2023, including Ards and North Down, `N09000011`), each with a name and +a hex-map position (`src/policyengine/data/uk_geography/`, sources in its +`SOURCES.md`). Building these regions needs no network access. + +Constituency regions (`constituency/`) row-filter `constituency_code_oa` +and local-authority regions (`local_authority/`) row-filter `la_code_oa`. +Their `dataset_path` is the dataset the certified bundle routes them to +(`data_releases.uk.region_datasets`), or `None`, meaning the national dataset, +when nothing is routed: + +| `regional_release_kind` | Area regions load | In the default registry | +| --- | --- | --- | +| none (today) | the national dataset | no | +| `area_coded_national` | another release's area-coded national dataset (the enhanced FRS), while the default is a national file without area codes | no | +| `local_area` | a certified local-area release | yes | + +Pass `include_constituencies=True` and `include_local_authorities=True` to +`build_uk_region_registry` to include them regardless. + +```python +from policyengine.countries.uk.regions import build_uk_region_registry + +registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True +) +registry.get("local_authority/N09000011").label # "Ards and North Down" +``` + ## UK parliamentary constituencies Constituency-level impacts group household output rows by the longwise -`constituency_code_oa` column carried by the dataset. If the constituency CSV is -available locally or from the PolicyEngine UK GCS bucket, PolicyEngine uses it -to attach names and map coordinates; otherwise results still compute and use -the code as the label. +`constituency_code_oa` column carried by the dataset. Names and map coordinates +come from the packaged constituency list; codes it does not list use the code +as the label. ```python from policyengine.outputs import compute_uk_constituency_impacts @@ -109,10 +140,9 @@ impacts = compute_uk_constituency_impacts( impacts.constituency_results ``` -To force a specific metadata file, pass `constituency_csv_path`. To avoid -downloading metadata and fall back to code-only labels, pass -`download_missing_assets=False`. The legacy `weight_matrix_path` and `year` -arguments are accepted for backward compatibility but ignored. +To use a different metadata file, pass `constituency_csv_path`. The legacy +`weight_matrix_path`, `year` and `download_missing_assets` arguments are +accepted for backward compatibility but ignored. ## UK local authorities @@ -126,11 +156,15 @@ impacts = compute_uk_local_authority_impacts( impacts.local_authority_results ``` -Local-authority impacts follow the same longwise pattern using `la_code_oa`. -Pass `local_authority_csv_path` to use a specific metadata CSV, or -`download_missing_assets=False` to skip metadata download and use code-only -labels. The legacy `weight_matrix_path` and `year` arguments are accepted for -backward compatibility but ignored. +Local-authority impacts follow the same longwise pattern using `la_code_oa`, +labelled from the packaged April 2023 list. Pass `local_authority_csv_path` to +use a specific metadata CSV. The legacy `weight_matrix_path`, `year` and +`download_missing_assets` arguments are accepted for backward compatibility +but ignored. + +Both impacts need simulations on a dataset whose households carry the area +codes. When the national default does not, run them on the dataset the bundle +routes the area regions to. ## Region registries diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 6bc65c3a..2fea93c1 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -249,6 +249,13 @@ path. The national `default_dataset` remains unchanged. Ordinary bundle installation downloads only that national default; a regional runtime is responsible for materializing the additional certified dataset. +`data_releases.{country}.regional_release_kind` records how certification +routed those region types: `local_area` for a non-default local-area release, +and `area_coded_national` (UK only) for another release's national dataset +whose households carry area codes, certified together with that release's +area weight matrices at their own repo and revision. It is absent when nothing +is routed, or for a legacy US per-state release. + Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's primary `data_package`. This prevents runtime code from guessing how a diff --git a/scripts/bundle.py b/scripts/bundle.py index 121ab8e2..5f00bfe7 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -44,6 +44,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-artifact-prefix", args.regional_artifact_prefix]) if args.regional_path_template: argv.extend(["--regional-path-template", args.regional_path_template]) + if args.regional_release_kind: + argv.extend(["--regional-release-kind", args.regional_release_kind]) + if args.drop_regional_release: + argv.append("--drop-regional-release") if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -196,10 +200,25 @@ def _parser() -> argparse.ArgumentParser: certify.add_argument( "--regional-manifest-uri", help=( - "Optional regional release_manifest.json URI to merge into US " - "Populace certification." + "Regional release_manifest.json URI whose dataset serves the local " + "region types (US states and districts, UK constituencies and local " + "authorities)." ), ) + certify.add_argument( + "--regional-release-kind", + choices=["local_area", "area_coded_national"], + help=( + "How to read the regional release. Defaults to local_area; " + "area_coded_national (UK only) routes area regions to another " + "national release's area-coded dataset and weight matrices." + ), + ) + certify.add_argument( + "--drop-regional-release", + action="store_true", + help="Remove the bundle's current regional routing on re-certification.", + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 8842adbb..75962159 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -8,7 +8,11 @@ Usage:: python scripts/bundle.py certify-data --country uk --data-producer populace \\ - --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" + +Pin the cut tag, never a branch: each cut rewrites ``releases//``. +Add ``--regional-manifest-uri`` to route local regions to a regional release +(see docs/engineering/skills/data-certification.md). Private data (UK) requires ``HUGGING_FACE_TOKEN`` or ``HF_TOKEN`` in the environment. After running: commit the changed bundle manifest, pyproject.toml, @@ -53,10 +57,29 @@ def main(argv=None) -> int: "--regional-manifest-uri", default=None, help=( - "Optional regional data release manifest to merge into US Populace " - "certification." + "Regional release_manifest.json URI whose dataset serves the local " + "region types (US states and districts, UK constituencies and local " + "authorities). Required on re-certification while the bundle routes " + "regions, unless --drop-regional-release is given." + ), + ) + parser.add_argument( + "--regional-release-kind", + default="local_area", + choices=["local_area", "area_coded_national"], + help=( + "local_area: a non-default local-area release. " + "area_coded_national (UK only): another certified national release " + "whose households carry area codes, such as the enhanced FRS; its " + "national dataset and area weight matrices serve constituency and " + "local-authority regions." ), ) + parser.add_argument( + "--drop-regional-release", + action="store_true", + help="Remove the bundle's current regional routing on re-certification.", + ) parser.add_argument( "--regional-artifact-prefix", default="states/", @@ -102,6 +125,8 @@ def main(argv=None) -> int: regional_manifest_uri=args.regional_manifest_uri, regional_artifact_prefix=args.regional_artifact_prefix, regional_path_template=args.regional_path_template, + regional_release_kind=args.regional_release_kind, + drop_regional_release=args.drop_regional_release, ) print(result.summary()) diff --git a/scripts/check_uk_uc_floor.py b/scripts/check_uk_uc_floor.py new file mode 100644 index 00000000..d79c6271 --- /dev/null +++ b/scripts/check_uk_uc_floor.py @@ -0,0 +1,57 @@ +"""Check a UK dataset against the Universal Credit coverage floor. + +Usage:: + + python scripts/check_uk_uc_floor.py --dataset \\ + [--year 2025] [--output receipt.json] + +A bundle dataset name (for example ``enhanced_frs_2024_25``) is materialized +from the certified bundle; private UK data needs ``HUGGING_FACE_TOKEN``. Exits +non-zero when either measure is outside the tolerance. The receipt holds +weighted national totals only, so it can be attached to a certification PR. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from policyengine.provenance.uk_uc_floor import check_uc_floor # noqa: E402 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--dataset", + required=True, + help="A bundle dataset name or a local UK dataset .h5 path.", + ) + parser.add_argument("--year", type=int, default=2025) + parser.add_argument("--output", type=Path, help="Write the JSON receipt here.") + args = parser.parse_args(argv) + + if Path(args.dataset).expanduser().is_file(): + dataset_path = str(Path(args.dataset).expanduser()) + dataset = Path(dataset_path).name + else: + from policyengine.provenance.dataset_materialization import ( + materialize_dataset, + ) + + source = materialize_dataset("uk", args.dataset) + dataset_path, dataset = source.path, source.source_uri + + receipt = check_uc_floor(dataset_path, args.year, dataset=dataset) + text = receipt.model_dump_json(indent=2) + if args.output: + args.output.write_text(text + "\n") + print(text) + return 0 if receipt.passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/policyengine/countries/uk/regions.py b/src/policyengine/countries/uk/regions.py index f74dd1f2..4e7a59cb 100644 --- a/src/policyengine/countries/uk/regions.py +++ b/src/policyengine/countries/uk/regions.py @@ -3,25 +3,31 @@ This module defines all UK geographic regions: - National (1) - Countries (4: England, Scotland, Wales, Northern Ireland) -- Constituencies (loaded from CSV at runtime) -- Local Authorities (loaded from CSV at runtime) - -Note: Constituencies and local authorities filter from the national dataset -using geography columns carried on each household. This keeps subnational -scoping tied to the dataset rows, not to a separate weight matrix whose -household dimension can drift from the default dataset. +- Constituencies (650, from the packaged 2024 list) +- Local Authorities (361, from the packaged April 2023 list) + +Note: Constituencies and local authorities filter household rows on the +geography columns each household carries (``constituency_code_oa`` and +``la_code_oa``). This keeps subnational scoping tied to the dataset rows, not +to a separate weight matrix whose household dimension can drift from the +default dataset. They load the dataset the certified bundle routes them to +(``region_datasets``), and the national dataset when it routes none. + +Their codes, names and hex positions come from the area lists shipped with the +package (``data/uk_geography``), so building them needs no network access. """ +import csv import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from policyengine.core.region import Region, RegionRegistry from policyengine.core.scoping_strategy import RowFilterStrategy from policyengine.data.uk_geography_assets import ( - CONSTITUENCY_ASSET_SPEC, - LOCAL_AUTHORITY_ASSET_SPEC, UK_GEOGRAPHY_BUCKET_URI, + packaged_uk_area_list_path, ) +from policyengine.provenance import manifest as release_manifest from policyengine.provenance.manifest import resolve_region_dataset_path if TYPE_CHECKING: @@ -55,81 +61,58 @@ ) +def _load_area_list(geography_type: str) -> list[dict]: + with open(packaged_uk_area_list_path(geography_type), encoding="utf-8") as file: + return [ + {"code": row["code"], "name": row["name"]} for row in csv.DictReader(file) + ] + + def _load_constituencies_from_csv() -> list[dict]: - """Load UK constituency data from CSV. + """Load the 650 UK parliamentary constituencies (2024 boundaries). - Constituencies are loaded from: - gs://policyengine-uk-data-private/constituencies_2024.csv + Read from the packaged ``data/uk_geography/constituencies_2024.csv``. Returns: List of dicts with 'code' and 'name' keys """ - try: - from policyengine_core.tools.google_cloud import download - except ImportError: - # If policyengine_core is not available, return empty list - return [] - - try: - csv_path = download( - gcs_bucket=CONSTITUENCY_ASSET_SPEC.bucket, - gcs_key=CONSTITUENCY_ASSET_SPEC.lookup_csv_filename, - ) - import pandas as pd - - df = pd.read_csv(csv_path) - return [{"code": row["code"], "name": row["name"]} for _, row in df.iterrows()] - except (OSError, KeyError, ValueError) as exc: - logger.warning("Failed to load constituencies CSV: %s", exc) - return [] - except Exception: - logger.error("Unexpected error loading constituencies CSV", exc_info=True) - return [] + return _load_area_list("constituency") def _load_local_authorities_from_csv() -> list[dict]: - """Load UK local authority data from CSV. + """Load the 361 UK local authority districts (April 2023). - Local authorities are loaded from: - gs://policyengine-uk-data-private/local_authorities_2021.csv + Read from the packaged ``data/uk_geography/local_authorities_2023.csv``. Returns: List of dicts with 'code' and 'name' keys """ - try: - from policyengine_core.tools.google_cloud import download - except ImportError: - # If policyengine_core is not available, return empty list - return [] - - try: - csv_path = download( - gcs_bucket=LOCAL_AUTHORITY_ASSET_SPEC.bucket, - gcs_key=LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename, - ) - import pandas as pd + return _load_area_list("local_authority") - df = pd.read_csv(csv_path) - return [{"code": row["code"], "name": row["name"]} for _, row in df.iterrows()] - except (OSError, KeyError, ValueError) as exc: - logger.warning("Failed to load local authorities CSV: %s", exc) - return [] - except Exception: - logger.error("Unexpected error loading local authorities CSV", exc_info=True) - return [] + +def area_regions_routed_to_local_release() -> bool: + """Whether the bundle routes UK area regions to a local-area release. + + Constituency and local-authority regions join the default registry only + then. A national-first bundle routes them to an area-coded national + release instead, whose consumers keep their own area handling. + """ + manifest = release_manifest.get_release_manifest("uk") + return manifest.regional_release_kind == "local_area" def build_uk_region_registry( - include_constituencies: bool = False, - include_local_authorities: bool = False, + include_constituencies: Optional[bool] = None, + include_local_authorities: Optional[bool] = None, ) -> RegionRegistry: """Build the UK region registry. Args: - include_constituencies: If True, load and include constituencies from CSV. - Defaults to False to avoid GCS dependency at import time. - include_local_authorities: If True, load and include local authorities from CSV. - Defaults to False to avoid GCS dependency at import time. + include_constituencies: Whether to include the 650 constituencies. + ``None`` includes them only when the bundle routes area regions + to a local-area release. + include_local_authorities: Whether to include the 361 local + authorities. ``None`` follows the same rule. Returns: RegionRegistry containing: @@ -138,6 +121,13 @@ def build_uk_region_registry( - Optionally: constituencies (if include_constituencies=True) - Optionally: local authorities (if include_local_authorities=True) """ + if include_constituencies is None or include_local_authorities is None: + routed_to_local = area_regions_routed_to_local_release() + if include_constituencies is None: + include_constituencies = routed_to_local + if include_local_authorities is None: + include_local_authorities = routed_to_local + regions: list[Region] = [] # 1. National region (has dedicated dataset) @@ -166,9 +156,10 @@ def build_uk_region_registry( ) ) - # 3. Constituencies (optional, loaded from CSV) + # 3. Constituencies (packaged list) if include_constituencies: constituencies = _load_constituencies_from_csv() + constituency_dataset_path = resolve_region_dataset_path("uk", "constituency") for const in constituencies: regions.append( Region( @@ -176,6 +167,7 @@ def build_uk_region_registry( label=const["name"], region_type="constituency", parent_code="uk", + dataset_path=constituency_dataset_path, scoping_strategy=RowFilterStrategy( variable_name="constituency_code_oa", variable_value=const["code"], @@ -183,9 +175,12 @@ def build_uk_region_registry( ) ) - # 4. Local Authorities (optional, loaded from CSV) + # 4. Local authorities (packaged list) if include_local_authorities: local_authorities = _load_local_authorities_from_csv() + local_authority_dataset_path = resolve_region_dataset_path( + "uk", "local_authority" + ) for la in local_authorities: regions.append( Region( @@ -193,6 +188,7 @@ def build_uk_region_registry( label=la["name"], region_type="local_authority", parent_code="uk", + dataset_path=local_authority_dataset_path, scoping_strategy=RowFilterStrategy( variable_name="la_code_oa", variable_value=la["code"], @@ -203,7 +199,8 @@ def build_uk_region_registry( return RegionRegistry(country_id="uk", regions=regions) -# Default registry with just core regions (national + countries) -# To get full registry with constituencies/LAs, call: +# Default registry: national and countries, plus constituencies and local +# authorities when the bundle routes them to a local-area release. To include +# them regardless, call: # build_uk_region_registry(include_constituencies=True, include_local_authorities=True) uk_region_registry = build_uk_region_registry() diff --git a/src/policyengine/data/qualification/uk_uc_floor.json b/src/policyengine/data/qualification/uk_uc_floor.json new file mode 100644 index 00000000..53821225 --- /dev/null +++ b/src/policyengine/data/qualification/uk_uc_floor.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "description": "Universal Credit coverage floor for UK certification candidates (PolicyEngine/policyengine.py#491, #553). Benefit units with Universal Credit and Universal Credit spend in Great Britain must each be within the tolerance of DWP's and the OBR's figures for the same year. These are the totals policyengine-uk-data calibrates to (targets dwp/uc/households and obr/universal_credit).", + "tolerance": 0.1, + "geography": "great_britain", + "years": { + "2025": { + "benefit_units": { + "value": 6700000, + "source": "DWP, Universal Credit deductions statistics, March 2025 to February 2026, Table 1: households on Universal Credit (households with a deduction divided by their share), calendar-2025 average", + "url": "https://www.gov.uk/government/statistics/universal-credit-quarterly-statistics-29-april-2013-to-12-february-2026/universal-credit-deductions-statistics-march-2025-to-february-2026" + }, + "spend": { + "value": 79286349140, + "source": "OBR, Economic and fiscal outlook, March 2026, detailed forecast tables (expenditure), Table 4.9: Universal Credit inside and outside the welfare cap, 2025-26 (£66.411bn + £12.876bn; DWP social security, Great Britain)", + "url": "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/" + } + }, + "2026": { + "benefit_units": { + "value": 7200000, + "source": "DWP, Universal Credit deductions statistics, March 2025 to February 2026, Table 1: households on Universal Credit, December 2025 to February 2026 plateau", + "url": "https://www.gov.uk/government/statistics/universal-credit-quarterly-statistics-29-april-2013-to-12-february-2026/universal-credit-deductions-statistics-march-2025-to-february-2026" + }, + "spend": { + "value": 87649386699, + "source": "OBR, Economic and fiscal outlook, March 2026, detailed forecast tables (expenditure), Table 4.9: Universal Credit inside and outside the welfare cap, 2026-27 (£71.710bn + £15.939bn; DWP social security, Great Britain)", + "url": "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/" + } + } + } +} diff --git a/src/policyengine/data/uk_geography/SOURCES.md b/src/policyengine/data/uk_geography/SOURCES.md new file mode 100644 index 00000000..1e736684 --- /dev/null +++ b/src/policyengine/data/uk_geography/SOURCES.md @@ -0,0 +1,38 @@ +# UK area lists + +Codes, names and hex-map positions for the UK area regions and the labels of +geography outputs. They are not the lookup CSVs paired with the legacy +weight matrices, whose row order indexes those matrices. + +## `local_authorities_2023.csv` + +Columns `code,name,x,y`; 361 local authority districts on the April 2023 frame: +296 in England, 22 in Wales, 32 in Scotland and 11 in Northern Ireland, +including Ards and North Down (`N09000011`) and the four unitaries created on +1 April 2023 (`E06000063`–`E06000066`), and none of the 17 districts they +replaced. + +- Codes and names: ONS, Local Authority Districts (April 2023) Names and Codes + in the United Kingdom (columns `LAD23CD`, `LAD23NM`), ONS Open Geography + Portal item `e8b361ba9e98418ba8ff2f892d00c352`. Contains public sector + information licensed under the Open Government Licence v3.0. +- Hex positions: PolicyEngine's April 2023 local-authority hex layout, from + PolicyEngine/policyengine-sim-api + `projects/policyengine-simulation-executor/src/policyengine_simulation_executor/static_runtime_files/uk_local_authorities/` + at commit `3ac09cf9205bc66a5809e70a634c457362d1a887`: + - `names.csv` sha256 `9e183902e54383d56f7f311d0eaac6a9ee92522a5f21c58ce5118265ae3f1bf4` + - `coordinates_lad23.csv` sha256 `b1489a5b0101ff6d91ef9262e992cca19b59359edb42d012c305a21a04571f64` +- Built by joining `coordinates_lad23.csv` to `names.csv` on `code`, sorted by + code. The names match the ONS file exactly. Against policyengine-uk-data's + `local_authorities_2021.csv` (360 rows on the same frame), this adds + `N09000011` and replaces the shared (0, 0) placeholder of the four 2023 + unitaries with their own hexes. + +## `constituencies_2024.csv` + +Columns `code,name,x,y,region,country`; the 650 parliamentary constituencies +of the 2024 boundaries: 543 in England, 57 in Scotland, 32 in Wales and 18 in +Northern Ireland. Copied unchanged from PolicyEngine/policyengine-uk-data +`policyengine_uk_data/storage/constituencies_2024.csv` at commit +`26ccdd1fd1aff6665704fda37388a86adecb5122` (sha256 `f4f35d61ffc004190c438a7e1583caefd85054f962671a00069aedd0a1b5b737`). +Codes and names: ONS (Open Government Licence v3.0). diff --git a/src/policyengine/data/uk_geography/constituencies_2024.csv b/src/policyengine/data/uk_geography/constituencies_2024.csv new file mode 100644 index 00000000..d0164c4a --- /dev/null +++ b/src/policyengine/data/uk_geography/constituencies_2024.csv @@ -0,0 +1,651 @@ +code,name,x,y,region,country +E14001063,Aldershot,56,-40,South East,England +E14001064,Aldridge-Brownhills,56,-30,West Midlands,England +E14001065,Altrincham and Sale West,52,-25,North West,England +E14001066,Amber Valley,58,-27,East Midlands,England +E14001067,Arundel and South Downs,61,-44,South East,England +E14001068,Ashfield,60,-27,East Midlands,England +E14001069,Ashford,72,-42,South East,England +E14001070,Ashton-under-Lyne,54,-23,North West,England +E14001071,Aylesbury,60,-35,South East,England +E14001072,Banbury,58,-33,South East,England +E14001073,Barking,68,-38,London,England +E14001074,Barnsley North,57,-23,Yorkshire and the Humber,England +E14001075,Barnsley South,58,-23,Yorkshire and the Humber,England +E14001076,Barrow and Furness,54,-16,North West,England +E14001077,Basildon and Billericay,67,-34,Eastern,England +E14001078,Basingstoke,55,-39,South East,England +E14001079,Bassetlaw,61,-26,East Midlands,England +E14001080,Bath,51,-40,South West,England +E14001081,Battersea,62,-41,London,England +E14001082,Beaconsfield,57,-37,South East,England +E14001083,Beckenham and Penge,65,-43,London,England +E14001084,Bedford,63,-32,Eastern,England +E14001085,Bermondsey and Old Southwark,64,-40,London,England +E14001086,Bethnal Green and Stepney,65,-39,London,England +E14001087,Beverley and Holderness,64,-22,Yorkshire and the Humber,England +E14001088,Bexhill and Battle,70,-44,South East,England +E14001089,Bexleyheath and Crayford,67,-39,London,England +E14001090,Bicester and Woodstock,59,-34,South East,England +E14001091,Birkenhead,49,-27,North West,England +E14001092,Birmingham Edgbaston,53,-33,West Midlands,England +E14001093,Birmingham Erdington,54,-31,West Midlands,England +E14001094,Birmingham Hall Green and Moseley,55,-32,West Midlands,England +E14001095,Birmingham Hodge Hill and Solihull North,55,-31,West Midlands,England +E14001096,Birmingham Ladywood,54,-32,West Midlands,England +E14001097,Birmingham Northfield,54,-34,West Midlands,England +E14001098,Birmingham Perry Barr,53,-31,West Midlands,England +E14001099,Birmingham Selly Oak,54,-33,West Midlands,England +E14001100,Birmingham Yardley,56,-32,West Midlands,England +E14001101,Bishop Auckland,54,-14,North East,England +E14001102,Blackburn,53,-19,North West,England +E14001103,Blackley and Middleton South,53,-23,North West,England +E14001104,Blackpool North and Fleetwood,53,-18,North West,England +E14001105,Blackpool South,52,-18,North West,England +E14001106,Blaydon and Consett,55,-14,North East,England +E14001107,Blyth and Ashington,55,-12,North East,England +E14001108,Bognor Regis and Littlehampton,63,-44,South East,England +E14001109,Bolsover,60,-26,East Midlands,England +E14001110,Bolton North East,52,-21,North West,England +E14001111,Bolton South and Walkden,52,-22,North West,England +E14001112,Bolton West,51,-21,North West,England +E14001113,Bootle,49,-22,North West,England +E14001114,Boston and Skegness,64,-26,East Midlands,England +E14001115,Bournemouth East,52,-43,South West,England +E14001116,Bournemouth West,52,-42,South West,England +E14001117,Bracknell,56,-39,South East,England +E14001118,Bradford East,58,-20,Yorkshire and the Humber,England +E14001119,Bradford South,56,-21,Yorkshire and the Humber,England +E14001120,Bradford West,57,-20,Yorkshire and the Humber,England +E14001121,Braintree,67,-31,Eastern,England +E14001122,Brent East,61,-38,London,England +E14001123,Brent West,60,-38,London,England +E14001124,Brentford and Isleworth,60,-40,London,England +E14001125,Brentwood and Ongar,66,-33,Eastern,England +E14001126,Bridgwater,48,-41,South West,England +E14001127,Bridlington and The Wolds,63,-20,Yorkshire and the Humber,England +E14001128,Brigg and Immingham,62,-24,Yorkshire and the Humber,England +E14001129,Brighton Kemptown and Peacehaven,67,-45,South East,England +E14001130,Brighton Pavilion,67,-44,South East,England +E14001131,Bristol Central,51,-38,South West,England +E14001132,Bristol East,52,-38,South West,England +E14001133,Bristol North East,51,-37,South West,England +E14001134,Bristol North West,50,-38,South West,England +E14001135,Bristol South,51,-39,South West,England +E14001136,Broadland and Fakenham,66,-27,Eastern,England +E14001137,Bromley and Biggin Hill,67,-42,London,England +E14001138,Bromsgrove,52,-33,West Midlands,England +E14001139,Broxbourne,66,-35,Eastern,England +E14001140,Broxtowe,59,-27,East Midlands,England +E14001141,Buckingham and Bletchley,60,-34,South East,England +E14001142,Burnley,55,-19,North West,England +E14001143,Burton and Uttoxeter,56,-28,West Midlands,England +E14001144,Bury North,53,-21,North West,England +E14001145,Bury South,53,-22,North West,England +E14001146,Bury St Edmunds and Stowmarket,68,-31,Eastern,England +E14001147,Calder Valley,56,-20,Yorkshire and the Humber,England +E14001148,Camborne and Redruth,43,-45,South West,England +E14001149,Cambridge,65,-30,Eastern,England +E14001150,Cannock Chase,54,-29,West Midlands,England +E14001151,Canterbury,71,-41,South East,England +E14001152,Carlisle,53,-14,North West,England +E14001153,Carshalton and Wallington,62,-43,London,England +E14001154,Castle Point,69,-36,Eastern,England +E14001155,Central Devon,47,-42,South West,England +E14001156,Central Suffolk and North Ipswich,68,-29,Eastern,England +E14001157,Chatham and Aylesford,69,-40,South East,England +E14001158,Cheadle,55,-26,North West,England +E14001159,Chelmsford,67,-33,Eastern,England +E14001160,Chelsea and Fulham,61,-40,London,England +E14001161,Cheltenham,52,-36,South West,England +E14001162,Chesham and Amersham,59,-36,South East,England +E14001163,Chester North and Neston,50,-28,North West,England +E14001164,Chester South and Eddisbury,51,-27,North West,England +E14001165,Chesterfield,59,-26,East Midlands,England +E14001166,Chichester,60,-44,South East,England +E14001167,Chingford and Woodford Green,64,-35,London,England +E14001168,Chippenham,52,-39,South West,England +E14001169,Chipping Barnet,62,-36,London,England +E14001170,Chorley,53,-20,North West,England +E14001171,Christchurch,53,-42,South West,England +E14001172,Cities of London and Westminster,63,-40,London,England +E14001173,City of Durham,55,-16,North East,England +E14001174,Clacton,69,-32,Eastern,England +E14001175,Clapham and Brixton Hill,62,-42,London,England +E14001176,Colchester,68,-32,Eastern,England +E14001177,Colne Valley,55,-23,Yorkshire and the Humber,England +E14001178,Congleton,54,-27,North West,England +E14001179,Corby and East Northamptonshire,62,-30,East Midlands,England +E14001180,Coventry East,57,-33,West Midlands,England +E14001181,Coventry North West,56,-33,West Midlands,England +E14001182,Coventry South,57,-34,West Midlands,England +E14001183,Cramlington and Killingworth,56,-12,North East,England +E14001184,Crawley,69,-44,South East,England +E14001185,Crewe and Nantwich,53,-27,North West,England +E14001186,Croydon East,65,-42,London,England +E14001187,Croydon South,64,-43,London,England +E14001188,Croydon West,63,-43,London,England +E14001189,Dagenham and Rainham,67,-37,London,England +E14001190,Darlington,55,-17,North East,England +E14001191,Dartford,68,-40,South East,England +E14001192,Daventry,60,-32,East Midlands,England +E14001193,Derby North,58,-28,East Midlands,England +E14001194,Derby South,57,-28,East Midlands,England +E14001195,Derbyshire Dales,57,-26,East Midlands,England +E14001196,Dewsbury and Batley,57,-22,Yorkshire and the Humber,England +E14001197,Didcot and Wantage,54,-38,South East,England +E14001198,Doncaster Central,60,-23,Yorkshire and the Humber,England +E14001199,Doncaster East and the Isle of Axholme,61,-23,Yorkshire and the Humber,England +E14001200,Doncaster North,61,-22,Yorkshire and the Humber,England +E14001201,Dorking and Horley,59,-43,South East,England +E14001202,Dover and Deal,72,-41,South East,England +E14001203,Droitwich and Evesham,54,-36,West Midlands,England +E14001204,Dudley,51,-31,West Midlands,England +E14001205,Dulwich and West Norwood,63,-42,London,England +E14001206,Dunstable and Leighton Buzzard,62,-33,Eastern,England +E14001207,Ealing Central and Acton,59,-39,London,England +E14001208,Ealing North,59,-38,London,England +E14001209,Ealing Southall,58,-39,London,England +E14001210,Earley and Woodley,56,-36,South East,England +E14001211,Easington,57,-16,North East,England +E14001212,East Grinstead and Uckfield,69,-43,South East,England +E14001213,East Ham,67,-38,London,England +E14001214,East Hampshire,55,-41,South East,England +E14001215,East Surrey,67,-43,South East,England +E14001216,East Thanet,71,-39,South East,England +E14001217,East Wiltshire,53,-41,South West,England +E14001218,East Worthing and Shoreham,65,-44,South East,England +E14001219,Eastbourne,69,-45,South East,England +E14001220,Eastleigh,54,-41,South East,England +E14001221,Edmonton and Winchmore Hill,64,-36,London,England +E14001222,Ellesmere Port and Bromborough,50,-27,North West,England +E14001223,Eltham and Chislehurst,66,-41,London,England +E14001224,Ely and East Cambridgeshire,66,-30,Eastern,England +E14001225,Enfield North,62,-35,London,England +E14001226,Epping Forest,67,-35,Eastern,England +E14001227,Epsom and Ewell,60,-43,South East,England +E14001228,Erewash,59,-28,East Midlands,England +E14001229,Erith and Thamesmead,67,-40,London,England +E14001230,Esher and Walton,58,-42,South East,England +E14001231,Exeter,48,-42,South West,England +E14001232,Exmouth and Exeter East,48,-43,South West,England +E14001233,Fareham and Waterlooville,55,-43,South East,England +E14001234,Farnham and Bordon,56,-42,South East,England +E14001235,Faversham and Mid Kent,71,-40,South East,England +E14001236,Feltham and Heston,59,-40,London,England +E14001237,Filton and Bradley Stoke,50,-37,South West,England +E14001238,Finchley and Golders Green,61,-37,London,England +E14001239,Folkestone and Hythe,71,-42,South East,England +E14001240,Forest of Dean,50,-35,South West,England +E14001241,Frome and East Somerset,50,-41,South West,England +E14001242,Fylde,51,-19,North West,England +E14001243,Gainsborough,61,-25,East Midlands,England +E14001244,Gateshead Central and Whickham,56,-15,North East,England +E14001245,Gedling,61,-28,East Midlands,England +E14001246,Gillingham and Rainham,70,-40,South East,England +E14001247,Glastonbury and Somerton,49,-41,South West,England +E14001248,Gloucester,51,-35,South West,England +E14001249,Godalming and Ash,57,-42,South East,England +E14001250,Goole and Pocklington,61,-21,Yorkshire and the Humber,England +E14001251,Gorton and Denton,55,-24,North West,England +E14001252,Gosport,57,-43,South East,England +E14001253,Grantham and Bourne,63,-28,East Midlands,England +E14001254,Gravesham,68,-39,South East,England +E14001255,Great Grimsby and Cleethorpes,63,-24,Yorkshire and the Humber,England +E14001256,Great Yarmouth,67,-27,Eastern,England +E14001257,Greenwich and Woolwich,66,-40,London,England +E14001258,Guildford,56,-41,South East,England +E14001259,Hackney North and Stoke Newington,64,-38,London,England +E14001260,Hackney South and Shoreditch,64,-39,London,England +E14001261,Halesowen,51,-33,West Midlands,England +E14001262,Halifax,55,-21,Yorkshire and the Humber,England +E14001263,Hamble Valley,56,-43,South East,England +E14001264,Hammersmith and Chiswick,60,-39,London,England +E14001265,Hampstead and Highgate,62,-38,London,England +E14001266,"Harborough, Oadby and Wigston",61,-31,East Midlands,England +E14001267,Harlow,67,-32,Eastern,England +E14001268,Harpenden and Berkhamsted,62,-34,Eastern,England +E14001269,Harrogate and Knaresborough,59,-18,Yorkshire and the Humber,England +E14001270,Harrow East,60,-37,London,England +E14001271,Harrow West,59,-37,London,England +E14001272,Hartlepool,59,-16,North East,England +E14001273,Harwich and North Essex,69,-31,Eastern,England +E14001274,Hastings and Rye,70,-43,South East,England +E14001275,Havant,59,-44,South East,England +E14001276,Hayes and Harlington,58,-38,London,England +E14001277,Hazel Grove,55,-25,North West,England +E14001278,Hemel Hempstead,64,-34,Eastern,England +E14001279,Hendon,61,-36,London,England +E14001280,Henley and Thame,58,-35,South East,England +E14001281,Hereford and South Herefordshire,51,-34,West Midlands,England +E14001282,Herne Bay and Sandwich,72,-40,South East,England +E14001283,Hertford and Stortford,66,-32,Eastern,England +E14001284,Hertsmere,66,-34,Eastern,England +E14001285,Hexham,53,-13,North East,England +E14001286,Heywood and Middleton North,54,-20,North West,England +E14001287,High Peak,56,-25,East Midlands,England +E14001288,Hinckley and Bosworth,58,-30,East Midlands,England +E14001289,Hitchin,64,-32,Eastern,England +E14001290,Holborn and St Pancras,62,-39,London,England +E14001291,Honiton and Sidmouth,49,-43,South West,England +E14001292,Hornchurch and Upminster,66,-37,London,England +E14001293,Hornsey and Friern Barnet,63,-36,London,England +E14001294,Horsham,62,-44,South East,England +E14001295,Houghton and Sunderland South,57,-15,North East,England +E14001296,Hove and Portslade,66,-44,South East,England +E14001297,Huddersfield,56,-22,Yorkshire and the Humber,England +E14001298,Huntingdon,63,-31,Eastern,England +E14001299,Hyndburn,54,-19,North West,England +E14001300,Ilford North,65,-36,London,England +E14001301,Ilford South,65,-37,London,England +E14001302,Ipswich,68,-30,Eastern,England +E14001303,Isle of Wight East,54,-45,South East,England +E14001304,Isle of Wight West,53,-45,South East,England +E14001305,Islington North,63,-38,London,England +E14001306,Islington South and Finsbury,63,-39,London,England +E14001307,Jarrow and Gateshead East,57,-14,North East,England +E14001308,Keighley and Ilkley,56,-19,Yorkshire and the Humber,England +E14001309,Kenilworth and Southam,56,-34,West Midlands,England +E14001310,Kensington and Bayswater,61,-39,London,England +E14001311,Kettering,61,-30,East Midlands,England +E14001312,Kingston and Surbiton,59,-42,London,England +E14001313,Kingston upon Hull East,63,-22,Yorkshire and the Humber,England +E14001314,Kingston upon Hull North and Cottingham,62,-21,Yorkshire and the Humber,England +E14001315,Kingston upon Hull West and Haltemprice,62,-22,Yorkshire and the Humber,England +E14001316,Kingswinford and South Staffordshire,52,-30,West Midlands,England +E14001317,Knowsley,50,-23,North West,England +E14001318,Lancaster and Wyre,54,-18,North West,England +E14001319,Leeds Central and Headingley,60,-20,Yorkshire and the Humber,England +E14001320,Leeds East,61,-20,Yorkshire and the Humber,England +E14001321,Leeds North East,59,-19,Yorkshire and the Humber,England +E14001322,Leeds North West,58,-19,Yorkshire and the Humber,England +E14001323,Leeds South,59,-21,Yorkshire and the Humber,England +E14001324,Leeds South West and Morley,58,-21,Yorkshire and the Humber,England +E14001325,Leeds West and Pudsey,59,-20,Yorkshire and the Humber,England +E14001326,Leicester East,60,-30,East Midlands,England +E14001327,Leicester South,60,-31,East Midlands,England +E14001328,Leicester West,59,-31,East Midlands,England +E14001329,Leigh and Atherton,51,-25,North West,England +E14001330,Lewes,68,-45,South East,England +E14001331,Lewisham East,66,-42,London,England +E14001332,Lewisham North,65,-40,London,England +E14001333,Lewisham West and East Dulwich,65,-41,London,England +E14001334,Leyton and Wanstead,64,-37,London,England +E14001335,Lichfield,56,-29,West Midlands,England +E14001336,Lincoln,62,-25,East Midlands,England +E14001337,Liverpool Garston,50,-25,North West,England +E14001338,Liverpool Riverside,49,-24,North West,England +E14001339,Liverpool Walton,49,-23,North West,England +E14001340,Liverpool Wavertree,49,-25,North West,England +E14001341,Liverpool West Derby,50,-24,North West,England +E14001342,Loughborough,59,-30,East Midlands,England +E14001343,Louth and Horncastle,63,-25,East Midlands,England +E14001344,Lowestoft,68,-28,Eastern,England +E14001345,Luton North,63,-33,Eastern,England +E14001346,Luton South and South Bedfordshire,63,-34,Eastern,England +E14001347,Macclesfield,56,-26,North West,England +E14001348,Maidenhead,57,-36,South East,England +E14001349,Maidstone and Malling,69,-41,South East,England +E14001350,Makerfield,51,-22,North West,England +E14001351,Maldon,69,-33,Eastern,England +E14001352,Manchester Central,54,-24,North West,England +E14001353,Manchester Rusholme,53,-25,North West,England +E14001354,Manchester Withington,54,-26,North West,England +E14001355,Mansfield,61,-27,East Midlands,England +E14001356,Melksham and Devizes,52,-40,South West,England +E14001357,Melton and Syston,61,-29,East Midlands,England +E14001358,Meriden and Solihull East,55,-33,West Midlands,England +E14001359,Mid Bedfordshire,62,-32,Eastern,England +E14001360,Mid Buckinghamshire,59,-35,South East,England +E14001361,Mid Cheshire,52,-27,North West,England +E14001362,Mid Derbyshire,57,-27,East Midlands,England +E14001363,Mid Dorset and North Poole,50,-43,South West,England +E14001364,Mid Leicestershire,58,-31,East Midlands,England +E14001365,Mid Norfolk,65,-28,Eastern,England +E14001366,Mid Sussex,68,-43,South East,England +E14001367,Middlesbrough and Thornaby East,57,-17,North East,England +E14001368,Middlesbrough South and East Cleveland,59,-17,North East,England +E14001369,Milton Keynes Central,61,-34,South East,England +E14001370,Milton Keynes North,61,-33,South East,England +E14001371,Mitcham and Morden,61,-43,London,England +E14001372,Morecambe and Lunesdale,54,-17,North West,England +E14001373,New Forest East,54,-43,South East,England +E14001374,New Forest West,53,-43,South East,England +E14001375,Newark,62,-26,East Midlands,England +E14001376,Newbury,54,-37,South East,England +E14001377,Newcastle upon Tyne Central and West,54,-13,North East,England +E14001378,Newcastle upon Tyne East and Wallsend,56,-14,North East,England +E14001379,Newcastle upon Tyne North,55,-13,North East,England +E14001380,Newcastle-under-Lyme,52,-28,West Midlands,England +E14001381,Newton Abbot,47,-43,South West,England +E14001382,Newton Aycliffe and Spennymoor,56,-16,North East,England +E14001383,Normanton and Hemsworth,59,-23,Yorkshire and the Humber,England +E14001384,North Bedfordshire,62,-31,Eastern,England +E14001385,North Cornwall,45,-43,South West,England +E14001386,North Cotswolds,53,-37,South West,England +E14001387,North Devon,46,-41,South West,England +E14001388,North Dorset,51,-42,South West,England +E14001389,North Durham,54,-15,North East,England +E14001390,North East Cambridgeshire,64,-29,Eastern,England +E14001391,North East Derbyshire,58,-26,East Midlands,England +E14001392,North East Hampshire,56,-38,South East,England +E14001393,North East Hertfordshire,65,-32,Eastern,England +E14001394,North East Somerset and Hanham,50,-39,South West,England +E14001395,North Herefordshire,52,-34,West Midlands,England +E14001396,North Norfolk,65,-27,Eastern,England +E14001397,North Northumberland,54,-12,North East,England +E14001398,North Shropshire,50,-29,West Midlands,England +E14001399,North Somerset,49,-39,South West,England +E14001400,North Warwickshire and Bedworth,57,-32,West Midlands,England +E14001401,North West Cambridgeshire,64,-30,Eastern,England +E14001402,North West Essex,66,-31,Eastern,England +E14001403,North West Hampshire,54,-39,South East,England +E14001404,North West Leicestershire,58,-29,East Midlands,England +E14001405,North West Norfolk,64,-28,Eastern,England +E14001406,Northampton North,61,-32,East Midlands,England +E14001407,Northampton South,60,-33,East Midlands,England +E14001408,Norwich North,66,-28,Eastern,England +E14001409,Norwich South,66,-29,Eastern,England +E14001410,Nottingham East,60,-29,East Midlands,England +E14001411,Nottingham North and Kimberley,60,-28,East Midlands,England +E14001412,Nottingham South,59,-29,East Midlands,England +E14001413,Nuneaton,57,-31,West Midlands,England +E14001414,Old Bexley and Sidcup,67,-41,London,England +E14001415,Oldham East and Saddleworth,55,-22,North West,England +E14001416,"Oldham West, Chadderton and Royton",54,-22,North West,England +E14001417,Orpington,66,-43,London,England +E14001418,Ossett and Denby Dale,58,-22,Yorkshire and the Humber,England +E14001419,Oxford East,58,-34,South East,England +E14001420,Oxford West and Abingdon,57,-35,South East,England +E14001421,Peckham,64,-41,London,England +E14001422,Pendle and Clitheroe,56,-18,North West,England +E14001423,Penistone and Stocksbridge,56,-23,Yorkshire and the Humber,England +E14001424,Penrith and Solway,52,-15,North West,England +E14001425,Peterborough,63,-29,Eastern,England +E14001426,Plymouth Moor View,46,-43,South West,England +E14001427,Plymouth Sutton and Devonport,47,-44,South West,England +E14001428,"Pontefract, Castleford and Knottingley",60,-22,Yorkshire and the Humber,England +E14001429,Poole,51,-43,South West,England +E14001430,Poplar and Limehouse,66,-39,London,England +E14001431,Portsmouth North,58,-43,South East,England +E14001432,Portsmouth South,58,-44,South East,England +E14001433,Preston,52,-19,North West,England +E14001434,Putney,61,-41,London,England +E14001435,Queen's Park and Maida Vale,62,-40,London,England +E14001436,Rawmarsh and Conisbrough,60,-24,Yorkshire and the Humber,England +E14001437,Rayleigh and Wickford,68,-34,Eastern,England +E14001438,Reading Central,55,-37,South East,England +E14001439,Reading West and Mid Berkshire,55,-36,South East,England +E14001440,Redcar,58,-17,North East,England +E14001441,Redditch,53,-35,West Midlands,England +E14001442,Reigate,68,-44,South East,England +E14001443,Ribble Valley,55,-18,North West,England +E14001444,Richmond and Northallerton,57,-18,Yorkshire and the Humber,England +E14001445,Richmond Park,59,-41,London,England +E14001446,Rochdale,54,-21,North West,England +E14001447,Rochester and Strood,69,-39,South East,England +E14001448,Romford,66,-36,London,England +E14001449,Romsey and Southampton North,54,-40,South East,England +E14001450,Rossendale and Darwen,55,-20,North West,England +E14001451,Rother Valley,60,-25,Yorkshire and the Humber,England +E14001452,Rotherham,59,-24,Yorkshire and the Humber,England +E14001453,Rugby,58,-32,West Midlands,England +E14001454,"Ruislip, Northwood and Pinner",60,-36,London,England +E14001455,Runcorn and Helsby,51,-28,North West,England +E14001456,Runnymede and Weybridge,57,-41,South East,England +E14001457,Rushcliffe,62,-28,East Midlands,England +E14001458,Rutland and Stamford,62,-29,East Midlands,England +E14001459,Salford,53,-24,North West,England +E14001460,Salisbury,52,-41,South West,England +E14001461,Scarborough and Whitby,61,-19,Yorkshire and the Humber,England +E14001462,Scunthorpe,61,-24,Yorkshire and the Humber,England +E14001463,Sefton Central,50,-20,North West,England +E14001464,Selby,60,-21,Yorkshire and the Humber,England +E14001465,Sevenoaks,68,-42,South East,England +E14001466,Sheffield Brightside and Hillsborough,58,-24,Yorkshire and the Humber,England +E14001467,Sheffield Central,58,-25,Yorkshire and the Humber,England +E14001468,Sheffield Hallam,57,-24,Yorkshire and the Humber,England +E14001469,Sheffield Heeley,57,-25,Yorkshire and the Humber,England +E14001470,Sheffield South East,59,-25,Yorkshire and the Humber,England +E14001471,Sherwood Forest,62,-27,East Midlands,England +E14001472,Shipley,57,-19,Yorkshire and the Humber,England +E14001473,Shrewsbury,51,-30,West Midlands,England +E14001474,Sittingbourne and Sheppey,70,-39,South East,England +E14001475,Skipton and Ripon,58,-18,Yorkshire and the Humber,England +E14001476,Sleaford and North Hykeham,63,-26,East Midlands,England +E14001477,Slough,56,-37,South East,England +E14001478,Smethwick,53,-32,West Midlands,England +E14001479,Solihull West and Shirley,55,-34,West Midlands,England +E14001480,South Basildon and East Thurrock,68,-36,Eastern,England +E14001481,South Cambridgeshire,65,-31,Eastern,England +E14001482,South Cotswolds,53,-38,South West,England +E14001483,South Derbyshire,57,-29,East Midlands,England +E14001484,South Devon,48,-45,South West,England +E14001485,South Dorset,51,-44,South West,England +E14001486,South East Cornwall,46,-44,South West,England +E14001487,South Holland and The Deepings,63,-27,East Midlands,England +E14001488,South Leicestershire,59,-32,East Midlands,England +E14001489,South Norfolk,67,-29,Eastern,England +E14001490,South Northamptonshire,59,-33,East Midlands,England +E14001491,South Ribble,52,-20,North West,England +E14001492,South Shields,58,-14,North East,England +E14001493,South Shropshire,50,-31,West Midlands,England +E14001494,South Suffolk,69,-30,Eastern,England +E14001495,South West Devon,47,-45,South West,England +E14001496,South West Hertfordshire,61,-35,Eastern,England +E14001497,South West Norfolk,65,-29,Eastern,England +E14001498,South West Wiltshire,51,-41,South West,England +E14001499,Southampton Itchen,55,-42,South East,England +E14001500,Southampton Test,54,-42,South East,England +E14001501,Southend East and Rochford,69,-34,Eastern,England +E14001502,Southend West and Leigh,68,-35,Eastern,England +E14001503,Southgate and Wood Green,63,-35,London,England +E14001504,Southport,50,-19,North West,England +E14001505,Spelthorne,58,-40,South East,England +E14001506,Spen Valley,57,-21,Yorkshire and the Humber,England +E14001507,St Albans,65,-34,Eastern,England +E14001508,St Austell and Newquay,45,-44,South West,England +E14001509,St Helens North,50,-21,North West,England +E14001510,St Helens South and Whiston,50,-22,North West,England +E14001511,St Ives,43,-46,South West,England +E14001512,St Neots and Mid Cambridgeshire,64,-31,Eastern,England +E14001513,Stafford,54,-28,West Midlands,England +E14001514,Staffordshire Moorlands,56,-27,West Midlands,England +E14001515,Stalybridge and Hyde,56,-24,North West,England +E14001516,Stevenage,64,-33,Eastern,England +E14001517,Stockport,54,-25,North West,England +E14001518,Stockton North,58,-16,North East,England +E14001519,Stockton West,56,-17,North East,England +E14001520,Stoke-on-Trent Central,55,-28,West Midlands,England +E14001521,Stoke-on-Trent North,55,-27,West Midlands,England +E14001522,Stoke-on-Trent South,55,-29,West Midlands,England +E14001523,"Stone, Great Wyrley and Penkridge",53,-28,West Midlands,England +E14001524,Stourbridge,51,-32,West Midlands,England +E14001525,Stratford and Bow,65,-38,London,England +E14001526,Stratford-on-Avon,54,-35,West Midlands,England +E14001527,Streatham and Croydon North,64,-42,London,England +E14001528,Stretford and Urmston,52,-24,North West,England +E14001529,Stroud,52,-37,South West,England +E14001530,Suffolk Coastal,69,-29,Eastern,England +E14001531,Sunderland Central,58,-15,North East,England +E14001532,Surrey Heath,57,-39,South East,England +E14001533,Sussex Weald,70,-42,South East,England +E14001534,Sutton and Cheam,60,-42,London,England +E14001535,Sutton Coldfield,56,-31,West Midlands,England +E14001536,Swindon North,53,-39,South West,England +E14001537,Swindon South,53,-40,South West,England +E14001538,Tamworth,57,-30,West Midlands,England +E14001539,Tatton,52,-26,North West,England +E14001540,Taunton and Wellington,49,-42,South West,England +E14001541,Telford,52,-29,West Midlands,England +E14001542,Tewkesbury,53,-36,South West,England +E14001543,The Wrekin,51,-29,West Midlands,England +E14001544,Thirsk and Malton,60,-18,Yorkshire and the Humber,England +E14001545,Thornbury and Yate,51,-36,South West,England +E14001546,Thurrock,67,-36,Eastern,England +E14001547,Tipton and Wednesbury,52,-31,West Midlands,England +E14001548,Tiverton and Minehead,47,-41,South West,England +E14001549,Tonbridge,68,-41,South East,England +E14001550,Tooting,61,-42,London,England +E14001551,Torbay,48,-44,South West,England +E14001552,Torridge and Tavistock,46,-42,South West,England +E14001553,Tottenham,62,-37,London,England +E14001554,Truro and Falmouth,44,-45,South West,England +E14001555,Tunbridge Wells,69,-42,South East,England +E14001556,Twickenham,58,-41,London,England +E14001557,Tynemouth,56,-13,North East,England +E14001558,Uxbridge and South Ruislip,58,-37,London,England +E14001559,Vauxhall and Camberwell Green,63,-41,London,England +E14001560,Wakefield and Rothwell,59,-22,Yorkshire and the Humber,England +E14001561,Wallasey,48,-27,North West,England +E14001562,Walsall and Bloxwich,55,-30,West Midlands,England +E14001563,Walthamstow,63,-37,London,England +E14001564,Warrington North,51,-23,North West,England +E14001565,Warrington South,51,-24,North West,England +E14001566,Warwick and Leamington,55,-35,West Midlands,England +E14001567,Washington and Gateshead South,55,-15,North East,England +E14001568,Watford,65,-35,Eastern,England +E14001569,Waveney Valley,67,-28,Eastern,England +E14001570,Weald of Kent,70,-41,South East,England +E14001571,Wellingborough and Rushden,63,-30,East Midlands,England +E14001572,Wells and Mendip Hills,50,-40,South West,England +E14001573,Welwyn Hatfield,65,-33,Eastern,England +E14001574,West Bromwich,52,-32,West Midlands,England +E14001575,West Dorset,50,-44,South West,England +E14001576,West Ham and Beckton,66,-38,London,England +E14001577,West Lancashire,49,-21,North West,England +E14001578,West Suffolk,67,-30,Eastern,England +E14001579,West Worcestershire,52,-35,West Midlands,England +E14001580,Westmorland and Lonsdale,53,-15,North West,England +E14001581,Weston-super-Mare,49,-40,South West,England +E14001582,Wetherby and Easingwold,62,-20,Yorkshire and the Humber,England +E14001583,Whitehaven and Workington,53,-16,North West,England +E14001584,Widnes and Halewood,51,-26,North West,England +E14001585,Wigan,51,-20,North West,England +E14001586,Wimbledon,60,-41,London,England +E14001587,Winchester,55,-40,South East,England +E14001588,Windsor,57,-38,South East,England +E14001589,Wirral West,49,-28,North West,England +E14001590,Witham,68,-33,Eastern,England +E14001591,Witney,56,-35,South East,England +E14001592,Woking,57,-40,South East,England +E14001593,Wokingham,55,-38,South East,England +E14001594,Wolverhampton North East,53,-29,West Midlands,England +E14001595,Wolverhampton South East,54,-30,West Midlands,England +E14001596,Wolverhampton West,53,-30,West Midlands,England +E14001597,Worcester,53,-34,West Midlands,England +E14001598,Worsley and Eccles,52,-23,North West,England +E14001599,Worthing West,64,-44,South East,England +E14001600,Wycombe,58,-36,South East,England +E14001601,Wyre Forest,50,-33,West Midlands,England +E14001602,Wythenshawe and Sale East,53,-26,North West,England +E14001603,Yeovil,50,-42,South West,England +E14001604,York Central,60,-19,Yorkshire and the Humber,England +E14001605,York Outer,61,-18,Yorkshire and the Humber,England +N05000001,Belfast East,45,-17,Northern Ireland,Northern Ireland +N05000002,Belfast North,45,-16,Northern Ireland,Northern Ireland +N05000003,Belfast South and Mid Down,45,-18,Northern Ireland,Northern Ireland +N05000004,Belfast West,44,-17,Northern Ireland,Northern Ireland +N05000005,East Antrim,45,-15,Northern Ireland,Northern Ireland +N05000006,East Londonderry,43,-15,Northern Ireland,Northern Ireland +N05000007,Fermanagh and South Tyrone,42,-17,Northern Ireland,Northern Ireland +N05000008,Foyle,42,-15,Northern Ireland,Northern Ireland +N05000009,Lagan Valley,44,-18,Northern Ireland,Northern Ireland +N05000010,Mid Ulster,43,-16,Northern Ireland,Northern Ireland +N05000011,Newry and Armagh,44,-19,Northern Ireland,Northern Ireland +N05000012,North Antrim,44,-15,Northern Ireland,Northern Ireland +N05000013,North Down,46,-16,Northern Ireland,Northern Ireland +N05000014,South Antrim,44,-16,Northern Ireland,Northern Ireland +N05000015,South Down,46,-18,Northern Ireland,Northern Ireland +N05000016,Strangford,46,-17,Northern Ireland,Northern Ireland +N05000017,Upper Bann,43,-18,Northern Ireland,Northern Ireland +N05000018,West Tyrone,42,-16,Northern Ireland,Northern Ireland +S14000021,East Renfrewshire,48,-11,Scotland,Scotland +S14000027,Na h-Eileanan an Iar,47,-2,Scotland,Scotland +S14000045,Midlothian,52,-11,Scotland,Scotland +S14000048,North Ayrshire and Arran,48,-10,Scotland,Scotland +S14000051,Orkney and Shetland,51,0,Scotland,Scotland +S14000060,Aberdeen North,52,-3,Scotland,Scotland +S14000061,Aberdeen South,52,-4,Scotland,Scotland +S14000062,Aberdeenshire North and Moray East,51,-3,Scotland,Scotland +S14000063,Airdrie and Shotts,50,-11,Scotland,Scotland +S14000064,Alloa and Grangemouth,50,-7,Scotland,Scotland +S14000065,Angus and Perthshire Glens,50,-5,Scotland,Scotland +S14000066,Arbroath and Broughty Ferry,52,-5,Scotland,Scotland +S14000067,"Argyll, Bute and South Lochaber",49,-5,Scotland,Scotland +S14000068,Bathgate and Linlithgow,51,-9,Scotland,Scotland +S14000069,"Caithness, Sutherland and Easter Ross",50,-2,Scotland,Scotland +S14000070,Coatbridge and Bellshill,50,-12,Scotland,Scotland +S14000071,Cowdenbeath and Kirkcaldy,52,-7,Scotland,Scotland +S14000072,Cumbernauld and Kirkintilloch,50,-8,Scotland,Scotland +S14000073,Dumfries and Galloway,51,-13,Scotland,Scotland +S14000074,"Dumfriesshire, Clydesdale and Tweeddale",52,-13,Scotland,Scotland +S14000075,Dundee Central,50,-6,Scotland,Scotland +S14000076,Dunfermline and Dollar,51,-7,Scotland,Scotland +S14000077,East Kilbride and Strathaven,48,-13,Scotland,Scotland +S14000078,Edinburgh East and Musselburgh,54,-10,Scotland,Scotland +S14000079,Edinburgh North and Leith,53,-9,Scotland,Scotland +S14000080,Edinburgh South,53,-10,Scotland,Scotland +S14000081,Edinburgh South West,52,-10,Scotland,Scotland +S14000082,Edinburgh West,52,-9,Scotland,Scotland +S14000083,Falkirk,51,-8,Scotland,Scotland +S14000084,Glasgow East,51,-10,Scotland,Scotland +S14000085,Glasgow North,49,-9,Scotland,Scotland +S14000086,Glasgow North East,50,-9,Scotland,Scotland +S14000087,Glasgow South,49,-11,Scotland,Scotland +S14000088,Glasgow South West,50,-10,Scotland,Scotland +S14000089,Glasgow West,49,-8,Scotland,Scotland +S14000090,Glenrothes and Mid Fife,52,-6,Scotland,Scotland +S14000091,Gordon and Buchan,50,-4,Scotland,Scotland +S14000092,Hamilton and Clyde Valley,51,-12,Scotland,Scotland +S14000093,Inverclyde and Renfrewshire West,48,-8,Scotland,Scotland +S14000094,"Inverness, Skye and West Ross-shire",49,-3,Scotland,Scotland +S14000095,Livingston,51,-11,Scotland,Scotland +S14000096,Lothian East,53,-11,Scotland,Scotland +S14000097,Mid Dunbartonshire,49,-7,Scotland,Scotland +S14000098,"Moray West, Nairn and Strathspey",49,-4,Scotland,Scotland +S14000099,"Motherwell, Wishaw and Carluke",52,-12,Scotland,Scotland +S14000100,North East Fife,51,-6,Scotland,Scotland +S14000101,Paisley and Renfrewshire North,48,-9,Scotland,Scotland +S14000102,Paisley and Renfrewshire South,49,-10,Scotland,Scotland +S14000103,Perth and Kinross-shire,51,-5,Scotland,Scotland +S14000104,Rutherglen,49,-12,Scotland,Scotland +S14000105,Stirling and Strathallan,49,-6,Scotland,Scotland +S14000106,West Dunbartonshire,48,-7,Scotland,Scotland +S14000107,"Ayr, Carrick and Cumnock",49,-13,Scotland,Scotland +S14000108,"Berwickshire, Roxburgh and Selkirk",53,-12,Scotland,Scotland +S14000109,Central Ayrshire,48,-12,Scotland,Scotland +S14000110,Kilmarnock and Loudoun,50,-13,Scotland,Scotland +S14000111,West Aberdeenshire and Kincardine,51,-4,Scotland,Scotland +W07000081,Aberafan Maesteg,46,-36,Wales,Wales +W07000082,Alyn and Deeside,49,-29,Wales,Wales +W07000083,Bangor Aberconwy,47,-31,Wales,Wales +W07000084,Blaenau Gwent and Rhymney,49,-33,Wales,Wales +W07000085,"Brecon, Radnor and Cwm Tawe",50,-32,Wales,Wales +W07000086,Bridgend,46,-37,Wales,Wales +W07000087,Caerfyrddin,49,-32,Wales,Wales +W07000088,Caerphilly,49,-35,Wales,Wales +W07000089,Cardiff East,48,-37,Wales,Wales +W07000090,Cardiff North,48,-36,Wales,Wales +W07000091,Cardiff South and Penarth,48,-38,Wales,Wales +W07000092,Cardiff West,47,-37,Wales,Wales +W07000093,Ceredigion Preseli,48,-34,Wales,Wales +W07000094,Clwyd East,49,-30,Wales,Wales +W07000095,Clwyd North,48,-30,Wales,Wales +W07000096,Dwyfor Meirionnydd,48,-31,Wales,Wales +W07000097,Gower,44,-37,Wales,Wales +W07000098,Llanelli,45,-36,Wales,Wales +W07000099,Merthyr Tydfil and Aberdare,49,-34,Wales,Wales +W07000100,Mid and South Pembrokeshire,44,-36,Wales,Wales +W07000101,Monmouthshire,50,-36,Wales,Wales +W07000102,Montgomeryshire and Glyndwr,49,-31,Wales,Wales +W07000103,Neath and Swansea East,47,-35,Wales,Wales +W07000104,Newport East,49,-37,Wales,Wales +W07000105,Newport West and Islwyn,49,-36,Wales,Wales +W07000106,Pontypridd,48,-35,Wales,Wales +W07000107,Rhondda and Ogmore,47,-36,Wales,Wales +W07000108,Swansea West,45,-37,Wales,Wales +W07000109,Torfaen,50,-34,Wales,Wales +W07000110,Vale of Glamorgan,47,-38,Wales,Wales +W07000111,Wrexham,50,-30,Wales,Wales +W07000112,Ynys Môn,46,-29,Wales,Wales diff --git a/src/policyengine/data/uk_geography/local_authorities_2023.csv b/src/policyengine/data/uk_geography/local_authorities_2023.csv new file mode 100644 index 00000000..ba0ccac2 --- /dev/null +++ b/src/policyengine/data/uk_geography/local_authorities_2023.csv @@ -0,0 +1,362 @@ +code,name,x,y +E06000001,Hartlepool,8,19 +E06000002,Middlesbrough,9,18 +E06000003,Redcar and Cleveland,9,19 +E06000004,Stockton-on-Tees,8,18 +E06000005,Darlington,7,18 +E06000006,Halton,1,11 +E06000007,Warrington,2,11 +E06000008,Blackburn with Darwen,4,15 +E06000009,Blackpool,2,15 +E06000010,"Kingston upon Hull, City of",10,15 +E06000011,East Riding of Yorkshire,11,16 +E06000012,North East Lincolnshire,11,14 +E06000013,North Lincolnshire,10,14 +E06000014,York,9,17 +E06000015,Derby,6,11 +E06000016,Leicester,8,8 +E06000017,Rutland,10,9 +E06000018,Nottingham,8,10 +E06000019,"Herefordshire, County of",0,8 +E06000020,Telford and Wrekin,2,9 +E06000021,Stoke-on-Trent,3,10 +E06000022,Bath and North East Somerset,1,3 +E06000023,"Bristol, City of",0,3 +E06000024,North Somerset,0,2 +E06000025,South Gloucestershire,1,4 +E06000026,Plymouth,-4,-2 +E06000027,Torbay,-3,-2 +E06000030,Swindon,2,4 +E06000031,Peterborough,11,9 +E06000032,Luton,10,7 +E06000033,Southend-on-Sea,16,6 +E06000034,Thurrock,15,4 +E06000035,Medway,15,1 +E06000036,Bracknell Forest,4,2 +E06000037,West Berkshire,2,2 +E06000038,Reading,2,3 +E06000039,Slough,6,4 +E06000040,Windsor and Maidenhead,4,3 +E06000041,Wokingham,3,3 +E06000042,Milton Keynes,6,5 +E06000043,Brighton and Hove,9,-2 +E06000044,Portsmouth,4,-1 +E06000045,Southampton,2,0 +E06000046,Isle of Wight,1,-2 +E06000047,County Durham,6,18 +E06000049,Cheshire East,4,11 +E06000050,Cheshire West and Chester,3,11 +E06000051,Shropshire,1,9 +E06000052,Cornwall,-5,-2 +E06000053,Isles of Scilly,-7,-3 +E06000054,Wiltshire,1,2 +E06000055,Bedford,9,7 +E06000056,Central Bedfordshire,9,6 +E06000057,Northumberland,5,20 +E06000058,"Bournemouth, Christchurch and Poole",0,0 +E06000059,Dorset,-1,0 +E06000060,Buckinghamshire,5,5 +E06000061,North Northamptonshire,9,9 +E06000062,West Northamptonshire,7,6 +E06000063,Cumberland,4,18 +E06000064,Westmorland and Furness,4,17 +E06000065,North Yorkshire,8,17 +E06000066,Somerset,0,1 +E07000008,Cambridge,12,8 +E07000009,East Cambridgeshire,12,9 +E07000010,Fenland,13,10 +E07000011,Huntingdonshire,10,8 +E07000012,South Cambridgeshire,11,8 +E07000032,Amber Valley,7,11 +E07000033,Bolsover,10,12 +E07000034,Chesterfield,9,12 +E07000035,Derbyshire Dales,7,12 +E07000036,Erewash,7,9 +E07000037,High Peak,7,13 +E07000038,North East Derbyshire,8,12 +E07000039,South Derbyshire,6,10 +E07000040,East Devon,-2,-1 +E07000041,Exeter,-3,-1 +E07000042,Mid Devon,-2,0 +E07000043,North Devon,-3,1 +E07000044,South Hams,-4,-3 +E07000045,Teignbridge,-2,-2 +E07000046,Torridge,-4,-1 +E07000047,West Devon,-3,0 +E07000061,Eastbourne,10,-2 +E07000062,Hastings,13,-2 +E07000063,Lewes,10,-1 +E07000064,Rother,12,-2 +E07000065,Wealden,11,-2 +E07000066,Basildon,14,5 +E07000067,Braintree,14,7 +E07000068,Brentwood,13,5 +E07000069,Castle Point,15,5 +E07000070,Chelmsford,14,6 +E07000071,Colchester,15,8 +E07000072,Epping Forest,12,5 +E07000073,Harlow,13,6 +E07000074,Maldon,15,7 +E07000075,Rochford,15,6 +E07000076,Tendring,16,8 +E07000077,Uttlesford,13,7 +E07000078,Cheltenham,1,5 +E07000079,Cotswold,2,5 +E07000080,Forest of Dean,-1,6 +E07000081,Gloucester,0,6 +E07000082,Stroud,0,5 +E07000083,Tewkesbury,1,6 +E07000084,Basingstoke and Deane,2,1 +E07000085,East Hampshire,4,0 +E07000086,Eastleigh,3,0 +E07000087,Fareham,2,-1 +E07000088,Gosport,3,-1 +E07000089,Hart,3,2 +E07000090,Havant,5,0 +E07000091,New Forest,1,0 +E07000092,Rushmoor,4,1 +E07000093,Test Valley,1,1 +E07000094,Winchester,3,1 +E07000095,Broxbourne,12,6 +E07000096,Dacorum,8,6 +E07000098,Hertsmere,9,5 +E07000099,North Hertfordshire,11,7 +E07000102,Three Rivers,7,5 +E07000103,Watford,8,5 +E07000105,Ashford,12,-1 +E07000106,Canterbury,15,0 +E07000107,Dartford,13,1 +E07000108,Dover,14,-1 +E07000109,Gravesham,14,1 +E07000110,Maidstone,14,0 +E07000111,Sevenoaks,12,0 +E07000112,Folkestone and Hythe,13,-1 +E07000113,Swale,16,0 +E07000114,Thanet,15,-1 +E07000115,Tonbridge and Malling,13,0 +E07000116,Tunbridge Wells,11,-1 +E07000117,Burnley,6,15 +E07000118,Chorley,3,14 +E07000119,Fylde,4,16 +E07000120,Hyndburn,5,15 +E07000121,Lancaster,3,17 +E07000122,Pendle,6,16 +E07000123,Preston,5,16 +E07000124,Ribble Valley,5,17 +E07000125,Rossendale,6,14 +E07000126,South Ribble,3,15 +E07000127,West Lancashire,2,13 +E07000128,Wyre,3,16 +E07000129,Blaby,7,7 +E07000130,Charnwood,8,9 +E07000131,Harborough,8,7 +E07000132,Hinckley and Bosworth,7,8 +E07000133,Melton,11,10 +E07000134,North West Leicestershire,6,9 +E07000135,Oadby and Wigston,9,8 +E07000136,Boston,12,12 +E07000137,East Lindsey,12,13 +E07000138,Lincoln,11,12 +E07000139,North Kesteven,11,11 +E07000140,South Holland,12,11 +E07000141,South Kesteven,12,10 +E07000142,West Lindsey,11,13 +E07000143,Breckland,14,10 +E07000144,Broadland,15,12 +E07000145,Great Yarmouth,15,11 +E07000146,King's Lynn and West Norfolk,13,11 +E07000147,North Norfolk,14,12 +E07000148,Norwich,14,11 +E07000149,South Norfolk,15,10 +E07000170,Ashfield,8,11 +E07000171,Bassetlaw,10,13 +E07000172,Broxtowe,7,10 +E07000173,Gedling,9,10 +E07000174,Mansfield,9,11 +E07000175,Newark and Sherwood,10,11 +E07000176,Rushcliffe,10,10 +E07000177,Cherwell,4,5 +E07000178,Oxford,4,4 +E07000179,South Oxfordshire,5,4 +E07000180,Vale of White Horse,3,4 +E07000181,West Oxfordshire,3,5 +E07000192,Cannock Chase,3,9 +E07000193,East Staffordshire,5,11 +E07000194,Lichfield,4,9 +E07000195,Newcastle-under-Lyme,2,10 +E07000196,South Staffordshire,2,8 +E07000197,Stafford,4,10 +E07000198,Staffordshire Moorlands,5,10 +E07000199,Tamworth,5,9 +E07000200,Babergh,14,8 +E07000202,Ipswich,15,9 +E07000203,Mid Suffolk,14,9 +E07000207,Elmbridge,7,2 +E07000208,Epsom and Ewell,8,0 +E07000209,Guildford,5,1 +E07000210,Mole Valley,6,1 +E07000211,Reigate and Banstead,7,0 +E07000212,Runnymede,5,3 +E07000213,Spelthorne,6,3 +E07000214,Surrey Heath,5,2 +E07000215,Tandridge,9,-1 +E07000216,Waverley,6,0 +E07000217,Woking,6,2 +E07000218,North Warwickshire,6,8 +E07000219,Nuneaton and Bedworth,6,7 +E07000220,Rugby,6,6 +E07000221,Stratford-on-Avon,3,6 +E07000222,Warwick,4,6 +E07000223,Adur,8,-2 +E07000224,Arun,6,-2 +E07000225,Chichester,5,-1 +E07000226,Crawley,8,-1 +E07000227,Horsham,6,-1 +E07000228,Mid Sussex,7,-1 +E07000229,Worthing,7,-2 +E07000234,Bromsgrove,2,7 +E07000235,Malvern Hills,-1,7 +E07000236,Redditch,4,7 +E07000237,Worcester,0,7 +E07000238,Wychavon,2,6 +E07000239,Wyre Forest,1,8 +E07000240,St Albans,10,6 +E07000241,Welwyn Hatfield,11,6 +E07000242,East Hertfordshire,13,8 +E07000243,Stevenage,12,7 +E07000244,East Suffolk,16,10 +E07000245,West Suffolk,13,9 +E08000001,Bolton,4,14 +E08000002,Bury,5,14 +E08000003,Manchester,5,12 +E08000004,Oldham,5,13 +E08000005,Rochdale,7,14 +E08000006,Salford,4,13 +E08000007,Stockport,6,12 +E08000008,Tameside,6,13 +E08000009,Trafford,4,12 +E08000010,Wigan,3,13 +E08000011,Knowsley,2,12 +E08000012,Liverpool,1,13 +E08000013,St. Helens,3,12 +E08000014,Sefton,2,14 +E08000015,Wirral,1,12 +E08000016,Barnsley,8,14 +E08000017,Doncaster,9,14 +E08000018,Rotherham,9,13 +E08000019,Sheffield,8,13 +E08000021,Newcastle upon Tyne,5,19 +E08000022,North Tyneside,6,20 +E08000023,South Tyneside,7,20 +E08000024,Sunderland,7,19 +E08000025,Birmingham,5,8 +E08000026,Coventry,5,6 +E08000027,Dudley,1,7 +E08000028,Sandwell,3,7 +E08000029,Solihull,5,7 +E08000030,Walsall,4,8 +E08000031,Wolverhampton,3,8 +E08000032,Bradford,7,16 +E08000033,Calderdale,7,15 +E08000034,Kirklees,8,15 +E08000035,Leeds,8,16 +E08000036,Wakefield,9,15 +E08000037,Gateshead,6,19 +E09000001,City of London,11,2 +E09000002,Barking and Dagenham,13,3 +E09000003,Barnet,10,5 +E09000004,Bexley,12,1 +E09000005,Brent,10,4 +E09000006,Bromley,11,0 +E09000007,Camden,11,4 +E09000008,Croydon,10,0 +E09000009,Ealing,9,4 +E09000010,Enfield,11,5 +E09000011,Greenwich,11,1 +E09000012,Hackney,12,3 +E09000013,Hammersmith and Fulham,8,3 +E09000014,Haringey,12,4 +E09000015,Harrow,8,4 +E09000016,Havering,14,3 +E09000017,Hillingdon,7,4 +E09000018,Hounslow,7,3 +E09000019,Islington,11,3 +E09000020,Kensington and Chelsea,9,3 +E09000021,Kingston upon Thames,7,1 +E09000022,Lambeth,10,2 +E09000023,Lewisham,10,1 +E09000024,Merton,8,1 +E09000025,Newham,13,2 +E09000026,Redbridge,14,4 +E09000027,Richmond upon Thames,8,2 +E09000028,Southwark,9,1 +E09000029,Sutton,9,0 +E09000030,Tower Hamlets,12,2 +E09000031,Waltham Forest,13,4 +E09000032,Wandsworth,9,2 +E09000033,Westminster,10,3 +N09000001,Antrim and Newtownabbey,-4,16 +N09000002,"Armagh City, Banbridge and Craigavon",-5,16 +N09000003,Belfast,-4,17 +N09000004,Causeway Coast and Glens,-5,18 +N09000005,Derry City and Strabane,-6,17 +N09000006,Fermanagh and Omagh,-6,16 +N09000007,Lisburn and Castlereagh,-5,15 +N09000008,Mid and East Antrim,-4,18 +N09000009,Mid Ulster,-5,17 +N09000010,"Newry, Mourne and Down",-4,15 +N09000011,Ards and North Down,-3,16 +S12000005,Clackmannanshire,2,24 +S12000006,Dumfries and Galloway,4,20 +S12000008,East Ayrshire,3,20 +S12000010,East Lothian,5,22 +S12000011,East Renfrewshire,2,20 +S12000013,Na h-Eileanan Siar,-1,27 +S12000014,Falkirk,2,23 +S12000017,Highland,1,26 +S12000018,Inverclyde,0,21 +S12000019,Midlothian,3,21 +S12000020,Moray,2,26 +S12000021,North Ayrshire,1,20 +S12000023,Orkney Islands,4,28 +S12000026,Scottish Borders,4,21 +S12000027,Shetland Islands,5,30 +S12000028,South Ayrshire,1,19 +S12000029,South Lanarkshire,2,21 +S12000030,Stirling,1,24 +S12000033,Aberdeen City,4,26 +S12000034,Aberdeenshire,3,26 +S12000035,Argyll and Bute,0,24 +S12000036,City of Edinburgh,4,22 +S12000038,Renfrewshire,1,22 +S12000039,West Dunbartonshire,0,23 +S12000040,West Lothian,3,22 +S12000041,Angus,2,25 +S12000042,Dundee City,3,25 +S12000045,East Dunbartonshire,1,23 +S12000047,Fife,3,24 +S12000048,Perth and Kinross,1,25 +S12000049,Glasgow City,1,21 +S12000050,North Lanarkshire,2,22 +W06000001,Isle of Anglesey,-2,12 +W06000002,Gwynedd,-2,10 +W06000003,Conwy,-1,10 +W06000004,Denbighshire,0,10 +W06000005,Flintshire,0,11 +W06000006,Wrexham,1,10 +W06000008,Ceredigion,-2,9 +W06000009,Pembrokeshire,-5,6 +W06000010,Carmarthenshire,-4,6 +W06000011,Swansea,-4,5 +W06000012,Neath Port Talbot,-3,5 +W06000013,Bridgend,-3,6 +W06000014,Vale of Glamorgan,-2,4 +W06000015,Cardiff,-2,5 +W06000016,Rhondda Cynon Taf,-3,7 +W06000018,Caerphilly,-2,6 +W06000019,Blaenau Gwent,0,9 +W06000020,Torfaen,-2,7 +W06000021,Monmouthshire,-1,8 +W06000022,Newport,-1,5 +W06000023,Powys,-1,9 +W06000024,Merthyr Tydfil,-2,8 diff --git a/src/policyengine/data/uk_geography_assets.py b/src/policyengine/data/uk_geography_assets.py index 14257314..d4d6bf78 100644 --- a/src/policyengine/data/uk_geography_assets.py +++ b/src/policyengine/data/uk_geography_assets.py @@ -2,6 +2,7 @@ import os from dataclasses import dataclass +from importlib.resources import files from pathlib import Path from typing import Optional, Sequence, Union @@ -51,6 +52,29 @@ class UKGeographyAssetPaths: lookup_csv_filename="local_authorities_2021.csv", ) +# Area lists shipped with the package: code, name and hex-map position for +# every constituency (2024 boundaries) and local authority (April 2023 +# districts). They define the constituency and local-authority regions and +# label geography outputs. They are not the lookup CSVs paired with the weight +# matrices above: a matrix's paired CSV gives each area its row, so it must +# stay the file the matrix was built with. See data/uk_geography/SOURCES.md. +UK_AREA_LIST_FILES = { + "constituency": "constituencies_2024.csv", + "local_authority": "local_authorities_2023.csv", +} + + +def packaged_uk_area_list_path(geography_type: str) -> str: + """Return the packaged area list for a UK geography type.""" + try: + filename = UK_AREA_LIST_FILES[geography_type] + except KeyError: + raise ValueError( + f"No packaged UK area list for geography type {geography_type!r}. " + f"Known types: {sorted(UK_AREA_LIST_FILES)}" + ) from None + return str(files("policyengine").joinpath("data", "uk_geography", filename)) + def _env_path(name: str) -> Optional[Path]: value = os.environ.get(name) diff --git a/src/policyengine/outputs/constituency_impact.py b/src/policyengine/outputs/constituency_impact.py index 2dbd04ee..85361ca6 100644 --- a/src/policyengine/outputs/constituency_impact.py +++ b/src/policyengine/outputs/constituency_impact.py @@ -71,13 +71,14 @@ def compute_uk_constituency_impacts( reform_simulation: Completed reform simulation. weight_matrix_path: Deprecated and ignored. Constituency outputs now group by ``constituency_code_oa`` on the household output. - constituency_csv_path: Optional path to constituencies_2024.csv. - If omitted, standard local paths are checked before downloading - from GCS. If still unavailable, results use geography codes as names. + constituency_csv_path: Optional path to a lookup CSV with ``code``, + ``name``, ``x`` and ``y`` columns. If omitted, the packaged 2024 + constituency list supplies names and hex positions. Codes missing + from the lookup use the code as the name. year: Deprecated and ignored. asset_strategies: Deprecated and ignored. - download_missing_assets: Whether to download the optional lookup CSV - from GCS when no local CSV is found. + download_missing_assets: Deprecated and ignored; the lookup is + packaged. Returns: ConstituencyImpact with constituency_results populated. diff --git a/src/policyengine/outputs/local_authority_impact.py b/src/policyengine/outputs/local_authority_impact.py index 40d50e8f..eeb3142e 100644 --- a/src/policyengine/outputs/local_authority_impact.py +++ b/src/policyengine/outputs/local_authority_impact.py @@ -71,13 +71,15 @@ def compute_uk_local_authority_impacts( reform_simulation: Completed reform simulation. weight_matrix_path: Deprecated and ignored. Local-authority outputs now group by ``la_code_oa`` on the household output. - local_authority_csv_path: Optional path to local_authorities_2021.csv. - If omitted, standard local paths are checked before downloading - from GCS. If still unavailable, results use geography codes as names. + local_authority_csv_path: Optional path to a lookup CSV with + ``code``, ``name``, ``x`` and ``y`` columns. If omitted, the + packaged April 2023 local-authority list supplies names and + hex positions. Codes missing from the lookup use the code as + the name. year: Deprecated and ignored. asset_strategies: Deprecated and ignored. - download_missing_assets: Whether to download the optional lookup CSV - from GCS when no local CSV is found. + download_missing_assets: Deprecated and ignored; the lookup is + packaged. Returns: LocalAuthorityImpact with local_authority_results populated. diff --git a/src/policyengine/outputs/uk_geography_impact.py b/src/policyengine/outputs/uk_geography_impact.py index ff2528ea..404778bd 100644 --- a/src/policyengine/outputs/uk_geography_impact.py +++ b/src/policyengine/outputs/uk_geography_impact.py @@ -6,9 +6,11 @@ import pandas as pd from policyengine.data.uk_geography_assets import ( + UK_AREA_LIST_FILES, UKGeographyAssetSpec, default_download_dir, default_local_search_dirs, + packaged_uk_area_list_path, ) @@ -26,7 +28,14 @@ def resolve_uk_geography_lookup_csv_path( lookup_csv_path: Optional[str] = None, download_missing_assets: bool = True, ) -> Optional[str]: - """Resolve a UK geography lookup CSV without requiring a weight matrix.""" + """Resolve a UK geography lookup CSV without requiring a weight matrix. + + An explicit ``lookup_csv_path`` wins. Constituencies and local + authorities otherwise use the area list shipped with the package, so + labels never depend on a local file or on the unversioned GCS bucket + (``download_missing_assets`` then has no effect). Other geography types + keep the local search and GCS fallback. + """ if lookup_csv_path: path = Path(lookup_csv_path).expanduser() if not path.is_file(): @@ -36,6 +45,9 @@ def resolve_uk_geography_lookup_csv_path( ) return str(path) + if spec.geography_type in UK_AREA_LIST_FILES: + return packaged_uk_area_list_path(spec.geography_type) + for search_dir in default_local_search_dirs(): path = search_dir / spec.lookup_csv_filename if path.is_file(): diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 3b4b27f1..1ea1bca5 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -20,13 +20,15 @@ data_producer="populace", manifest_uri=( "hf://dataset/policyengine/populace-uk-private" - "@populace-uk-2023-0cdbb27-c239dfe51c11-20260615T201302Z" - "/releases/populace-uk-2023-0cdbb27-c239dfe51c11-20260615T201302Z" - "/release_manifest.json" + "@microcosm-uk-2024-25-national-20261002T230158Z-5c6b3f68" + "/releases/microcosm-uk-2024-25-national/release_manifest.json" ), ) print(result.summary()) +Pin the cut tag (``@``), never a branch: a release line's directory +(``releases//``) is rewritten by each cut. + ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose data release predates release manifests need a dedicated data-producer strategy @@ -46,11 +48,17 @@ import requests +from policyengine.data.uk_geography_assets import ( + CONSTITUENCY_ASSET_SPEC, + LOCAL_AUTHORITY_ASSET_SPEC, +) from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, DataReleaseManifest, + RegionalReleaseKind, _specifier_matches, fetch_pypi_wheel_metadata, + release_build_id, ) MANIFEST_URI_PATTERN = re.compile( @@ -74,6 +82,29 @@ # source-enrichment releases; `populace-data` is its earlier name. Both denote # the same producer, so every Populace-specific rule below must accept either. POPULACE_DATA_PACKAGES = frozenset({"populace-data", "microcosm-data"}) +# The Hugging Face repos each country's Populace/Microcosm releases publish to. +POPULACE_RELEASE_REPOS = { + "us": "policyengine/populace-us", + "uk": "policyengine/populace-uk-private", +} +# Region types a regional release serves. Their regions load the routed dataset +# and keep row-filtering it on their own geography column. +LOCAL_AREA_REGION_TYPES = { + "us": ("state", "congressional_district"), + "uk": ("constituency", "local_authority"), +} +REGIONAL_RELEASE_KINDS: tuple[RegionalReleaseKind, ...] = ( + "local_area", + "area_coded_national", +) +# A UK area-coded national release (the enhanced FRS) serves constituency and +# local-authority runs with these weight matrices as well as its households. +# Consumers bind a matrix only to the release of the dataset it reweights, so +# they travel with that dataset. +UK_AREA_WEIGHT_MATRIX_PATHS = ( + CONSTITUENCY_ASSET_SPEC.weight_matrix_filename, + LOCAL_AUTHORITY_ASSET_SPEC.weight_matrix_filename, +) # Artifact kinds published beside the release manifest rather than at the repo # root, so their manifest paths are relative to the release directory. RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) @@ -252,6 +283,23 @@ def validate_release_manifest( if not artifact.sha256: warnings.append(f"artifact {name!r} has no sha256") + basis, compatibility_warnings = model_compatibility_basis( + manifest, model_package, model_version + ) + return basis, warnings + compatibility_warnings + + +def model_compatibility_basis( + manifest: DataReleaseManifest, + model_package: str, + model_version: str, +) -> tuple[str, list[str]]: + """Return the basis on which a release serves a model version. + + Raises when the release was neither built with ``model_version`` nor + claims compatibility with it. Returns ``(compatibility_basis, warnings)``. + """ + warnings: list[str] = [] built_with = None if manifest.build is not None and manifest.build.built_with_model_package: build_pkg = manifest.build.built_with_model_package @@ -389,11 +437,9 @@ def should_validate_vendored_artifacts( manifest: DataReleaseManifest, uri_parts: dict, ) -> bool: - return ( - country == "us" - and manifest.data_package.name in POPULACE_DATA_PACKAGES - and uri_parts["repo_id"] == "policyengine/populace-us" - ) + return manifest.data_package.name in POPULACE_DATA_PACKAGES and uri_parts[ + "repo_id" + ] == POPULACE_RELEASE_REPOS.get(country) def _state_code_from_path(path: str, artifact_prefix: str) -> Optional[str]: @@ -472,29 +518,87 @@ def merge_us_state_release_manifest( return DataReleaseManifest.model_validate(primary_payload) -def merge_us_local_area_release_manifest( +def _local_area_region_types(country: str) -> tuple[str, ...]: + try: + return LOCAL_AREA_REGION_TYPES[country] + except KeyError: + raise CertificationError( + f"Regional data releases are not supported for country {country!r}." + ) from None + + +def _route_region_types( + country: str, + primary_manifest: DataReleaseManifest, + primary_payload: dict, + path: str, + release_label: str, +) -> None: + """Point the country's local region types at ``path`` in the payload.""" + metadata = primary_payload.setdefault("metadata", {}) + region_datasets = metadata.setdefault("region_datasets", {}) + default_dataset = primary_manifest.default_datasets["national"] + default_artifact = primary_manifest.artifacts[default_dataset] + region_datasets.setdefault( + "national", + {"path_template": default_artifact.path}, + ) + regional_template = {"path_template": path} + for region_type in _local_area_region_types(country): + existing = region_datasets.get(region_type) + if existing is not None and existing != regional_template: + raise CertificationError( + f"{release_label} conflicts with the {region_type!r} " + "dataset template in the primary manifest." + ) + region_datasets[region_type] = regional_template + + +def _add_regional_artifact( + merged_artifacts: dict, + name: str, + artifact, + regional_repo_type: Optional[Literal["model", "dataset"]], +) -> None: + if name in merged_artifacts: + raise CertificationError( + f"Regional artifact {name!r} conflicts with the primary manifest." + ) + if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + raise CertificationError( + f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + ) + artifact_payload = artifact.model_dump(mode="json", exclude_none=True) + if regional_repo_type is not None: + artifact_payload["repo_type"] = regional_repo_type + merged_artifacts[name] = artifact_payload + + +def merge_local_area_release_manifest( + country: str, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, regional_repo_type: Optional[Literal["model", "dataset"]] = None, ) -> DataReleaseManifest: - """Add one certified local-area dataset to a primary US release. + """Add one certified local-area dataset to a primary release. - The supplemental release remains non-default nationally. State and - congressional-district simulations load the shared artifact and apply the - existing row-filtering strategy at runtime. + The supplemental release remains non-default nationally. The country's + local region types (US states and congressional districts, UK + constituencies and local authorities) load the shared artifact and apply + their existing row-filtering strategy at runtime. """ + label = f"{country.upper()} local-area release" + _local_area_region_types(country) if regional_manifest.dataset_role != "non_default_local_area": raise CertificationError( - "US local-area release must declare dataset_role='non_default_local_area'." + f"{label} must declare dataset_role='non_default_local_area'." ) if regional_manifest.is_default is not False: - raise CertificationError("US local-area release must declare is_default=false.") + raise CertificationError(f"{label} must declare is_default=false.") if regional_manifest.default_datasets: - raise CertificationError( - "US local-area release must not declare default datasets." - ) + raise CertificationError(f"{label} must not declare default datasets.") local_area_artifacts = [ (name, artifact) @@ -503,82 +607,200 @@ def merge_us_local_area_release_manifest( ] if len(local_area_artifacts) != 1: raise CertificationError( - "US local-area release must contain exactly one microdata artifact." + f"{label} must contain exactly one microdata artifact." ) artifact_name, artifact = local_area_artifacts[0] if not artifact.path.endswith(".h5"): - raise CertificationError("US local-area artifact must be an H5 file.") + raise CertificationError( + f"{country.upper()} local-area artifact must be an H5 file." + ) if not artifact.repo_id or not artifact.revision or not artifact.sha256: raise CertificationError( - "US local-area artifact must declare repo_id, revision, and sha256." + f"{country.upper()} local-area artifact must declare repo_id, " + "revision, and sha256." ) primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) merged_artifacts = primary_payload.setdefault("artifacts", {}) - if artifact_name in merged_artifacts: + _add_regional_artifact( + merged_artifacts, artifact_name, artifact, regional_repo_type + ) + _route_region_types( + country, primary_manifest, primary_payload, artifact.path, label + ) + + return DataReleaseManifest.model_validate(primary_payload) + + +def merge_us_local_area_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, +) -> DataReleaseManifest: + """Add one certified local-area dataset to a primary US release.""" + + return merge_local_area_release_manifest( + "us", + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ) + + +def merge_area_coded_national_release_manifest( + country: str, + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, +) -> DataReleaseManifest: + """Route UK area regions to another release's national dataset. + + This serves a national-first switch: the primary release's national + dataset carries no area codes, and the regional manifest is a certified + national release (the enhanced FRS) whose households do. Its national + dataset and its area weight matrices join the certified datasets with + their own repo and revision pins, and constituency and local-authority + regions load that dataset. The regional release's own default stays + non-default here. + """ + + label = f"{country.upper()} area-coded national release" + if country != "uk": raise CertificationError( - f"Regional artifact {artifact_name!r} conflicts with the primary manifest." + "Area-coded national regional releases are only supported for the UK." ) - if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + if regional_manifest.dataset_role not in (None, "national_default"): raise CertificationError( - f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + f"{label} must be a national release, not " + f"dataset_role={regional_manifest.dataset_role!r}. Certify a " + "local-area release with --regional-release-kind local_area." + ) + area_dataset = regional_manifest.default_datasets.get("national") + if area_dataset is None or area_dataset not in regional_manifest.artifacts: + raise CertificationError( + f"{label} must declare a national default dataset among its artifacts." + ) + area_artifact = regional_manifest.artifacts[area_dataset] + if area_artifact.kind != "microdata" or not area_artifact.path.endswith(".h5"): + raise CertificationError( + f"{label} national dataset {area_dataset!r} must be an H5 microdata " + "artifact." ) - artifact_payload = artifact.model_dump( - mode="json", - exclude_none=True, - ) - if regional_repo_type is not None: - artifact_payload["repo_type"] = regional_repo_type - merged_artifacts[artifact_name] = artifact_payload - metadata = primary_payload.setdefault("metadata", {}) - region_datasets = metadata.setdefault("region_datasets", {}) - default_dataset = primary_manifest.default_datasets["national"] - default_artifact = primary_manifest.artifacts[default_dataset] - region_datasets.setdefault( - "national", - {"path_template": default_artifact.path}, - ) - regional_template = {"path_template": artifact.path} - for region_type in ("state", "congressional_district"): - existing = region_datasets.get(region_type) - if existing is not None and existing != regional_template: + carried = {area_dataset: area_artifact} + weights = { + name: artifact + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "weights" + } + weight_paths = {artifact.path for artifact in weights.values()} + missing = [path for path in UK_AREA_WEIGHT_MATRIX_PATHS if path not in weight_paths] + if missing: + raise CertificationError( + f"{label} must carry the area weight matrices {missing}." + ) + carried.update(weights) + for name, artifact in carried.items(): + if not artifact.repo_id or not artifact.revision or not artifact.sha256: raise CertificationError( - f"US local-area release conflicts with the {region_type!r} " - "dataset template in the primary manifest." + f"{label} artifact {name!r} must declare repo_id, revision, and sha256." ) - region_datasets[region_type] = regional_template + + primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) + merged_artifacts = primary_payload.setdefault("artifacts", {}) + for name, artifact in carried.items(): + _add_regional_artifact(merged_artifacts, name, artifact, regional_repo_type) + _route_region_types( + country, primary_manifest, primary_payload, area_artifact.path, label + ) return DataReleaseManifest.model_validate(primary_payload) -def merge_us_regional_release_manifest( +def merge_regional_release_manifest( + country: str, primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, + release_kind: RegionalReleaseKind = "local_area", regional_repo_type: Optional[Literal["model", "dataset"]] = None, artifact_prefix: str = "states/", path_template: str = "states/{state_code}.h5", -) -> DataReleaseManifest: - """Merge a typed shared local-area release or a legacy per-state release.""" +) -> tuple[DataReleaseManifest, Optional[RegionalReleaseKind]]: + """Merge a regional release into a primary release. + Returns the merged manifest and the routing kind to record. The kind is + ``None`` only for a legacy US per-state release. + """ + + if release_kind not in REGIONAL_RELEASE_KINDS: + raise CertificationError( + f"Unknown regional release kind {release_kind!r}; expected one of " + f"{list(REGIONAL_RELEASE_KINDS)}." + ) + if release_kind == "area_coded_national": + return ( + merge_area_coded_national_release_manifest( + country, + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ), + release_kind, + ) if regional_manifest.dataset_role == "non_default_local_area": - return merge_us_local_area_release_manifest( - primary_manifest, - regional_manifest, - regional_repo_type=regional_repo_type, + return ( + merge_local_area_release_manifest( + country, + primary_manifest, + regional_manifest, + regional_repo_type=regional_repo_type, + ), + "local_area", + ) + if country == "us" and regional_manifest.dataset_role is None: + return ( + merge_us_state_release_manifest( + primary_manifest, + regional_manifest, + artifact_prefix=artifact_prefix, + path_template=path_template, + ), + None, ) - if regional_manifest.dataset_role is not None: + if country == "uk" and regional_manifest.dataset_role in (None, "national_default"): raise CertificationError( - "Unsupported US regional release dataset_role " - f"{regional_manifest.dataset_role!r}." + "UK regional release is a national release, not a local-area one. " + "To route constituency and local-authority regions to its national " + "dataset, pass --regional-release-kind area_coded_national." ) - return merge_us_state_release_manifest( + raise CertificationError( + f"Unsupported {country.upper()} regional release dataset_role " + f"{regional_manifest.dataset_role!r}." + ) + + +def merge_us_regional_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, + artifact_prefix: str = "states/", + path_template: str = "states/{state_code}.h5", +) -> DataReleaseManifest: + """Merge a typed shared local-area release or a legacy per-state release.""" + + manifest, _ = merge_regional_release_manifest( + "us", primary_manifest, regional_manifest, + regional_repo_type=regional_repo_type, artifact_prefix=artifact_prefix, path_template=path_template, ) + return manifest def build_country_manifest_payload( @@ -663,9 +885,10 @@ def build_country_manifest_payload( certified_artifact["sha256"] = default_artifact.sha256 build = manifest.build if build is not None: - if build.build_id: - certification["data_build_id"] = build.build_id - certified_artifact["build_id"] = build.build_id + build_id = release_build_id(build) + if build_id: + certification["data_build_id"] = build_id + certified_artifact["build_id"] = build_id model_build = build.built_with_model_package if model_build is not None: certification["built_with_model_version"] = model_build.version @@ -711,6 +934,7 @@ def build_bundle_data_release_payload( uri_parts: dict, regional_manifest_uri: Optional[str] = None, regional_uri_parts: Optional[dict] = None, + regional_release_kind: Optional[RegionalReleaseKind] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -728,6 +952,8 @@ def build_bundle_data_release_payload( payload["regional_release_manifest_uri"] = https_manifest_url( regional_uri_parts ) + if regional_release_kind is not None: + payload["regional_release_kind"] = regional_release_kind payload["default_dataset_uri"] = certified_artifact.get("uri") return payload @@ -747,6 +973,7 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -766,6 +993,7 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -788,30 +1016,51 @@ def certify( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) regional_uri_parts = None + recorded_release_kind = None + regional_artifacts: dict = {} + regional_warnings: list[str] = [] if regional_manifest_uri is not None: - if country != "us": - raise CertificationError( - "Regional data release manifests are only supported for US " - "Populace certification." - ) regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token ) - manifest = merge_us_regional_release_manifest( + # The regional release serves the same model, so it must make its + # own compatibility claim for the pinned version. + try: + _, regional_warnings = model_compatibility_basis( + regional_manifest, model_package, model_version + ) + except CertificationError as exc: + raise CertificationError( + f"Regional release {regional_manifest_uri}: {exc}" + ) from exc + regional_warnings = [ + f"regional release: {warning}" for warning in regional_warnings + ] + primary_artifact_names = set(manifest.artifacts) + manifest, recorded_release_kind = merge_regional_release_manifest( + country, manifest, regional_manifest, + release_kind=regional_release_kind, regional_repo_type=regional_uri_parts["repo_type"], artifact_prefix=regional_artifact_prefix, path_template=regional_path_template, ) + regional_artifacts = { + name: artifact + for name, artifact in manifest.artifacts.items() + if name not in primary_artifact_names + } compatibility_basis, warnings = validate_release_manifest( manifest, model_package, model_version ) + warnings = warnings + regional_warnings for filename in required_supplemental_release_files( country, manifest, uri_parts @@ -827,6 +1076,12 @@ def certify( raise CertificationError( f"Certified dataset artifact is not reachable: {default_artifact.uri}" ) + if check_artifacts and recorded_release_kind is not None: + for name, artifact in regional_artifacts.items(): + if not head_artifact(artifact, token=token): + raise CertificationError( + f"Regional artifact {name!r} is not reachable: {artifact.uri}" + ) model_wheel = fetch_pypi_wheel_metadata(model_package, model_version) country_payload = build_country_manifest_payload( @@ -857,6 +1112,7 @@ def certify( uri_parts=uri_parts, regional_manifest_uri=regional_manifest_uri, regional_uri_parts=regional_uri_parts, + regional_release_kind=recorded_release_kind, ), manifest_sha256, warnings, @@ -898,12 +1154,33 @@ def certify_data_release( regional_manifest_uri: Optional[str] = None, regional_artifact_prefix: str = "states/", regional_path_template: str = "states/{state_code}.h5", + regional_release_kind: RegionalReleaseKind = "local_area", + drop_regional_release: bool = False, ) -> CertificationResult: if country not in COUNTRY_MODEL_PACKAGES: raise CertificationError(f"Unknown country {country!r}.") model_package = COUNTRY_MODEL_PACKAGES[country] model_version = model_version or installed_model_version(model_package) strategy = certification_strategy(country, data_producer) + bundle_path = bundle_path or default_bundle_source_path() + bundle = json.loads(bundle_path.read_text()) + + # Certification rewrites data_releases. wholesale, so a run + # without the regional manifest would silently drop the regional routing. + current_regional_uri = ((bundle.get("data_releases") or {}).get(country) or {}).get( + "regional_source_manifest_uri" + ) + if ( + current_regional_uri is not None + and regional_manifest_uri is None + and not drop_regional_release + ): + raise CertificationError( + f"The bundle's {country.upper()} data release routes regions to " + f"{current_regional_uri}. Pass --regional-manifest-uri (that release " + "or its replacement) to keep regional routing, or " + "--drop-regional-release to remove it." + ) data_release, manifest_sha256, warnings = strategy.certify( country=country, @@ -915,9 +1192,8 @@ def certify_data_release( regional_manifest_uri=regional_manifest_uri, regional_artifact_prefix=regional_artifact_prefix, regional_path_template=regional_path_template, + regional_release_kind=regional_release_kind, ) - bundle_path = bundle_path or default_bundle_source_path() - bundle = json.loads(bundle_path.read_text()) bundle.setdefault("data_releases", {})[country] = data_release bundle.setdefault("countries", {}).setdefault(country, {})["model_package"] = ( model_package diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 0995c0d5..9029b5f0 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -47,11 +47,22 @@ class BuiltWithModelPackage(PackageVersion): class DataBuildInfo(BaseModel): build_id: Optional[str] = None + cut_tag: Optional[str] = None + """The immutable tag of one cut of a release line. Microcosm keeps + ``build_id`` equal to the line name for every cut, so the cut tag is what + tells two cuts apart (see ``release_build_id``).""" built_at: Optional[str] = None built_with_core_package: Optional[PackageVersion] = None built_with_model_package: Optional[BuiltWithModelPackage] = None +def release_build_id(build: Optional[DataBuildInfo]) -> Optional[str]: + """Identify one data build: its cut tag when it has one, else its build id.""" + if build is None: + return None + return build.cut_tag or build.build_id + + class ArtifactPathReference(BaseModel): path: str revision: Optional[str] = None @@ -165,6 +176,9 @@ class CertifiedDataArtifact(BaseModel): build_id: Optional[str] = None +RegionalReleaseKind = Literal["local_area", "area_coded_national"] + + class CountryReleaseManifest(BaseModel): schema_version: int = 1 bundle_id: Optional[str] = None @@ -176,6 +190,11 @@ class CountryReleaseManifest(BaseModel): default_dataset: str datasets: dict[str, ArtifactPathReference] = Field(default_factory=dict) region_datasets: dict[str, ArtifactPathTemplate] = Field(default_factory=dict) + regional_release_kind: Optional[RegionalReleaseKind] = None + """How certification routed the local region types, when it did: + ``local_area`` for a non-default local-area release, and + ``area_coded_national`` for another release's national dataset that + carries area codes (and its area weight matrices).""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) @@ -457,11 +476,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="exact_build_model_version", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=built_with_model.version, built_with_model_git_sha=built_with_model.git_sha, data_build_fingerprint=built_with_model.data_build_fingerprint, @@ -476,11 +491,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="matching_data_build_fingerprint", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=built_with_model.version, built_with_model_git_sha=built_with_model.git_sha, data_build_fingerprint=built_with_model.data_build_fingerprint, @@ -496,11 +507,7 @@ def certify_data_release_compatibility( return DataCertification( compatibility_basis="legacy_compatible_model_package", certified_for_model_version=runtime_model_version, - data_build_id=( - data_release_manifest.build.build_id - if data_release_manifest.build is not None - else None - ), + data_build_id=release_build_id(data_release_manifest.build), built_with_model_version=( built_with_model.version if built_with_model is not None else None ), diff --git a/src/policyengine/provenance/uk_uc_floor.py b/src/policyengine/provenance/uk_uc_floor.py new file mode 100644 index 00000000..2fe476b3 --- /dev/null +++ b/src/policyengine/provenance/uk_uc_floor.py @@ -0,0 +1,159 @@ +"""Universal Credit coverage floor for UK certification candidates. + +In August 2026 (#491) the UK default was reverted after a certified dataset +carried about half of Universal Credit's caseload and spend: its calibration +had no benefit targets, and no check compared its outputs with administrative +totals. This is the standing floor #491 proposed. A candidate dataset's +benefit units with Universal Credit, and its Universal Credit spend, in Great +Britain must each be within 10% of DWP's and the OBR's figures for the same +year. + +Both benchmarks cover Great Britain (Northern Ireland's Universal Credit is +administered separately), so Northern Ireland households are excluded from +the measurement. The receipt holds weighted national totals only. + +Run it with ``scripts/check_uk_uc_floor.py``. +""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Optional + +import numpy as np +from pydantic import BaseModel + +UC_FLOOR_RESOURCE = ("data", "qualification", "uk_uc_floor.json") +NORTHERN_IRELAND_REGION = "NORTHERN_IRELAND" + + +class UCFloorBenchmark(BaseModel): + value: float + source: str + url: str + + +class UCFloorYear(BaseModel): + benefit_units: UCFloorBenchmark + spend: UCFloorBenchmark + + +class UCFloorBenchmarks(BaseModel): + schema_version: int + description: str + tolerance: float + geography: str + years: dict[int, UCFloorYear] + + +class UCFloorMeasure(BaseModel): + name: str + measured: float + benchmark: float + ratio: float + passed: bool + source: str + + +class UCFloorReceipt(BaseModel): + schema_version: int = 1 + check: str = "uk_uc_floor" + dataset: str + year: int + geography: str + tolerance: float + model_version: Optional[str] = None + measures: list[UCFloorMeasure] + passed: bool + + +def load_uc_floor_benchmarks() -> UCFloorBenchmarks: + resource = files("policyengine").joinpath(*UC_FLOOR_RESOURCE) + return UCFloorBenchmarks.model_validate(json.loads(resource.read_text())) + + +def measure_great_britain_uc(simulation, year: int) -> tuple[float, float]: + """Return weighted (benefit units with UC, UC spend) in Great Britain.""" + universal_credit = np.asarray( + simulation.calculate("universal_credit", year), dtype=float + ) + weights = np.asarray(simulation.calculate("benunit_weight", year), dtype=float) + northern_ireland_household = ( + np.asarray(simulation.calculate("region", year)).astype(str) + == NORTHERN_IRELAND_REGION + ).astype(float) + # Mapping to benefit units spreads a household's value over its members; + # any positive share marks a benefit unit in a Northern Ireland household. + northern_ireland = ( + np.asarray( + simulation.map_result(northern_ireland_household, "household", "benunit"), + dtype=float, + ) + > 0 + ) + great_britain = ~northern_ireland + benefit_units = float(weights[great_britain & (universal_credit > 0)].sum()) + spend = float((universal_credit * weights)[great_britain].sum()) + return benefit_units, spend + + +def evaluate_uc_floor( + *, + benefit_units: float, + spend: float, + year: int, + dataset: str, + model_version: Optional[str] = None, + benchmarks: Optional[UCFloorBenchmarks] = None, +) -> UCFloorReceipt: + benchmarks = benchmarks or load_uc_floor_benchmarks() + if year not in benchmarks.years: + raise ValueError( + f"No Universal Credit floor benchmarks for {year}; " + f"available years: {sorted(benchmarks.years)}" + ) + year_benchmarks = benchmarks.years[year] + measures = [] + for name, measured, benchmark in ( + ("benefit_units", benefit_units, year_benchmarks.benefit_units), + ("spend", spend, year_benchmarks.spend), + ): + ratio = measured / benchmark.value + measures.append( + UCFloorMeasure( + name=name, + measured=measured, + benchmark=benchmark.value, + ratio=ratio, + passed=abs(ratio - 1) <= benchmarks.tolerance, + source=benchmark.source, + ) + ) + return UCFloorReceipt( + dataset=dataset, + year=year, + geography=benchmarks.geography, + tolerance=benchmarks.tolerance, + model_version=model_version, + measures=measures, + passed=all(measure.passed for measure in measures), + ) + + +def check_uc_floor(dataset_path: str, year: int, *, dataset: str) -> UCFloorReceipt: + """Measure a UK dataset file and evaluate it against the floor.""" + from importlib.metadata import version + + from policyengine_uk import Microsimulation + + benefit_units, spend = measure_great_britain_uc( + Microsimulation(dataset=dataset_path), year + ) + return evaluate_uc_floor( + benefit_units=benefit_units, + spend=spend, + year=year, + dataset=dataset, + model_version=version("policyengine-uk"), + ) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index b18a77f0..1a148ee5 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -14,6 +14,9 @@ CertificationError, build_country_manifest_payload, certify_data_release, + merge_area_coded_national_release_manifest, + merge_local_area_release_manifest, + merge_regional_release_manifest, merge_us_local_area_release_manifest, merge_us_regional_release_manifest, merge_us_state_release_manifest, @@ -22,7 +25,10 @@ should_validate_vendored_artifacts, validate_release_manifest, ) -from policyengine.provenance.manifest import DataReleaseManifest +from policyengine.provenance.manifest import ( + CountryReleaseManifest, + DataReleaseManifest, +) TAG = "populace-us-2024-aaaaaaa-20260101" MANIFEST_URI = ( @@ -171,6 +177,13 @@ def _local_area_release_manifest_payload() -> dict: "dataset_role": "non_default_local_area", "is_default": False, "default_datasets": {}, + "build": { + "build_id": US_LOCAL_AREA_TAG, + "built_with_model_package": { + "name": "policyengine-us", + "version": "1.723.0", + }, + }, "artifacts": { "populace_us_2024_acs_local": { "kind": "microdata", @@ -222,6 +235,12 @@ def _uk_release_manifest_payload() -> dict: return payload +def _tmp_bundle(tmp_path: Path) -> Path: + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text(json.dumps(_bundle_source_payload()) + "\n") + return bundle_path + + def _bundle_source_payload() -> dict: return { "schema_version": 2, @@ -697,6 +716,7 @@ def test__given_us_local_area_manifest__then_certifies_shared_regional_dataset( assert ( release["datasets"]["populace_us_2024_acs_local"]["repo_type"] == "dataset" ) + assert release["regional_release_kind"] == "local_area" assert result.dataset_count == 5 def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsupported( @@ -790,6 +810,7 @@ def test__given_missing_populace_us_source_coverage__then_raises(self, tmp_path) data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) def test__given_unreachable_artifact__then_raises(self, tmp_path): @@ -821,6 +842,7 @@ def test__given_unreachable_artifact__then_raises(self, tmp_path): data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) def test__given_unreachable_vendored_artifact__then_raises(self, tmp_path): @@ -856,6 +878,7 @@ def test__given_unreachable_vendored_artifact__then_raises(self, tmp_path): data_producer="populace", manifest_uri=MANIFEST_URI, model_version="1.723.0", + bundle_path=_tmp_bundle(tmp_path), ) @@ -969,3 +992,473 @@ def test__given_microcosm_producer__then_still_drops_area_region_templates(self) assert self._payload()["region_datasets"] == { "national": {"path_template": "populace_us_2024.h5"} } + + +UK_NATIONAL_LINE = "microcosm-uk-2024-25-national" +UK_NATIONAL_TAG = f"{UK_NATIONAL_LINE}-20261101T000000Z-aaaaaaaa" +UK_NATIONAL_URI = ( + "hf://dataset/policyengine/populace-uk-private" + f"@{UK_NATIONAL_TAG}/releases/{UK_NATIONAL_LINE}/release_manifest.json" +) +UK_DENSE_LINE = "microcosm-uk-2024-25-dense" +UK_DENSE_TAG = f"{UK_DENSE_LINE}-20261101T000000Z-bbbbbbbb" +UK_DENSE_URI = ( + "hf://dataset/policyengine/populace-uk-private" + f"@{UK_DENSE_TAG}/releases/{UK_DENSE_LINE}/release_manifest.json" +) +EFRS_URI = ( + "hf://model/policyengine/policyengine-uk-data-private" + "@0aa3eee1cdae8f8125f4aff126a8f01957595dc3/releases/1.58.0/release_manifest.json" +) +UK_MODEL_VERSION = "2.126.12" +EFRS_REPO = "policyengine/policyengine-uk-data-private" + + +def _microcosm_uk_national_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "build": { + "build_id": UK_NATIONAL_LINE, + "cut_tag": UK_NATIONAL_TAG, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + }, + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_NATIONAL_TAG, + "sha256": "a" * 64, + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_NATIONAL_TAG, + }, + }, + } + + +def _microcosm_uk_dense_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "build": { + "build_id": UK_DENSE_LINE, + "cut_tag": UK_DENSE_TAG, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + }, + "artifacts": { + "microcosm_uk_2024_25_dense": { + "kind": "microdata", + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_DENSE_TAG, + "sha256": "b" * 64, + }, + "rowwise_candidate_manifest": { + "kind": "diagnostics", + "path": "rowwise_candidate_manifest.json", + "repo_id": "policyengine/populace-uk-private", + "revision": UK_DENSE_TAG, + }, + }, + } + + +def _efrs_release_payload() -> dict: + """Shaped like policyengine-uk-data's enhanced FRS 1.58.0 release.""" + + def artifact(kind: str, path: str, digit: str) -> dict: + return { + "kind": kind, + "path": path, + "repo_id": EFRS_REPO, + "revision": "1.58.0", + "sha256": digit * 64, + } + + return { + "schema_version": 1, + "data_package": {"name": "policyengine-uk-data", "version": "1.58.0"}, + "default_datasets": { + "national": "enhanced_frs_2024_25", + "baseline": "frs_2024_25", + }, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": "==2.122.2"}, + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"}, + ], + "build": { + "build_id": "policyengine-uk-data-1.58.0", + "built_with_model_package": { + "name": "policyengine-uk", + "version": "2.122.2", + }, + }, + "artifacts": { + "enhanced_frs_2024_25": artifact( + "microdata", "enhanced_frs_2024_25.h5", "1" + ), + "frs_2024_25": artifact("microdata", "frs_2024_25.h5", "2"), + "local_authority_weights": artifact( + "weights", "local_authority_weights.h5", "3" + ), + "local_geography_weights": artifact( + "weights", "local_geography_weights.csv.gz", "4" + ), + "parliamentary_constituency_weights": artifact( + "weights", "parliamentary_constituency_weights.h5", "5" + ), + }, + } + + +def _response(payload: dict) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.content = json.dumps(payload).encode() + return response + + +def _certify_uk(bundle_path: Path, responses: list, **kwargs): + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=responses, + ), + patch( + "policyengine.provenance.certification.head_artifact", + return_value=True, + ), + patch( + "policyengine.provenance.certification.head_artifact_reference", + return_value=True, + ), + patch( + "policyengine.provenance.certification.fetch_pypi_wheel_metadata", + return_value={"sha256": "d" * 64, "url": "https://example"}, + ), + patch( + "policyengine.provenance.certification.policyengine_version", + return_value="9.9.9", + ), + ): + return certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=UK_NATIONAL_URI, + model_version=UK_MODEL_VERSION, + bundle_path=bundle_path, + **kwargs, + ) + + +def _sim_api_routes_both_area_types(release: dict) -> bool: + # policyengine-sim-api's uk_area_regions_routed: a UK default without area + # codes is accepted only when both area region types are routed. + manifest = CountryReleaseManifest.model_validate(release) + return all( + region_type in manifest.region_datasets + for region_type in ("constituency", "local_authority") + ) + + +class TestUKRegionalMerge: + def _national(self) -> DataReleaseManifest: + return DataReleaseManifest.model_validate(_microcosm_uk_national_payload()) + + def test__given_uk_local_area_release__then_routes_constituencies_and_las(self): + merged, kind = merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_microcosm_uk_dense_payload()), + regional_repo_type="dataset", + ) + + assert kind == "local_area" + assert merged.metadata["region_datasets"] == { + "national": {"path_template": "microcosm_uk_2024_25.h5"}, + "constituency": {"path_template": "microcosm_uk_2024_25_dense.h5"}, + "local_authority": {"path_template": "microcosm_uk_2024_25_dense.h5"}, + } + dense = merged.artifacts["microcosm_uk_2024_25_dense"] + assert dense.revision == UK_DENSE_TAG + assert dense.repo_type == "dataset" + # Only the microdata joins; the local release's evidence stays there. + assert "rowwise_candidate_manifest" not in merged.artifacts + + def test__given_efrs_as_area_coded_national__then_carries_dataset_and_matrices( + self, + ): + merged, kind = merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_efrs_release_payload()), + release_kind="area_coded_national", + regional_repo_type="model", + ) + + assert kind == "area_coded_national" + assert merged.default_datasets == {"national": "microcosm_uk_2024_25"} + region_datasets = merged.metadata["region_datasets"] + assert region_datasets["constituency"] == { + "path_template": "enhanced_frs_2024_25.h5" + } + assert region_datasets["local_authority"] == region_datasets["constituency"] + carried = { + "enhanced_frs_2024_25", + "local_authority_weights", + "local_geography_weights", + "parliamentary_constituency_weights", + } + assert carried <= set(merged.artifacts) + assert "frs_2024_25" not in merged.artifacts + for name in carried: + artifact = merged.artifacts[name] + assert (artifact.repo_id, artifact.revision) == (EFRS_REPO, "1.58.0") + assert artifact.repo_type == "model" + + def test__given_national_release_without_kind__then_names_the_flag(self): + with pytest.raises( + CertificationError, match="--regional-release-kind area_coded_national" + ): + merge_regional_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_efrs_release_payload()), + ) + + def test__given_area_coded_release_without_a_matrix__then_rejects_it(self): + payload = _efrs_release_payload() + payload["artifacts"].pop("local_authority_weights") + + with pytest.raises(CertificationError, match="local_authority_weights.h5"): + merge_area_coded_national_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(payload), + ) + + def test__given_local_area_release_as_area_coded_national__then_rejects_it(self): + with pytest.raises(CertificationError, match="must be a national release"): + merge_area_coded_national_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(_microcosm_uk_dense_payload()), + ) + + def test__given_us_area_coded_national__then_rejects_it(self): + with pytest.raises(CertificationError, match="only supported for the UK"): + merge_area_coded_national_release_manifest( + "us", + DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ), + DataReleaseManifest.model_validate(_efrs_release_payload()), + ) + + def test__given_uk_local_area_release__then_generic_merge_checks_role(self): + payload = _microcosm_uk_dense_payload() + payload["is_default"] = True + + with pytest.raises( + CertificationError, match="UK local-area release must declare is_default" + ): + merge_local_area_release_manifest( + "uk", + self._national(), + DataReleaseManifest.model_validate(payload), + ) + + +class TestUKRegionalCertification: + def test__given_national_first_efrs_route__then_bundle_routes_areas_to_efrs( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + + result = _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_efrs_release_payload()), + ], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["default_dataset"] == "microcosm_uk_2024_25" + assert release["regional_release_kind"] == "area_coded_national" + assert release["regional_source_manifest_uri"] == EFRS_URI + assert release["region_datasets"]["constituency"] == { + "path_template": "enhanced_frs_2024_25.h5" + } + assert _sim_api_routes_both_area_types(release) + # sim-api binds a matrix only to the release of the dataset it + # reweights: the same repo and revision. + datasets = release["datasets"] + for name in ( + "enhanced_frs_2024_25", + "parliamentary_constituency_weights", + "local_authority_weights", + ): + assert (datasets[name]["repo_id"], datasets[name]["revision"]) == ( + EFRS_REPO, + "1.58.0", + ) + assert datasets[name]["repo_type"] == "model" + # Diagnostics stay under the release directory of the national cut. + assert datasets["calibration_diagnostics"]["path"] == ( + f"releases/{UK_NATIONAL_LINE}/calibration_diagnostics.json" + ) + # The cut tag, not the line name, identifies the certified build. + assert release["version"] == UK_NATIONAL_TAG + assert release["certification"]["data_build_id"] == UK_NATIONAL_TAG + assert result.build_id == UK_NATIONAL_TAG + + def test__given_ship_together_dense_route__then_bundle_routes_areas_to_local( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["regional_release_kind"] == "local_area" + assert release["region_datasets"]["local_authority"] == { + "path_template": "microcosm_uk_2024_25_dense.h5" + } + assert release["datasets"]["microcosm_uk_2024_25_dense"]["revision"] == ( + UK_DENSE_TAG + ) + assert _sim_api_routes_both_area_types(release) + + def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): + efrs = _efrs_release_payload() + efrs["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.122.2"} + ] + + with pytest.raises(CertificationError, match="Regional release .*2.126.12"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(efrs), + ], + regional_manifest_uri=EFRS_URI, + regional_release_kind="area_coded_national", + ) + + def test__given_unreachable_regional_artifact__then_refuses(self, tmp_path): + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + ), + patch( + "policyengine.provenance.certification.head_artifact", + side_effect=lambda artifact, token=None: ( + artifact.path != "microcosm_uk_2024_25_dense.h5" + ), + ), + pytest.raises(CertificationError, match="Regional artifact"), + ): + certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=UK_NATIONAL_URI, + regional_manifest_uri=UK_DENSE_URI, + model_version=UK_MODEL_VERSION, + bundle_path=_tmp_bundle(tmp_path), + ) + + def test__given_routed_bundle__then_recertifying_without_regional_refuses( + self, tmp_path + ): + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + with pytest.raises(CertificationError, match="--drop-regional-release"): + _certify_uk(bundle_path, [_response(_microcosm_uk_national_payload())]) + + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert "regional_source_manifest_uri" not in release + assert "regional_release_kind" not in release + assert set(release["region_datasets"]) == {"national"} + + def test__given_bundle_wrapper_regional_kind__then_forwards_to_certifier( + self, monkeypatch + ): + bundle_script = _load_bundle_script(monkeypatch) + captured: dict[str, list[str]] = {} + fake = ModuleType("certify_data_release") + + def fake_main(argv): + captured["argv"] = argv + return 0 + + fake.main = fake_main + monkeypatch.setitem(sys.modules, "certify_data_release", fake) + + assert ( + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--regional-manifest-uri", + EFRS_URI, + "--regional-release-kind", + "area_coded_national", + "--drop-regional-release", + ] + ) + == 0 + ) + argv = captured["argv"] + assert argv[argv.index("--regional-release-kind") + 1] == ( + "area_coded_national" + ) + assert "--drop-regional-release" in argv diff --git a/tests/test_constituency_impact.py b/tests/test_constituency_impact.py index 4b7e3d1d..575f6242 100644 --- a/tests/test_constituency_impact.py +++ b/tests/test_constituency_impact.py @@ -6,6 +6,7 @@ import pytest from microdf import MicroDataFrame +from policyengine.data.uk_geography_assets import packaged_uk_area_list_path from policyengine.outputs.constituency_impact import ( compute_uk_constituency_impacts, ) @@ -148,31 +149,29 @@ def test_relative_change(tmp_path): ) -def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( +def test_compute_labels_with_packaged_area_list_by_default( tmp_path, ): - """The helper can enrich labels from local CSV without matrix assets.""" + """Labels come from the packaged list, not local or downloaded files.""" baseline = _make_sim( { - "constituency_code_oa": ["C001", "C002"], + "constituency_code_oa": ["E14001063", "UNLISTED"], "household_net_income": [100.0, 200.0], "household_weight": [1.0, 1.0], } ) reform = _make_sim( { - "constituency_code_oa": ["C001", "C002"], + "constituency_code_oa": ["E14001063", "UNLISTED"], "household_net_income": [110.0, 220.0], "household_weight": [1.0, 1.0], } ) - csv_path = tmp_path / CONSTITUENCY_ASSET_SPEC.lookup_csv_filename - pd.DataFrame( - [ - {"code": "C001", "name": "A", "x": 0, "y": 0}, - {"code": "C002", "name": "B", "x": 1, "y": 1}, - ] - ).to_csv(csv_path, index=False) + # A stale lookup in a local search dir (the old GCS download cache) is + # ignored. + pd.DataFrame([{"code": "E14001063", "name": "Stale", "x": 0, "y": 0}]).to_csv( + tmp_path / CONSTITUENCY_ASSET_SPEC.lookup_csv_filename, index=False + ) with patch( "policyengine.outputs.uk_geography_impact.default_local_search_dirs", @@ -184,9 +183,13 @@ def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( download_missing_assets=True, ) - assert impact.constituency_csv_path == str(csv_path) - assert len(impact.constituency_results) == 2 - assert impact.constituency_results[0]["constituency_name"] == "A" + assert impact.constituency_csv_path == packaged_uk_area_list_path("constituency") + assert impact.constituency_csv_path.endswith("constituencies_2024.csv") + by_code = {row["constituency_code"]: row for row in impact.constituency_results} + assert by_code["E14001063"]["constituency_name"] == "Aldershot" + assert (by_code["E14001063"]["x"], by_code["E14001063"]["y"]) == (56, -40) + assert by_code["UNLISTED"]["constituency_name"] == "UNLISTED" + assert by_code["UNLISTED"]["x"] is None def test_compute_constituency_impacts_does_not_require_lookup_csv_or_matrix( diff --git a/tests/test_local_authority_impact.py b/tests/test_local_authority_impact.py index 8e54a50a..5a614e8e 100644 --- a/tests/test_local_authority_impact.py +++ b/tests/test_local_authority_impact.py @@ -6,6 +6,7 @@ import pytest from microdf import MicroDataFrame +from policyengine.data.uk_geography_assets import packaged_uk_area_list_path from policyengine.outputs.local_authority_impact import ( compute_uk_local_authority_impacts, ) @@ -150,31 +151,29 @@ def test_relative_change(tmp_path): ) -def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( +def test_compute_labels_with_packaged_area_list_by_default( tmp_path, ): - """The helper can enrich labels from local CSV without matrix assets.""" + """Labels come from the packaged list, not local or downloaded files.""" baseline = _make_sim( { - "la_code_oa": ["LA001", "LA002"], + "la_code_oa": ["N09000011", "UNLISTED"], "household_net_income": [100.0, 200.0], "household_weight": [1.0, 1.0], } ) reform = _make_sim( { - "la_code_oa": ["LA001", "LA002"], - "household_net_income": [115.0, 230.0], + "la_code_oa": ["N09000011", "UNLISTED"], + "household_net_income": [110.0, 220.0], "household_weight": [1.0, 1.0], } ) - csv_path = tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename - pd.DataFrame( - [ - {"code": "LA001", "name": "A", "x": 0, "y": 0}, - {"code": "LA002", "name": "B", "x": 1, "y": 1}, - ] - ).to_csv(csv_path, index=False) + # A stale lookup in a local search dir (the old GCS download cache) is + # ignored. + pd.DataFrame([{"code": "N09000011", "name": "Stale", "x": 0, "y": 0}]).to_csv( + tmp_path / LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename, index=False + ) with patch( "policyengine.outputs.uk_geography_impact.default_local_search_dirs", @@ -186,9 +185,17 @@ def test_compute_uses_local_lookup_csv_without_matrix_or_gcs( download_missing_assets=True, ) - assert impact.local_authority_csv_path == str(csv_path) - assert len(impact.local_authority_results) == 2 - assert impact.local_authority_results[0]["local_authority_name"] == "A" + assert impact.local_authority_csv_path == packaged_uk_area_list_path( + "local_authority" + ) + assert impact.local_authority_csv_path.endswith("local_authorities_2023.csv") + by_code = { + row["local_authority_code"]: row for row in impact.local_authority_results + } + assert by_code["N09000011"]["local_authority_name"] == "Ards and North Down" + assert (by_code["N09000011"]["x"], by_code["N09000011"]["y"]) == (-3, 16) + assert by_code["UNLISTED"]["local_authority_name"] == "UNLISTED" + assert by_code["UNLISTED"]["x"] is None def test_compute_local_authority_impacts_does_not_require_lookup_csv_or_matrix( diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index ae4a7925..2cfce077 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -688,6 +688,41 @@ def test__given_matching_fingerprint__then_certification_allows_reuse(self): assert certification.built_with_model_version == "1.601.0" assert certification.certified_for_model_version == "1.602.0" + def test__given_release_with_cut_tag__then_cut_tag_is_the_data_build_id(self): + # Microcosm keeps build_id equal to the line name for every cut, so + # the runtime check must name the cut the same way certification did. + get_data_release_manifest.cache_clear() + payload = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "build": { + "build_id": "microcosm-uk-2024-25-national", + "cut_tag": "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa", + "built_with_model_package": { + "name": "policyengine-us", + "version": "1.601.0", + }, + }, + "compatible_model_packages": [], + "default_datasets": {"national": "enhanced_cps_2024"}, + "artifacts": {}, + } + + with patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ): + certification = certify_data_release_compatibility( + "us", + runtime_model_version="1.601.0", + ) + + assert certification.compatibility_basis == "exact_build_model_version" + assert certification.data_build_id == ( + "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + ) + get_data_release_manifest.cache_clear() + def test__given_private_manifest_unavailable__then_bundled_certification_is_used( self, ): diff --git a/tests/test_uk_area_lists.py b/tests/test_uk_area_lists.py new file mode 100644 index 00000000..62d17fee --- /dev/null +++ b/tests/test_uk_area_lists.py @@ -0,0 +1,114 @@ +"""Tests for the UK area lists shipped with the package.""" + +import csv +import re +from collections import Counter +from unittest.mock import patch + +import pytest + +from policyengine.countries.uk.regions import build_uk_region_registry +from policyengine.data.uk_geography_assets import ( + LOCAL_AUTHORITY_ASSET_SPEC, + UK_AREA_LIST_FILES, + packaged_uk_area_list_path, +) + +GSS_CODE = re.compile(r"^[A-Z]\d{8}$") + +# The 17 districts abolished on 1 April 2023, when Cumberland, Westmorland and +# Furness, North Yorkshire and Somerset became unitary authorities. +ABOLISHED_2023_DISTRICTS = { + "E07000026", + "E07000027", + "E07000028", + "E07000029", + "E07000030", + "E07000031", + "E07000163", + "E07000164", + "E07000165", + "E07000166", + "E07000167", + "E07000168", + "E07000169", + "E07000187", + "E07000188", + "E07000189", + "E07000246", +} +NEW_2023_UNITARIES = {"E06000063", "E06000064", "E06000065", "E06000066"} + + +def _rows(geography_type: str) -> list[dict]: + with open(packaged_uk_area_list_path(geography_type), encoding="utf-8") as file: + return list(csv.DictReader(file)) + + +@pytest.mark.parametrize("geography_type", sorted(UK_AREA_LIST_FILES)) +def test_area_list_rows_are_well_formed(geography_type): + rows = _rows(geography_type) + codes = [row["code"] for row in rows] + + assert len(codes) == len(set(codes)) + assert all(GSS_CODE.match(code) for code in codes) + assert all(row["name"].strip() and row["name"] != row["code"] for row in rows) + hexes = [(int(row["x"]), int(row["y"])) for row in rows] + assert len(hexes) == len(set(hexes)), "two areas share a hex" + + +def test_local_authority_list_is_the_april_2023_frame(): + codes = {row["code"] for row in _rows("local_authority")} + + assert len(codes) == 361 + assert Counter(code[0] for code in codes) == {"E": 296, "W": 22, "S": 32, "N": 11} + assert "N09000011" in codes # Ards and North Down + assert NEW_2023_UNITARIES <= codes + assert not ABOLISHED_2023_DISTRICTS & codes + + +def test_constituency_list_is_the_2024_frame_including_northern_ireland(): + codes = [row["code"] for row in _rows("constituency")] + + assert len(codes) == 650 + assert Counter(code[:3] for code in codes) == { + "E14": 543, + "S14": 57, + "W07": 32, + "N05": 18, + } + + +def test_weight_matrix_lookup_stays_paired_with_its_matrix(): + # The legacy local-authority matrix indexes areas by the row order of the + # CSV it was built with, so the packaged list must not replace it. + assert ( + LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename == "local_authorities_2021.csv" + ) + assert UK_AREA_LIST_FILES["local_authority"] != ( + LOCAL_AUTHORITY_ASSET_SPEC.lookup_csv_filename + ) + + +def test_unknown_geography_type_has_no_packaged_list(): + with pytest.raises(ValueError, match="No packaged UK area list"): + packaged_uk_area_list_path("ward") + + +def test_registry_builds_every_area_without_network(): + # Before the lists were packaged, these regions came from a GCS download + # that silently returned no areas when it failed. + with patch( + "policyengine_core.tools.google_cloud.download_gcs_file", + side_effect=AssertionError("network access"), + ): + registry = build_uk_region_registry( + include_constituencies=True, + include_local_authorities=True, + ) + + assert len(registry.get_by_type("national")) == 1 + assert len(registry.get_by_type("country")) == 4 + assert len(registry.get_by_type("constituency")) == 650 + assert len(registry.get_by_type("local_authority")) == 361 + assert registry.get("local_authority/N09000011").label == "Ards and North Down" diff --git a/tests/test_uk_regions.py b/tests/test_uk_regions.py index f4676ab8..2c45c62d 100644 --- a/tests/test_uk_regions.py +++ b/tests/test_uk_regions.py @@ -17,6 +17,7 @@ build_uk_region_registry, uk_region_registry, ) +from policyengine.provenance.manifest import CountryReleaseManifest class TestUKCountries: @@ -303,6 +304,146 @@ def test__given_local_authorities_included__then_filters_on_dataset_geography( assert local_authority.scoping_strategy.variable_value == "LA001" +MICROCOSM_NATIONAL_TAG = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" +MICROCOSM_DENSE_TAG = "microcosm-uk-2024-25-dense-20261101T000000Z-bbbbbbbb" +MICROCOSM_NATIONAL_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + f"@{MICROCOSM_NATIONAL_TAG}" +) +EFRS_AREA_URI = ( + "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.0" +) +MICROCOSM_DENSE_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + f"@{MICROCOSM_DENSE_TAG}" +) + + +def _uk_routed_manifest(kind: str) -> CountryReleaseManifest: + """A certified Microcosm national bundle whose area regions are routed.""" + if kind == "area_coded_national": + area_name, area_reference = ( + "enhanced_frs_2024_25", + { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.0", + }, + ) + else: + area_name, area_reference = ( + "microcosm_uk_2024_25_dense", + { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": MICROCOSM_DENSE_TAG, + }, + ) + area_template = {"path_template": area_reference["path"]} + return CountryReleaseManifest.model_validate( + { + "country_id": "uk", + "policyengine_version": "9.9.9", + "model_package": {"name": "policyengine-uk", "version": "2.126.12"}, + "data_package": { + "name": "microcosm-data", + "version": "0.1.0", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "release_manifest_revision": MICROCOSM_NATIONAL_TAG, + }, + "default_dataset": "microcosm_uk_2024_25", + "datasets": { + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": MICROCOSM_NATIONAL_TAG, + }, + area_name: area_reference, + }, + "region_datasets": { + "national": {"path_template": "microcosm_uk_2024_25.h5"}, + "constituency": area_template, + "local_authority": area_template, + }, + "regional_release_kind": kind, + } + ) + + +class TestUKAreaRegionRouting: + """Area regions load the dataset certification routes them to.""" + + def test__given_unrouted_bundle__then_area_regions_filter_the_national_dataset( + self, + ): + registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True + ) + + assert registry.get("constituency/E14001063").dataset_path is None + assert registry.get("local_authority/N09000011").dataset_path is None + assert len(uk_region_registry.get_by_type("constituency")) == 0 + + def test__given_national_first_bundle__then_area_regions_load_the_efrs( + self, monkeypatch + ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("area_coded_national"), + ) + + default = build_uk_region_registry() + registry = build_uk_region_registry( + include_constituencies=True, include_local_authorities=True + ) + + # An area-coded national route does not add area regions by default. + assert len(default) == 5 + assert registry.get("uk").dataset_path == MICROCOSM_NATIONAL_URI + assert registry.get("country/wales").dataset_path is None + constituency = registry.get("constituency/E14001063") + assert constituency.dataset_path == EFRS_AREA_URI + assert constituency.scoping_strategy.variable_name == "constituency_code_oa" + local_authority = registry.get("local_authority/N09000011") + assert local_authority.dataset_path == EFRS_AREA_URI + assert local_authority.scoping_strategy.variable_name == "la_code_oa" + + def test__given_local_area_bundle__then_default_registry_adds_area_regions( + self, monkeypatch + ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("local_area"), + ) + + registry = build_uk_region_registry() + + assert len(registry.get_by_type("constituency")) == 650 + assert len(registry.get_by_type("local_authority")) == 361 + assert registry.get("constituency/N05000001").dataset_path == ( + MICROCOSM_DENSE_URI + ) + assert registry.get("local_authority/E06000063").dataset_path == ( + MICROCOSM_DENSE_URI + ) + assert registry.get("uk").dataset_path == MICROCOSM_NATIONAL_URI + + def test__given_local_area_bundle__then_explicit_exclusion_wins(self, monkeypatch): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_routed_manifest("local_area"), + ) + + registry = build_uk_region_registry( + include_constituencies=False, include_local_authorities=False + ) + + assert len(registry) == 5 + + # The pinned UK model's Region enum contains nine English ITL1 regions and # Scotland, Wales and Northern Ireland. Country is derived, not a stored input. _REGION_COUNTRIES = [ diff --git a/tests/test_uk_uc_floor.py b/tests/test_uk_uc_floor.py new file mode 100644 index 00000000..46174934 --- /dev/null +++ b/tests/test_uk_uc_floor.py @@ -0,0 +1,84 @@ +"""Tests for the UK Universal Credit coverage floor.""" + +import pytest + +from policyengine.provenance.uk_uc_floor import ( + evaluate_uc_floor, + load_uc_floor_benchmarks, + measure_great_britain_uc, +) + + +def test_benchmarks_cite_a_source_for_every_measure(): + benchmarks = load_uc_floor_benchmarks() + + assert benchmarks.tolerance == 0.1 + assert benchmarks.geography == "great_britain" + assert 2025 in benchmarks.years + for year in benchmarks.years.values(): + for benchmark in (year.benefit_units, year.spend): + assert benchmark.value > 0 + assert benchmark.source + assert benchmark.url.startswith("https://") + + +def test_floor_passes_within_tolerance_and_fails_outside_it(): + year = load_uc_floor_benchmarks().years[2025] + + passing = evaluate_uc_floor( + benefit_units=year.benefit_units.value * 0.95, + spend=year.spend.value * 1.09, + year=2025, + dataset="candidate", + ) + failing = evaluate_uc_floor( + benefit_units=year.benefit_units.value * 0.5, + spend=year.spend.value, + year=2025, + dataset="candidate", + ) + + assert passing.passed + assert not failing.passed + measures = {measure.name: measure for measure in failing.measures} + assert not measures["benefit_units"].passed + assert measures["benefit_units"].ratio == pytest.approx(0.5) + assert measures["spend"].passed + + +def test_floor_refuses_a_year_without_benchmarks(): + with pytest.raises(ValueError, match="No Universal Credit floor benchmarks"): + evaluate_uc_floor(benefit_units=1, spend=1, year=2019, dataset="candidate") + + +def test_measurement_excludes_northern_ireland(): + from policyengine_uk import Simulation + + simulation = Simulation( + situation={ + "people": { + "a": {"age": {2025: 30}}, + "b": {"age": {2025: 40}}, + "c": {"age": {2025: 50}}, + }, + "benunits": { + "ni_one": {"members": ["a"]}, + "ni_two": {"members": ["b"]}, + "london": {"members": ["c"]}, + }, + "households": { + "belfast": { + "members": ["a", "b"], + "region": {2025: "NORTHERN_IRELAND"}, + }, + "london": {"members": ["c"], "region": {2025: "LONDON"}}, + }, + } + ) + universal_credit = simulation.calculate("universal_credit", 2025) + assert (universal_credit > 0).all() + + benefit_units, spend = measure_great_britain_uc(simulation, 2025) + + assert benefit_units == pytest.approx(1) + assert spend == pytest.approx(float(universal_credit[2]))