From 531b6102abc3e4dd9911f819e2a85a645cb623c5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 27 Sep 2026 18:49:22 -0400 Subject: [PATCH 1/2] Add the US block -> congressional-district plan registry Record, for every populated 2020 census block, its district under each registered plan, so a block draw can derive every plan's district from the chosen block and SOI's 117th-plan district tables become exact block sums. - 117th_congress: 2020 BAF CD layer; North Carolina carried from its 2019 plan onto 2020 blocks through the Census 2010-2020 block relationship file (method reproduces Census's 2016 plan for 99.992% of NC population; districts within 0.09% of NCGA's official 2020 figures). - 118th/119th_congress: Census block equivalency files. - 120th_congress: Census's 120th file (2026-08-31), with Missouri on its 2022 map per the 2026-09-25 Supreme Court order in 26A388. The builder cross-checks apportionment, Census state vs national files, Census's redrawn-state lists and split blocks, and agrees exactly with the block ladder; the loader re-checks every invariant; the artifact is byte-reproducible and pinned in us_cd_plan_registry.provenance.json. Co-Authored-By: Claude Opus 5.5 --- changelog.d/us-cd-plan-registry.added.md | 1 + .../build/us_runtime/US_CD_PLAN_REGISTRY.md | 137 ++ .../build/us_runtime/cd_plan_registry.py | 1127 ++++++++++++++++ .../us_cd_plan_registry.provenance.json | 1128 +++++++++++++++++ .../us/test_us_cd_plan_registry.py | 519 ++++++++ .../us/test_us_cd_plan_registry_crosswalk.py | 189 +++ .../us/test_us_cd_plan_registry_properties.py | 470 +++++++ tools/build_us_cd_plan_registry_artifact.py | 776 ++++++++++++ 8 files changed, 4347 insertions(+) create mode 100644 changelog.d/us-cd-plan-registry.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_crosswalk.py create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py create mode 100644 tools/build_us_cd_plan_registry_artifact.py diff --git a/changelog.d/us-cd-plan-registry.added.md b/changelog.d/us-cd-plan-registry.added.md new file mode 100644 index 000000000..9a0a63d5a --- /dev/null +++ b/changelog.d/us-cd-plan-registry.added.md @@ -0,0 +1 @@ +Add a SHA-pinned US block -> congressional-district plan registry: every populated 2020 block's district under the 117th (SOI's geography; North Carolina carried from its 2019 plan), 118th, 119th and 120th (November 2026, Missouri on its 2022 map) plans, with a validating loader, a Census-sourced builder and property tests. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md b/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md new file mode 100644 index 000000000..30f580c73 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md @@ -0,0 +1,137 @@ +# US congressional-district plan registry + +A household's location is one 2020 census tabulation block, and every other +geography is a lookup from that block. Congressional districts are the one +layer with several live versions on the same blocks, so this registry records, +for every populated 2020 block, its district under each registered plan. A +block draw can then derive every plan's district from the chosen block, and +SOI's 117th-Congress district tables become exact block sums with no +crosswalk. + +Code: `cd_plan_registry.py`. Build: `tools/build_us_cd_plan_registry_artifact.py`. +Receipt (artifact SHA-256, sources, per-plan totals, every check's result): +`us_cd_plan_registry.provenance.json`. + +## Plans + +| Plan id | Districts | Source | Apportionment | +|---|---|---|---| +| `117th_congress` | 2021–2023; the geography of the IRS SOI congressional-district tables for tax years 2020–2022 | Census 2020 Block Assignment Files, `CD` layer; North Carolina carried from its 2019 plan (below) | 2010 | +| `118th_congress` | 2023–2025 | Census 118th Congressional District block equivalency file | 2020 | +| `119th_congress` | 2025–2027; the block ladder's primary plan | Census 119th Congressional District block equivalency file | 2020 | +| `120th_congress` | the November 3, 2026 election, including the 2025–26 mid-decade redraws | Census 120th Congressional District block equivalency file (published 2026-08-31); Missouri from its 2022 map (below) | 2020 | + +District geoids are `state_fips * 100 + district`. At-large states and the +DC delegate are district `00`, the repo-wide convention. Each plan's metadata +records its source files (URL and SHA-256), its apportionment, the blocks +Census lists as split by a district line, and any known deviations. + +### North Carolina in the 117th plan + +The 2020 BAF `CD` layer carries the 116th-Congress plans on 2020 blocks. +Census reports North Carolina as the only state whose districts changed +between the 116th and 117th Congress: the 2020 election used the 2019 +remedial plan (HB 1029, S.L. 2019-249), which Census never tabulated on 2020 +blocks. On 2020 blocks the two North Carolina plans put only 55% of the +state's population in the same district, so the 116th rows would be wrong for +SOI's 117th-plan targets there. + +The build therefore carries the General Assembly's block file for the 2019 +plan (on 2010 blocks) onto 2020 blocks through the Census 2010–2020 +tabulation block relationship file. Each 2020 block takes the district whose +2010 blocks cover the most of its land area, then water area, then the lower +district code. The 2020 blocks with no North Carolina 2010 counterpart take +their block group's majority district by population, falling back to the +tract and then the county. Two checks run on every build and are recorded in +the receipt: + +- **The method reproduces Census.** Run on the 2016 plan, the same method + matches Census's own 2020-block version of that plan (the BAF `CD` layer) + for 99.992% of North Carolina's population. The build refuses below 99.9%. +- **The result matches the official figures.** The carried plan's district + populations are within 704 people (0.09%) of the General Assembly's official + 2020-census populations for the 2019 plan. The build refuses above 0.5%. + +289 of the state's 2020 blocks (16,671 people) straddle a 2019-plan district +line. Four have no North Carolina 2010 counterpart; one of them is populated +(13 people). + +### Missouri in the 120th plan + +Census's 120th rows for Missouri encode the 2025 HB 1 map, and Census's 120th +landing page warns they may not be the districts in place for November 2026. +The Missouri Supreme Court held on 2026-09-03 (von Glahn v. Hoskins) that HB 1 +cannot take effect before a referendum on the November 2026 ballot. The U.S. +Supreme Court held on 2026-09-25 (26A388, per curiam) that "the 2022 map—not +the 2025 map—must be used in the 2026 congressional election." Missouri's +`120th_congress` rows are therefore its 2022 map, taken from the 119th file. +The build checks that those rows are unchanged from the 118th. This follows +the court record as of 2026-09-27; a later ruling would mean a rebuild. + +## Invariants + +Checked when the artifact is built (`assemble_us_cd_plan_registry`) and again +on every load (`load_us_cd_plan_registry`): + +- **Universe.** The blocks are exactly the 2020 P.L. 94-171 blocks with + positive population in the 50 states and DC (5,769,942 blocks, 331,449,281 + people), sorted and unique: the block ladder's universe. +- **Partition.** Every block has exactly one district under every plan, and + that district lies in the block's state. A populated block that a plan's + source leaves unassigned is a build error, never a gap. +- **Apportionment.** Each state's districts under a plan are exactly its House + apportionment for that plan's census: `00` for an at-large state or DC, + otherwise `01` through `n`. So every apportioned district contains a + populated block. +- **Conservation.** Therefore, for every plan, district populations sum + exactly to state populations, and those sum to the national total. + `summarize_cd_plan_registry` re-checks this. + +The build also cross-checks its sources against each other and refuses on any +disagreement: + +- the packaged seat counts against the Census apportionment table; +- each Census block equivalency file's state files against that state's rows + in its national file; +- the states whose blocks change from the 118th to the 119th, and from the + 119th to the 120th, against the states Census ships a state file for (its + list of redrawn states); +- the one block Census lists as split by a district line (Colorado + 080010096072000) against the district Census tabulates it with; +- with `--block-ladder`, the registry against the ladder: the same blocks, + the same populations, and the same 119th district on every block. + +The property tests (`test_us_cd_plan_registry_properties.py`) check the +partition, conservation, apportionment and determinism invariants on random +geographies. They also check that dropping, crossing or merging any assignment +is refused, and that the crosswalk picks the largest overlap regardless of row +order. + +## Build + +```bash +uv run python tools/build_us_cd_plan_registry_artifact.py \ + --out build/us/us_cd_plan_registry_2020.npz \ + --block-ladder build/us/us_block_ladder_2020.npz \ + --provenance-json packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json +``` + +The tool downloads and caches its sources, builds the NPZ (about 18 MB), loads +it back through the validating loader, and writes the receipt. The output is +byte-reproducible from the pinned sources. A national build takes about two +minutes and 2.4 GB of memory. + +## Use + +```python +from microcosm.build.us_runtime.cd_plan_registry import load_pinned_us_cd_plan_registry + +registry = load_pinned_us_cd_plan_registry(path) # refuses any other file +registry.plans["117th_congress"] # district per block, aligned to registry.block_geoid +registry.plan_sources["117th_congress"]["known_deviations"] +``` + +`load_pinned_us_cd_plan_registry` refuses any file whose SHA-256 differs from +the packaged receipt. `load_us_cd_plan_registry(path, expected_sha256=...)` +pins to another digest. The registry assigns no households. A block draw +looks up each plan's district from the household's block. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py new file mode 100644 index 000000000..4b57dd372 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py @@ -0,0 +1,1127 @@ +"""US block -> congressional-district plan registry. + +A household's location is one 2020 census tabulation block, and every other +geography is a lookup from that block. Congressional districts are the one +layer that has several live versions on the same 2020 blocks, so this module +keeps them in one versioned artifact: for every populated 2020 block, the +district the block belongs to under each registered plan. + +- ``117th_congress``: the plan the IRS SOI congressional-district tables use + (see ``congressional_district_vintage``). +- ``118th_congress``: the first post-2020-apportionment plan. +- ``119th_congress``: the current plan and the block ladder's primary one. +- ``120th_congress``: the plans in effect for the November 2026 election, + including the 2025-26 mid-decade redraws. + +Each plan is one ``int16`` array aligned to the registry's sorted +``block_geoid``, holding district geoids ``state_fips * 100 + district`` +(at-large states and the DC delegate are district ``00``, the repo-wide +convention). A new plan is one more array plus one metadata entry. + +The artifact is an NPZ built by ``tools/build_us_cd_plan_registry_artifact.py`` +from primary Census sources. Its embedded metadata records each plan's source +files (URL + SHA-256), the apportionment it must match, and any known +deviations. The build is byte-reproducible, and its SHA-256 is pinned in the +packaged ``us_cd_plan_registry.provenance.json``. + +Invariants, checked by :func:`assemble_us_cd_plan_registry` when the artifact +is built and again by :func:`load_us_cd_plan_registry` on every load: + +- The block universe is exactly the 2020 P.L. 94-171 blocks with positive + population (the block ladder's universe), sorted and unique. +- Every block has exactly one district under every plan. A populated block a + plan's source leaves unassigned is a build error, never a gap. +- Every district lies in its block's state. +- Each state's districts under a plan are exactly its House apportionment + for that plan's census: ``00`` for an at-large state or the DC delegate, + otherwise ``01`` through ``n``. So every apportioned district contains at + least one populated block. + +Summing block population by any plan's district therefore reproduces every +state's population exactly: the plans are partitions of the same blocks. + +This module is pure apart from the NPZ read. Nothing here assigns households; +the block draw derives every plan's district from the chosen block. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from array import array +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any + +import numpy as np + +from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, +) +from microcosm.build.us_runtime.congressional_district_vintage_crosswalk import ( + normalize_district_code, +) + +US_CD_PLAN_REGISTRY_KIND = "us_cd_plan_registry" +US_CD_PLAN_REGISTRY_SCHEMA_VERSION = 1 +US_CD_PLAN_REGISTRY_BLOCK_VINTAGE = "2020_tabulation_blocks" +#: NPZ key prefix of each plan's district array (``cd__119th_congress``). +CD_PLAN_ARRAY_PREFIX = "cd__" +#: The plan the block ladder's own ``congressional_district_geoid`` carries. +PRIMARY_CD_PLAN = CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE +#: Keys every plan's metadata must carry (``block_location`` requires +#: ``vintage`` and ``source``; the rest are this registry's provenance). +REQUIRED_PLAN_SOURCE_KEYS = ( + "vintage", + "source", + "url", + "apportionment", + "source_files", +) + +US_CD_PLAN_REGISTRY_PROVENANCE_RESOURCE = "us_cd_plan_registry.provenance.json" + +#: Census "Apportionment Data" historical table, the source of +#: :data:`US_HOUSE_APPORTIONMENT`. The registry build re-derives both columns +#: from this file and refuses to run if they disagree with the constant. +US_HOUSE_APPORTIONMENT_SOURCE = { + "url": ( + "https://www2.census.gov/programs-surveys/decennial/2020/data/" + "apportionment/apportionment.csv" + ), + "sha256": "d9be9c833ad936ee309a83fa6428a3ae2f82d5d8e5b2c69c4d6ff1e79ba57fcd", +} + +#: Voting House seats by state FIPS for each decennial apportionment +#: (``Number of Representatives`` in the Census table; DC has none). +# fmt: off +US_HOUSE_APPORTIONMENT: Mapping[str, Mapping[str, int]] = { + "2010_census": { + "01": 7, "02": 1, "04": 9, "05": 4, "06": 53, "08": 7, "09": 5, + "10": 1, "11": 0, "12": 27, "13": 14, "15": 2, "16": 2, "17": 18, + "18": 9, "19": 4, "20": 4, "21": 6, "22": 6, "23": 2, "24": 8, + "25": 9, "26": 14, "27": 8, "28": 4, "29": 8, "30": 1, "31": 3, + "32": 4, "33": 2, "34": 12, "35": 3, "36": 27, "37": 13, "38": 1, + "39": 16, "40": 5, "41": 5, "42": 18, "44": 2, "45": 7, "46": 1, + "47": 9, "48": 36, "49": 4, "50": 1, "51": 11, "53": 10, "54": 3, + "55": 8, "56": 1, + }, + "2020_census": { + "01": 7, "02": 1, "04": 9, "05": 4, "06": 52, "08": 8, "09": 5, + "10": 1, "11": 0, "12": 28, "13": 14, "15": 2, "16": 2, "17": 17, + "18": 9, "19": 4, "20": 4, "21": 6, "22": 6, "23": 2, "24": 8, + "25": 9, "26": 13, "27": 8, "28": 4, "29": 8, "30": 2, "31": 3, + "32": 4, "33": 2, "34": 12, "35": 3, "36": 26, "37": 14, "38": 1, + "39": 15, "40": 5, "41": 6, "42": 17, "44": 2, "45": 7, "46": 1, + "47": 9, "48": 38, "49": 4, "50": 1, "51": 11, "53": 10, "54": 2, + "55": 8, "56": 1, + }, +} +# fmt: on + +_PLAN_ID = re.compile(r"^[0-9a-z][0-9a-z_]*$") +_BLOCK_COLUMNS = ("GEOID", "BLOCKID", "BLOCK", "BLOCK_KEY") +_DISTRICT_COLUMNS = ("CDFP", "DISTRICT") +_BLOCK_STATE_DIVISOR = 10**13 +_MIN_BLOCK_GEOID = 10**13 # state FIPS 01 +_MAX_BLOCK_GEOID = 10**15 + + +def expected_district_counts(apportionment: str) -> dict[str, int]: + """District count per state FIPS under ``apportionment``. + + A state with one seat has one at-large district; DC, with no voting seat, + has one delegate district. Both carry district code ``00``. + """ + + try: + seats = US_HOUSE_APPORTIONMENT[apportionment] + except KeyError: + raise ValueError( + f"Unknown apportionment {apportionment!r}; registered: " + f"{sorted(US_HOUSE_APPORTIONMENT)}." + ) from None + return {state: max(count, 1) for state, count in seats.items()} + + +@dataclass(frozen=True) +class CdBlockAssignment: + """One plan's block -> district assignment, as sorted parallel arrays. + + ``block_geoid`` is sorted and unique (``int64``); ``district_geoid`` holds + ``state_fips * 100 + district`` aligned to it. Arrays rather than a dict + because a national plan has about 8.2 million blocks. + """ + + block_geoid: np.ndarray + district_geoid: np.ndarray + + def __post_init__(self) -> None: + if self.block_geoid.shape != self.district_geoid.shape: + raise ValueError("block_geoid and district_geoid must be aligned.") + if len(self.block_geoid) and not bool(np.all(np.diff(self.block_geoid) > 0)): + raise ValueError("CdBlockAssignment block_geoid must be sorted and unique.") + + def __len__(self) -> int: + return len(self.block_geoid) + + def states(self) -> set[str]: + return { + f"{state:02d}" + for state in np.unique(self.block_geoid // _BLOCK_STATE_DIVISOR).tolist() + } + + def for_state(self, state_fips: str) -> CdBlockAssignment: + mask = self.block_geoid // _BLOCK_STATE_DIVISOR == int(state_fips) + return CdBlockAssignment(self.block_geoid[mask], self.district_geoid[mask]) + + +def cd_block_assignment( + pairs: Mapping[int, int] | Iterable[tuple[int, int]], *, label: str +) -> CdBlockAssignment: + """Build an assignment from ``{block: district}`` pairs; duplicates refused.""" + + items = list(pairs.items()) if isinstance(pairs, Mapping) else list(pairs) + blocks = np.fromiter( + (block for block, _ in items), dtype=np.int64, count=len(items) + ) + districts = np.fromiter( + (district for _, district in items), dtype=np.int64, count=len(items) + ) + return _sorted_assignment(blocks, districts, label=label) + + +def parse_cd_block_assignment(lines: Iterable[str], *, label: str) -> CdBlockAssignment: + """Parse a Census CD block file into a block -> district geoid assignment. + + Reads every Census block-to-district format the registry uses, choosing + columns by header name: + + - 2020 Block Assignment File ``CD`` layer: ``BLOCKID|DISTRICT``; + - 118th/119th BEF: ``GEOID,CDFP`` (the 118th state files write + ``GEOID, CDFP``); + - 120th BEF: ``GEOID,STATEFP,COUNTYFP,TRACTCE,BLOCKCE,CDFP``, where + ``STATEFP`` is cross-checked against the geoid; + - a state legislature's plan block file, quoted CSV such as + ``"Block","District"`` or ``"Block_Key","District"`` (these are often on + 2010 blocks; the parser does not care which vintage the geoids are). + + ``ZZ``/``ZZZ``/blank (no district, usually water) rows are dropped; the + delegate code ``98`` and at-large ``00`` both become district ``00``. + A block listed twice is an error. ``lines`` is consumed as a stream. + """ + + iterator = iter(lines) + header = _required_header(iterator, source=label) + delimiter = "|" if "|" in header else "," + columns = [part.strip().strip('"').upper() for part in header.split(delimiter)] + block_index = _column_index(columns, _BLOCK_COLUMNS, label=label) + district_index = _column_index(columns, _DISTRICT_COLUMNS, label=label) + state_index = columns.index("STATEFP") if "STATEFP" in columns else None + width = len(columns) + + blocks = array("q") + districts = array("q") + for line_number, line in enumerate(iterator, start=2): + stripped = line.strip() + if not stripped: + continue + parts = [part.strip().strip('"') for part in stripped.split(delimiter)] + if len(parts) != width: + raise ValueError( + f"{label} line {line_number} has {len(parts)} fields, " + f"expected {width}: {stripped!r}." + ) + district = normalize_district_code(parts[district_index]) + if district is None: + continue + geoid = parts[block_index] + block = _block_geoid(geoid, source=f"{label} line {line_number}") + if state_index is not None and parts[state_index] != geoid[:2]: + raise ValueError( + f"{label} line {line_number}: STATEFP " + f"{parts[state_index]!r} disagrees with GEOID {geoid}." + ) + blocks.append(block) + districts.append((block // _BLOCK_STATE_DIVISOR) * 100 + int(district)) + if not blocks: + raise ValueError(f"{label} contained no district assignments.") + return _sorted_assignment( + np.frombuffer(blocks, dtype=np.int64), + np.frombuffer(districts, dtype=np.int64), + label=label, + ) + + +def concat_cd_block_assignments( + parts: Iterable[CdBlockAssignment], *, label: str +) -> CdBlockAssignment: + """Union per-state assignments into one; a block in two parts is an error.""" + + parts = list(parts) + if not parts: + raise ValueError(f"{label}: no assignments to combine.") + return _sorted_assignment( + np.concatenate([part.block_geoid for part in parts]), + np.concatenate([part.district_geoid for part in parts]), + label=label, + ) + + +def replace_state_assignments( + base: CdBlockAssignment, + replacement: CdBlockAssignment, + *, + state_fips: str, +) -> CdBlockAssignment: + """Return ``base`` with every block of ``state_fips`` taken from ``replacement``. + + Used when one state's plan comes from a different source than the rest of + the plan. ``replacement`` must hold only that state's blocks and cover + every block ``base`` assigns in that state. + """ + + state = int(state_fips) + if (replacement.block_geoid // _BLOCK_STATE_DIVISOR != state).any(): + raise ValueError( + f"Replacement for state {state_fips} contains blocks from other states." + ) + base_in_state = base.block_geoid // _BLOCK_STATE_DIVISOR == state + uncovered = np.setdiff1d(base.block_geoid[base_in_state], replacement.block_geoid) + if len(uncovered): + raise ValueError( + f"Replacement for state {state_fips} leaves {len(uncovered)} block(s) " + f"unassigned, e.g. {int(uncovered[0]):015d}." + ) + return concat_cd_block_assignments( + [ + CdBlockAssignment( + base.block_geoid[~base_in_state], base.district_geoid[~base_in_state] + ), + replacement, + ], + label=f"state {state_fips} replacement", + ) + + +@dataclass(frozen=True) +class BlockRelationship: + """Census 2010 -> 2020 tabulation-block intersections (``rel2020/t10t20``). + + One row per intersecting (2010 block, 2020 block) pair: both 15-digit + geoids and the land and water area of the intersection in square meters. + """ + + block_2010: np.ndarray + block_2020: np.ndarray + area_land: np.ndarray + area_water: np.ndarray + + +_RELATIONSHIP_COLUMNS = ( + "STATE_2010", + "COUNTY_2010", + "TRACT_2010", + "BLK_2010", + "STATE_2020", + "COUNTY_2020", + "TRACT_2020", + "BLK_2020", + "AREALAND_INT", + "AREAWATER_INT", +) + + +def parse_block_relationship_2010_2020( + lines: Iterable[str], *, label: str +) -> BlockRelationship: + """Parse a Census 2010-to-2020 tabulation block relationship file. + + The state files (``TAB2010_TAB2020_ST{fips}.zip``) are pipe-delimited with + a BOM on the header; the block geoid on each side is its state, county, + tract and block columns concatenated. + """ + + iterator = iter(lines) + header = _required_header(iterator, source=label) + columns = [part.strip().upper() for part in header.split("|")] + missing = [name for name in _RELATIONSHIP_COLUMNS if name not in columns] + if missing: + raise ValueError(f"{label} header is missing columns {missing}.") + index = {name: columns.index(name) for name in _RELATIONSHIP_COLUMNS} + width = len(columns) + block_2010 = array("q") + block_2020 = array("q") + area_land = array("q") + area_water = array("q") + for line_number, line in enumerate(iterator, start=2): + stripped = line.rstrip("\r\n") + if not stripped.strip(): + continue + parts = stripped.split("|") + if len(parts) != width: + raise ValueError( + f"{label} line {line_number} has {len(parts)} fields, expected {width}." + ) + source = f"{label} line {line_number}" + block_2010.append( + _block_geoid( + "".join(parts[index[name]] for name in _RELATIONSHIP_COLUMNS[:4]), + source=source, + ) + ) + block_2020.append( + _block_geoid( + "".join(parts[index[name]] for name in _RELATIONSHIP_COLUMNS[4:8]), + source=source, + ) + ) + area_land.append(int(parts[index["AREALAND_INT"]] or 0)) + area_water.append(int(parts[index["AREAWATER_INT"]] or 0)) + if not block_2020: + raise ValueError(f"{label} contained no relationships.") + return BlockRelationship( + block_2010=np.frombuffer(block_2010, dtype=np.int64).copy(), + block_2020=np.frombuffer(block_2020, dtype=np.int64).copy(), + area_land=np.frombuffer(area_land, dtype=np.int64).copy(), + area_water=np.frombuffer(area_water, dtype=np.int64).copy(), + ) + + +def crosswalk_plan_to_2020_blocks( + plan_2010: CdBlockAssignment, + relationship: BlockRelationship, + *, + state_fips: str, +) -> tuple[CdBlockAssignment, dict[str, np.ndarray]]: + """Carry a plan drawn on 2010 blocks onto the 2020 blocks of ``state_fips``. + + Each 2020 block takes the district whose 2010 blocks cover the most of its + land area; ties go to more water area, then the lower district code. A + 2020 block none of whose 2010 counterparts the plan assigns (for example + one whose 2010 side lies in another state) is left out and reported. + + Returns the assignment and diagnostics: ``blocks_2020`` (every 2020 block + of the state in the relationship file), ``straddling`` (2020 blocks whose + 2010 counterparts lie in more than one district) and ``unassigned``. + """ + + state = int(state_fips) + rows = relationship.block_2020 // _BLOCK_STATE_DIVISOR == state + block_2020 = relationship.block_2020[rows] + if len(block_2020) == 0: + raise ValueError( + f"The relationship file has no 2020 blocks in state {state_fips}." + ) + district, covered = _align(plan_2010, relationship.block_2010[rows]) + block = block_2020[covered] + district = district[covered] + land = relationship.area_land[rows][covered] + water = relationship.area_water[rows][covered] + + all_2020 = np.unique(block_2020) + if len(block) == 0: + empty = np.zeros(0, dtype=np.int64) + return CdBlockAssignment(empty, empty.copy()), { + "blocks_2020": all_2020, + "straddling": empty.copy(), + "unassigned": all_2020, + } + + # Sum intersection area per (2020 block, district). + order = np.lexsort((district, block)) + block, district = block[order], district[order] + land, water = land[order], water[order] + starts = np.flatnonzero( + np.r_[True, (np.diff(block) != 0) | (np.diff(district) != 0)] + ) + pair_block = block[starts] + pair_district = district[starts] + pair_land = np.add.reduceat(land, starts) + pair_water = np.add.reduceat(water, starts) + + # Largest land, then water, then the lowest district code, per block. + choice = np.lexsort((pair_district, -pair_water, -pair_land, pair_block)) + ranked_block = pair_block[choice] + first = np.flatnonzero(np.r_[True, np.diff(ranked_block) != 0]) + assigned_block = ranked_block[first] + assigned_district = pair_district[choice][first] + candidates = np.diff(np.r_[first, len(ranked_block)]) + + diagnostics = { + "blocks_2020": all_2020, + "straddling": assigned_block[candidates > 1], + "unassigned": np.setdiff1d(all_2020, assigned_block), + } + return CdBlockAssignment(assigned_block, assigned_district), diagnostics + + +def fill_unassigned_blocks( + assignment: CdBlockAssignment, + unassigned: np.ndarray, + *, + block_geoid: np.ndarray, + population: np.ndarray, +) -> tuple[CdBlockAssignment, list[dict[str, Any]]]: + """Give each unassigned block its neighbourhood's majority district. + + The neighbourhood is the smallest of the block's block group, tract and + county that contains assigned blocks; the district holding most of their + population (``block_geoid``/``population``; blocks absent there count as + zero) wins, then the one with more blocks, then the lower code. Returns + the completed assignment and one record per filled block. + """ + + weights = dict( + zip( + np.asarray(block_geoid).tolist(), + np.asarray(population).tolist(), + strict=True, + ) + ) + fills: list[dict[str, Any]] = [] + new_blocks: list[int] = [] + new_districts: list[int] = [] + for block in sorted(np.asarray(unassigned).tolist()): + for rule, divisor in ( + ("block_group", 10**3), + ("tract", 10**4), + ("county", 10**10), + ): + prefix = block // divisor + lo = int(np.searchsorted(assignment.block_geoid, prefix * divisor)) + hi = int(np.searchsorted(assignment.block_geoid, (prefix + 1) * divisor)) + if hi <= lo: + continue + tally: dict[int, list[int]] = {} + for neighbour, district in zip( + assignment.block_geoid[lo:hi].tolist(), + assignment.district_geoid[lo:hi].tolist(), + strict=True, + ): + entry = tally.setdefault(district, [0, 0]) + entry[0] += weights.get(neighbour, 0) + entry[1] += 1 + district = min(tally, key=lambda d: (-tally[d][0], -tally[d][1], d)) + fills.append( + { + "block": f"{block:015d}", + "district": f"{district % 100:02d}", + "rule": rule, + "population": int(weights.get(block, 0)), + } + ) + new_blocks.append(block) + new_districts.append(district) + break + else: + raise ValueError( + f"No assigned block shares a county with block {block:015d}." + ) + filled = concat_cd_block_assignments( + [ + assignment, + CdBlockAssignment( + np.asarray(new_blocks, dtype=np.int64), + np.asarray(new_districts, dtype=np.int64), + ), + ], + label="filled assignment", + ) + return filled, fills + + +def assignment_agreement( + left: CdBlockAssignment, + right: CdBlockAssignment, + *, + block_geoid: np.ndarray, + population: np.ndarray, +) -> dict[str, Any]: + """Share of the given blocks, and of their population, placed alike. + + Blocks either assignment leaves out count as disagreements. + """ + + blocks = np.asarray(block_geoid, dtype=np.int64) + weights = np.asarray(population, dtype=np.int64) + left_district, left_covered = _align(left, blocks) + right_district, right_covered = _align(right, blocks) + same = left_covered & right_covered & (left_district == right_district) + total = int(weights.sum()) + return { + "blocks": int(len(blocks)), + "blocks_agreeing": int(same.sum()), + "population": total, + "population_agreeing": int(weights[same].sum()), + "population_share_agreeing": ( + float(weights[same].sum() / total) if total else 1.0 + ), + } + + +def assemble_us_cd_plan_registry( + *, + block_geoid: np.ndarray, + population: np.ndarray, + plan_assignments: Mapping[str, CdBlockAssignment], + plan_sources: Mapping[str, Mapping[str, Any]], + metadata: Mapping[str, Any] | None = None, +) -> dict[str, np.ndarray]: + """Join per-plan block assignments into the registry's NPZ payload. + + ``block_geoid``/``population`` define the universe: unique blocks, every + population positive, in any order. Each ``plan_assignments[plan]`` may + also cover blocks outside the universe (unpopulated blocks, Puerto Rico); + those are counted in the plan's metadata and otherwise ignored. Refuses a + populated block a plan leaves unassigned, a district outside its block's + state, and a state whose district count differs from its apportionment. + The payload depends only on the inputs' contents, not their order. + """ + + if not plan_assignments: + raise ValueError("CD plan registry needs at least one plan.") + if set(plan_assignments) != set(plan_sources): + raise ValueError( + "plan_assignments and plan_sources must name the same plans; got " + f"{sorted(plan_assignments)} and {sorted(plan_sources)}." + ) + universe = _int64_array(np.asarray(block_geoid), label="block_geoid") + weights = _int64_array(np.asarray(population), label="population") + if universe.shape != weights.shape or universe.ndim != 1: + raise ValueError("block_geoid and population must be aligned 1-D arrays.") + if len(universe) == 0: + raise ValueError("CD plan registry needs at least one populated block.") + order = np.argsort(universe, kind="stable") + blocks = universe[order] + population = weights[order] + _validate_block_geoids(blocks, label="block_geoid") + if not bool(np.all(np.diff(blocks) > 0)): + raise ValueError("block_geoid values must be unique.") + if (population <= 0).any(): + bad = blocks[population <= 0][:5].tolist() + raise ValueError( + "population must be strictly positive (the universe is populated " + f"blocks); non-positive at {[f'{b:015d}' for b in bad]}." + ) + + payload: dict[str, np.ndarray] = {"block_geoid": blocks, "population": population} + plans_metadata: dict[str, Any] = {} + for plan in sorted(plan_assignments): + spec = _validated_plan_spec(plan, plan_sources[plan]) + districts, covered = _align(plan_assignments[plan], blocks) + if not covered.all(): + missing = blocks[~covered] + by_state: dict[str, int] = {} + for state in (missing // _BLOCK_STATE_DIVISOR).tolist(): + by_state[f"{state:02d}"] = by_state.get(f"{state:02d}", 0) + 1 + raise ValueError( + f"CD plan {plan!r} leaves {len(missing)} populated block(s) " + f"unassigned (by state: {by_state}); examples: " + f"{[f'{b:015d}' for b in missing[:5].tolist()]}." + ) + district_counts = _validate_plan_districts( + blocks, districts, plan=plan, apportionment=spec["apportionment"] + ) + payload[f"{CD_PLAN_ARRAY_PREFIX}{plan}"] = districts.astype(np.int16) + plans_metadata[plan] = { + **spec, + "districts": int(sum(district_counts.values())), + "blocks_outside_universe": int(len(plan_assignments[plan]) - len(blocks)), + } + + body = { + **(dict(metadata) if metadata else {}), + "kind": US_CD_PLAN_REGISTRY_KIND, + "schema_version": US_CD_PLAN_REGISTRY_SCHEMA_VERSION, + "block_vintage": US_CD_PLAN_REGISTRY_BLOCK_VINTAGE, + "primary_plan": PRIMARY_CD_PLAN, + "plans": plans_metadata, + } + payload["metadata_json"] = np.asarray(json.dumps(body, sort_keys=True)) + return payload + + +@dataclass(frozen=True) +class UsCdPlanRegistry: + """Every populated 2020 block's district under each registered plan. + + Attributes: + block_geoid: sorted, unique 15-digit 2020 block geoids (``int64``). + population: 2020 P.L. 94-171 population per block (``int64``, > 0). + plans: district geoid (``state_fips * 100 + district``, ``int64``) + per plan id, aligned to ``block_geoid``. + plan_sources: per-plan provenance (vintage, source, url, source files, + apportionment, known deviations), as recorded in the artifact. + metadata: the artifact's full embedded metadata. + sha256: SHA-256 of the artifact file as loaded. + """ + + block_geoid: np.ndarray + population: np.ndarray + plans: Mapping[str, np.ndarray] + plan_sources: Mapping[str, Mapping[str, Any]] + metadata: Mapping[str, Any] + sha256: str + + def __len__(self) -> int: + return len(self.block_geoid) + + @property + def plan_ids(self) -> tuple[str, ...]: + return tuple(sorted(self.plans)) + + def state_population(self) -> dict[str, int]: + """Population per state FIPS.""" + + return _sum_by( + self.block_geoid // _BLOCK_STATE_DIVISOR, self.population, width=2 + ) + + def district_population(self, plan: str) -> dict[str, int]: + """Population per district geoid (``SSDD``) under ``plan``.""" + + if plan not in self.plans: + raise KeyError( + f"CD plan {plan!r} is not in the registry ({self.plan_ids})." + ) + return _sum_by(self.plans[plan], self.population, width=4) + + +def load_us_cd_plan_registry( + path: str | Path, *, expected_sha256: str | None = None +) -> UsCdPlanRegistry: + """Load and validate a CD plan registry artifact (NPZ). + + Re-checks every build invariant (see the module docstring), so a file + edited after the build fails here. With ``expected_sha256`` the file must + also match that digest; :func:`load_pinned_us_cd_plan_registry` pins to + the packaged provenance. + """ + + source = Path(path) + if not source.exists(): + raise FileNotFoundError(f"US CD plan registry artifact not found: {source}") + digest = _sha256(source) + if expected_sha256 is not None and digest != expected_sha256: + raise ValueError( + f"US CD plan registry {source} has SHA-256 {digest}, expected " + f"{expected_sha256}." + ) + with np.load(source, allow_pickle=False) as arrays: + keys = set(arrays.files) + for required in ("block_geoid", "population", "metadata_json"): + if required not in keys: + raise ValueError(f"US CD plan registry is missing array {required!r}.") + metadata = json.loads(str(arrays["metadata_json"])) + block_geoid = _int64_array(arrays["block_geoid"], label="block_geoid") + population = _int64_array(arrays["population"], label="population") + plan_arrays = { + key[len(CD_PLAN_ARRAY_PREFIX) :]: _int64_array(arrays[key], label=key) + for key in keys + if key.startswith(CD_PLAN_ARRAY_PREFIX) + } + unexpected = { + key + for key in keys - {"block_geoid", "population", "metadata_json"} + if not key.startswith(CD_PLAN_ARRAY_PREFIX) + } + if unexpected: + raise ValueError( + f"US CD plan registry has unexpected arrays {sorted(unexpected)}." + ) + + for key, expected in ( + ("kind", US_CD_PLAN_REGISTRY_KIND), + ("schema_version", US_CD_PLAN_REGISTRY_SCHEMA_VERSION), + ("block_vintage", US_CD_PLAN_REGISTRY_BLOCK_VINTAGE), + ): + if metadata.get(key) != expected: + raise ValueError( + f"US CD plan registry metadata {key}={metadata.get(key)!r}, " + f"expected {expected!r}." + ) + plans_metadata = metadata.get("plans") + if not isinstance(plans_metadata, Mapping) or not plans_metadata: + raise ValueError("US CD plan registry metadata must list its plans.") + if set(plans_metadata) != set(plan_arrays): + raise ValueError( + "US CD plan registry metadata plans " + f"{sorted(plans_metadata)} do not match its arrays {sorted(plan_arrays)}." + ) + + _validate_block_geoids(block_geoid, label="block_geoid") + if len(block_geoid) == 0: + raise ValueError("US CD plan registry has zero blocks.") + if not bool(np.all(np.diff(block_geoid) > 0)): + raise ValueError("US CD plan registry block_geoid must be sorted and unique.") + if len(population) != len(block_geoid): + raise ValueError( + "US CD plan registry population is not aligned to block_geoid." + ) + if (population <= 0).any(): + raise ValueError( + "US CD plan registry population must be positive for every block." + ) + + plan_sources: dict[str, dict[str, Any]] = {} + for plan in sorted(plan_arrays): + spec = _validated_plan_spec(plan, plans_metadata[plan]) + values = plan_arrays[plan] + if len(values) != len(block_geoid): + raise ValueError( + f"CD plan {plan!r} has {len(values)} values for {len(block_geoid)} blocks." + ) + counts = _validate_plan_districts( + block_geoid, values, plan=plan, apportionment=spec["apportionment"] + ) + recorded = plans_metadata[plan].get("districts") + if recorded is not None and recorded != sum(counts.values()): + raise ValueError( + f"CD plan {plan!r} metadata records {recorded} districts, " + f"the array has {sum(counts.values())}." + ) + plan_sources[plan] = dict(plans_metadata[plan]) + + return UsCdPlanRegistry( + block_geoid=block_geoid, + population=population, + plans=plan_arrays, + plan_sources=plan_sources, + metadata=metadata, + sha256=digest, + ) + + +def packaged_us_cd_plan_registry_provenance() -> dict[str, Any]: + """The packaged build receipt: artifact SHA-256, sources, per-plan summary.""" + + resource = files("microcosm.build.us_runtime").joinpath( + US_CD_PLAN_REGISTRY_PROVENANCE_RESOURCE + ) + payload = json.loads(resource.read_text(encoding="utf-8")) + digest = payload.get("output_sha256") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("US CD plan registry provenance has no valid output_sha256.") + return payload + + +def load_pinned_us_cd_plan_registry(path: str | Path) -> UsCdPlanRegistry: + """Load the registry, refusing any file but the one the package pins.""" + + return load_us_cd_plan_registry( + path, expected_sha256=packaged_us_cd_plan_registry_provenance()["output_sha256"] + ) + + +def check_cd_plan_registry_against_block_ladder( + registry: UsCdPlanRegistry, + *, + block_geoid: np.ndarray, + population: np.ndarray, + congressional_district_geoid: np.ndarray, + plan: str = PRIMARY_CD_PLAN, +) -> dict[str, Any]: + """Differential check of the registry against a block ladder's own arrays. + + The ladder and the registry are built independently from the same Census + sources, so they must cover the same blocks with the same populations and + agree exactly on ``plan`` (the ladder's primary CD plan). Raises on any + difference; returns a small receipt on agreement. + """ + + ladder_blocks = _int64_array(np.asarray(block_geoid), label="ladder block_geoid") + order = np.argsort(ladder_blocks, kind="stable") + ladder_blocks = ladder_blocks[order] + ladder_population = _int64_array(np.asarray(population), label="ladder population")[ + order + ] + ladder_cd = _int64_array( + np.asarray(congressional_district_geoid), + label="ladder congressional_district_geoid", + )[order] + if plan not in registry.plans: + raise KeyError( + f"CD plan {plan!r} is not in the registry ({registry.plan_ids})." + ) + if len(ladder_blocks) != len(registry.block_geoid) or not np.array_equal( + ladder_blocks, registry.block_geoid + ): + only_ladder = np.setdiff1d(ladder_blocks, registry.block_geoid) + only_registry = np.setdiff1d(registry.block_geoid, ladder_blocks) + raise ValueError( + "Block ladder and CD plan registry cover different blocks: " + f"{len(only_ladder)} only in the ladder (e.g. " + f"{[f'{b:015d}' for b in only_ladder[:3].tolist()]}), " + f"{len(only_registry)} only in the registry (e.g. " + f"{[f'{b:015d}' for b in only_registry[:3].tolist()]})." + ) + population_mismatch = ladder_population != registry.population + if population_mismatch.any(): + raise ValueError( + f"Block ladder and CD plan registry disagree on the population of " + f"{int(population_mismatch.sum())} block(s), e.g. " + f"{[f'{b:015d}' for b in ladder_blocks[population_mismatch][:3].tolist()]}." + ) + cd_mismatch = ladder_cd != registry.plans[plan] + if cd_mismatch.any(): + raise ValueError( + f"Block ladder and CD plan registry disagree on the {plan} district of " + f"{int(cd_mismatch.sum())} block(s), e.g. " + f"{[f'{b:015d}' for b in ladder_blocks[cd_mismatch][:3].tolist()]}." + ) + return { + "plan": plan, + "blocks": int(len(ladder_blocks)), + "population": int(ladder_population.sum()), + "agreement": "exact", + } + + +def summarize_cd_plan_registry(registry: UsCdPlanRegistry) -> dict[str, Any]: + """Per-plan totals for the build receipt, re-checking conservation. + + For every plan, district populations summed within each state must equal + that state's population; a failure here means the arrays are corrupt. + """ + + state_population = registry.state_population() + plans: dict[str, Any] = {} + for plan in registry.plan_ids: + district_population = registry.district_population(plan) + by_state: dict[str, int] = {} + districts_by_state: dict[str, int] = {} + for district, value in district_population.items(): + by_state[district[:2]] = by_state.get(district[:2], 0) + value + districts_by_state[district[:2]] = ( + districts_by_state.get(district[:2], 0) + 1 + ) + if by_state != state_population: + raise ValueError(f"CD plan {plan!r} does not conserve state populations.") + values = np.asarray(list(district_population.values()), dtype=np.int64) + primary = registry.plans.get(PRIMARY_CD_PLAN) + changed_states: list[str] = [] + if primary is not None and plan != PRIMARY_CD_PLAN: + differs = registry.plans[plan] != primary + changed_states = sorted( + { + f"{state:02d}" + for state in ( + registry.block_geoid[differs] // _BLOCK_STATE_DIVISOR + ).tolist() + } + ) + plans[plan] = { + "districts": int(len(district_population)), + "districts_by_state": dict(sorted(districts_by_state.items())), + "min_district_population": int(values.min()), + "max_district_population": int(values.max()), + "states_differing_from_primary": changed_states, + } + return { + "blocks": int(len(registry)), + "population": int(registry.population.sum()), + "states": len(state_population), + "primary_plan": PRIMARY_CD_PLAN, + "plans": plans, + } + + +def _sorted_assignment( + blocks: np.ndarray, districts: np.ndarray, *, label: str +) -> CdBlockAssignment: + blocks = _int64_array(np.asarray(blocks), label=f"{label} block_geoid") + districts = _int64_array(np.asarray(districts), label=f"{label} district_geoid") + if blocks.shape != districts.shape: + raise ValueError(f"{label}: block and district arrays are not aligned.") + _validate_block_geoids(blocks, label=label) + order = np.argsort(blocks, kind="stable") + blocks = blocks[order] + districts = districts[order] + duplicated = blocks[1:][np.diff(blocks) == 0] + if len(duplicated): + raise ValueError( + f"{label} assigns {len(duplicated)} block(s) more than once, e.g. " + f"{int(duplicated[0]):015d}." + ) + return CdBlockAssignment(blocks, districts) + + +def _align( + assignment: CdBlockAssignment, blocks: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Districts of ``blocks`` (any order) under ``assignment``, and a coverage mask.""" + + if len(assignment) == 0: + return np.zeros(len(blocks), dtype=np.int64), np.zeros(len(blocks), dtype=bool) + index = np.searchsorted(assignment.block_geoid, blocks) + clipped = np.minimum(index, len(assignment) - 1) + covered = assignment.block_geoid[clipped] == blocks + return np.where(covered, assignment.district_geoid[clipped], 0), covered + + +def _validated_plan_spec(plan: str, spec: Mapping[str, Any]) -> dict[str, Any]: + if not _PLAN_ID.fullmatch(plan): + raise ValueError( + f"CD plan id {plan!r} must be lowercase letters, digits and underscores." + ) + if not isinstance(spec, Mapping): + raise ValueError(f"CD plan {plan!r} metadata must be a mapping.") + missing = [key for key in REQUIRED_PLAN_SOURCE_KEYS if not spec.get(key)] + if missing: + raise ValueError(f"CD plan {plan!r} metadata is missing {missing}.") + if spec["vintage"] != plan: + raise ValueError( + f"CD plan {plan!r} records vintage {spec['vintage']!r}; they must match." + ) + if spec["apportionment"] not in US_HOUSE_APPORTIONMENT: + raise ValueError( + f"CD plan {plan!r} apportionment {spec['apportionment']!r} is not one of " + f"{sorted(US_HOUSE_APPORTIONMENT)}." + ) + source_files = spec["source_files"] + if not isinstance(source_files, Mapping): + raise ValueError(f"CD plan {plan!r} source_files must be a mapping.") + for key, entry in source_files.items(): + if not ( + isinstance(entry, Mapping) + and entry.get("url") + and re.fullmatch(r"[0-9a-f]{64}", str(entry.get("sha256", ""))) + ): + raise ValueError( + f"CD plan {plan!r} source file {key!r} must record a url and a SHA-256." + ) + return dict(spec) + + +def _validate_plan_districts( + blocks: np.ndarray, districts: np.ndarray, *, plan: str, apportionment: str +) -> dict[str, int]: + """Check state consistency and district rosters; return counts by state.""" + + if (districts <= 0).any(): + raise ValueError(f"CD plan {plan!r} has non-positive district geoids.") + block_state = blocks // _BLOCK_STATE_DIVISOR + mismatched = districts // 100 != block_state + if mismatched.any(): + examples = [ + f"{block:015d} -> {district:04d}" + for block, district in zip( + blocks[mismatched][:5].tolist(), + districts[mismatched][:5].tolist(), + strict=True, + ) + ] + raise ValueError( + f"CD plan {plan!r} puts {int(mismatched.sum())} block(s) in another " + f"state's district: {examples}." + ) + unique = np.unique(districts) + rosters: dict[str, list[int]] = {} + for district in unique.tolist(): + rosters.setdefault(f"{district // 100:02d}", []).append(district % 100) + expected = expected_district_counts(apportionment) + wrong = {} + for state, roster in sorted(rosters.items()): + count = expected.get(state) + want = ( + None + if count is None + else ([0] if count == 1 else list(range(1, count + 1))) + ) + if roster != want: + wrong[state] = {"districts": roster, "apportioned": count} + if wrong: + raise ValueError( + f"CD plan {plan!r} district rosters differ from the {apportionment} " + f"apportionment (at-large and delegate: 00; otherwise 01..n): {wrong}." + ) + counts = {state: len(roster) for state, roster in rosters.items()} + return counts + + +def _validate_block_geoids(blocks: np.ndarray, *, label: str) -> None: + bad = (blocks < _MIN_BLOCK_GEOID) | (blocks >= _MAX_BLOCK_GEOID) + if bad.any(): + raise ValueError( + f"{label} values must be 15-digit block geoids; invalid: " + f"{blocks[bad][:5].tolist()}." + ) + + +def _sum_by(keys: np.ndarray, values: np.ndarray, *, width: int) -> dict[str, int]: + unique, inverse = np.unique(keys, return_inverse=True) + sums = np.bincount( + inverse, weights=values.astype(np.float64), minlength=len(unique) + ) + # Exact integer sums: bincount's float64 is exact below 2**53, far above + # the US population. + return { + f"{int(key):0{width}d}": int(total) + for key, total in zip(unique.tolist(), sums.tolist(), strict=True) + } + + +def _column_index(columns: list[str], names: tuple[str, ...], *, label: str) -> int: + for name in names: + if name in columns: + return columns.index(name) + raise ValueError(f"{label} header {columns} has none of the columns {list(names)}.") + + +def _required_header(iterator: Iterable[str], *, source: str) -> str: + for line in iterator: + stripped = line.strip().lstrip("") + if stripped: + return stripped + raise ValueError(f"{source} is empty.") + + +def _block_geoid(value: str, *, source: str) -> int: + if not (value.isdigit() and len(value) == 15): + raise ValueError(f"{source}: block geoid must be 15 digits, got {value!r}.") + return int(value) + + +def _int64_array(values: np.ndarray, *, label: str) -> np.ndarray: + if not np.issubdtype(values.dtype, np.integer): + raise ValueError(f"{label} must be an integer array, got {values.dtype}.") + return values.astype(np.int64, copy=False) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +__all__ = [ + "CD_PLAN_ARRAY_PREFIX", + "PRIMARY_CD_PLAN", + "US_CD_PLAN_REGISTRY_KIND", + "US_HOUSE_APPORTIONMENT", + "US_HOUSE_APPORTIONMENT_SOURCE", + "BlockRelationship", + "CdBlockAssignment", + "UsCdPlanRegistry", + "assemble_us_cd_plan_registry", + "assignment_agreement", + "cd_block_assignment", + "check_cd_plan_registry_against_block_ladder", + "concat_cd_block_assignments", + "crosswalk_plan_to_2020_blocks", + "expected_district_counts", + "fill_unassigned_blocks", + "load_pinned_us_cd_plan_registry", + "load_us_cd_plan_registry", + "packaged_us_cd_plan_registry_provenance", + "parse_block_relationship_2010_2020", + "parse_cd_block_assignment", + "replace_state_assignments", + "summarize_cd_plan_registry", +] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json b/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json new file mode 100644 index 000000000..058c52aee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json @@ -0,0 +1,1128 @@ +{ + "apportionment_source": { + "sha256": "d9be9c833ad936ee309a83fa6428a3ae2f82d5d8e5b2c69c4d6ff1e79ba57fcd", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/apportionment/apportionment.csv" + }, + "artifact": "us_cd_plan_registry_2020.npz", + "block_ladder_check": { + "agreement": "exact", + "block_ladder_sha256": "7ba39b959068181b56a7ce1dc8f2477b31af33f28d0a1803ff6cba84cdfab06b", + "blocks": 5769942, + "plan": "119th_congress", + "population": 331449281 + }, + "block_vintage": "2020_tabulation_blocks", + "blocks": 5769942, + "kind": "us_cd_plan_registry", + "output_sha256": "aa2535a9585a6eb4c4a862d6cc65702d0fa1923476c7b0cf8be6fe95f3480c88", + "plan_sources": { + "117th_congress": { + "apportionment": "2010_census", + "known_deviations": { + "37": { + "agreement_with_the_116th_plan": { + "blocks": 174988, + "blocks_agreeing": 101110, + "population": 10439388, + "population_agreeing": 5776745, + "population_share_agreeing": 0.5533605035084432 + }, + "filled_blocks": [ + { + "block": "370710333061012", + "district": "05", + "population": 13, + "rule": "block_group" + }, + { + "block": "370899320003017", + "district": "11", + "population": 0, + "rule": "block_group" + }, + { + "block": "371499203042028", + "district": "11", + "population": 0, + "rule": "block_group" + }, + { + "block": "371690703012013", + "district": "10", + "population": 0, + "rule": "block_group" + } + ], + "method": "Each 2020 block takes the 2019-plan district whose 2010 blocks cover the most of its land area (then water area, then the lower district code). A 2020 block with no North Carolina 2010 counterpart takes the district holding most of its block group's population (else its tract's, else its county's).", + "method_check": { + "blocks": 174988, + "blocks_agreeing": 174919, + "description": "The same method run on the 2016 plan, compared with Census's own 2020-block version of that plan (the 2020 BAF CD layer), over populated blocks.", + "minimum_population_share": 0.999, + "population": 10439388, + "population_agreeing": 10438522, + "population_share_agreeing": 0.9999170449455466 + }, + "note": "North Carolina is the one state whose 117th districts differ from its 116th. Its 117th districts are the 2019 remedial plan (HB 1029 3rd edition, S.L. 2019-249, 'C-Goodwin-A-1-TC'), which Census never tabulated on 2020 blocks, so this state's rows are not from the 2020 BAF CD layer (the 2016 plan). The General Assembly's block file for the 2019 plan (2010 blocks) is carried onto 2020 blocks through the Census 2010-2020 tabulation block relationship file.", + "official_district_population_check": { + "description": "District populations of the carried 2019 plan minus the General Assembly's official 2020-census populations for that plan.", + "differences": { + "01": -28, + "02": 513, + "03": 41, + "04": 11, + "05": 267, + "06": 91, + "07": -170, + "08": -100, + "09": 158, + "10": -704, + "11": -29, + "12": 14, + "13": -64 + }, + "largest_share": 0.000907, + "maximum_share": 0.005 + }, + "source_files": { + "census_block_relationship_2010_2020_nc": { + "member": "tab2010_tab2020_st37_nc.txt", + "sha256": "b8749f2c00fa4339c107433cc8ff875fbfda1d920a32cfb2f5b731cff3abe8a6", + "url": "https://www2.census.gov/geo/docs/maps-data/data/rel2020/t10t20/TAB2010_TAB2020_ST37.zip" + }, + "ncga_2016_plan": { + "member": "2016 Contingent Congressional Plan - Corrected.csv", + "sha256": "6702777cbbd13323adb8ad5be77bda9be0e9b7fe6ecf215cf0aad4a03116a628", + "url": "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2016/baf.zip" + }, + "ncga_2019_plan": { + "member": "C-Goodwin-A-1-TC.csv", + "sha256": "d669b2143e5cb44cef7a37774cc5cd2f1c5ad2ab5a59488d074fa419102dabfa", + "url": "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2019/HB1029%203rd%20Edition%20-%20Blockfile.zip" + }, + "ncga_2020_district_populations_2019_plan": { + "sha256": "7737126c3bef52144495512b780fd66deafab8c45da97d293bd9eaa86f39d5f3", + "url": "https://www.ncleg.gov/Files/GIS/Maps_Reports/Decennial_ReCalc/2020/DeviationReports/2020_Deviation_Rpt_Cong.xlsx" + } + }, + "straddling_blocks": 289, + "straddling_populated_blocks": 183, + "straddling_population": 16671 + } + }, + "notes": "Districts of the 117th Congress (2021-2023), the geography of the IRS SOI congressional-district tables for tax years 2020-2022. Every state but North Carolina comes from the 2020 Block Assignment Files' CD layer, which carries the 116th-Congress plans on 2020 blocks; Census reports North Carolina as the only state whose districts changed between the 116th and 117th Congress (see known_deviations).", + "source": "Census 2020 Block Assignment Files, CD layer", + "source_files": { + "baf2020_ak": { + "member": "BlockAssign_ST02_AK_CD.txt", + "sha256": "686f4283976326621be7173a21036ba2ea65e0b08346e6aac0952718d23b31db", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST02_AK.zip" + }, + "baf2020_al": { + "member": "BlockAssign_ST01_AL_CD.txt", + "sha256": "4031d99d9acc6b0d40fb72123ed2029dc1e66525c6fd1d87ef01228ada18411d", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST01_AL.zip" + }, + "baf2020_ar": { + "member": "BlockAssign_ST05_AR_CD.txt", + "sha256": "199b6bd6c96f2e7b868d307d13c393dffe76f55e15e0df7a67932412351de996", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST05_AR.zip" + }, + "baf2020_az": { + "member": "BlockAssign_ST04_AZ_CD.txt", + "sha256": "0fe372e474a5aea77ebc421f926185a7318fc7c963cf74813ac7bd4c2af0e1cd", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST04_AZ.zip" + }, + "baf2020_ca": { + "member": "BlockAssign_ST06_CA_CD.txt", + "sha256": "edc540d7e782507c080d6caea087b302f3f406c8572629763f0f7017ebe427a1", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST06_CA.zip" + }, + "baf2020_co": { + "member": "BlockAssign_ST08_CO_CD.txt", + "sha256": "9199c97ed89fe42cfea45492095a26c02ae8eda42656abd220a907fdf39c917c", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST08_CO.zip" + }, + "baf2020_ct": { + "member": "BlockAssign_ST09_CT_CD.txt", + "sha256": "1125c49564224cced90844ab2345df9a2c191a0084c5029ff53ddc9c320c54ba", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST09_CT.zip" + }, + "baf2020_dc": { + "member": "BlockAssign_ST11_DC_CD.txt", + "sha256": "c789c1df0921dfdd9925d1c9747cec0b1c51895f3c1cfd5ee8938e15277eaa48", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST11_DC.zip" + }, + "baf2020_de": { + "member": "BlockAssign_ST10_DE_CD.txt", + "sha256": "8f81f5239dca88f378f098325ae630d215bf0c5fd8986a088d2b2b2c4746f6d9", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST10_DE.zip" + }, + "baf2020_fl": { + "member": "BlockAssign_ST12_FL_CD.txt", + "sha256": "a8eeda07f30d1010f984950dae030bb6276c6d11a99f64a0a4bbebccd9996451", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST12_FL.zip" + }, + "baf2020_ga": { + "member": "BlockAssign_ST13_GA_CD.txt", + "sha256": "97fee9eee9eb0b338e9f8ea937c2d667c5293a4c1e38182bfe1c4786c4dbd111", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST13_GA.zip" + }, + "baf2020_hi": { + "member": "BlockAssign_ST15_HI_CD.txt", + "sha256": "99ff6595ad6566804cc4682c9e73b9d854ae127154bffb9b0f2e6e97a7fa8147", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST15_HI.zip" + }, + "baf2020_ia": { + "member": "BlockAssign_ST19_IA_CD.txt", + "sha256": "883baba3cc35a45b585293e58ce23c08d3644c5f4e59a92dc329c76d4fde1447", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST19_IA.zip" + }, + "baf2020_id": { + "member": "BlockAssign_ST16_ID_CD.txt", + "sha256": "31c1e32901138ff732f9eded36bde73231af5e1529fd3a9317d106561a7a4f8a", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST16_ID.zip" + }, + "baf2020_il": { + "member": "BlockAssign_ST17_IL_CD.txt", + "sha256": "97c913144c29b33ebaa17546e4976bb19e5cf65d33b26a1ea7609736aeb2c9b6", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST17_IL.zip" + }, + "baf2020_in": { + "member": "BlockAssign_ST18_IN_CD.txt", + "sha256": "3158704fa33bc433087d21eff4f93702d0999b3f405085fb58fb64e72cc58eb0", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST18_IN.zip" + }, + "baf2020_ks": { + "member": "BlockAssign_ST20_KS_CD.txt", + "sha256": "d331ad02ad4ba10a15bce5691c22834b3ca853357637bd3f6df02cc9c75fb816", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST20_KS.zip" + }, + "baf2020_ky": { + "member": "BlockAssign_ST21_KY_CD.txt", + "sha256": "e3b7573bbad03ab78db313535b0e763ddb1268f0b0d59d760aa0e688793d055a", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST21_KY.zip" + }, + "baf2020_la": { + "member": "BlockAssign_ST22_LA_CD.txt", + "sha256": "4b160562136f38e136b4384d591c5fcda351c6da2486bab07b39cc7576afe79e", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST22_LA.zip" + }, + "baf2020_ma": { + "member": "BlockAssign_ST25_MA_CD.txt", + "sha256": "6d77b72047fef535efe396b675d4ec8cedfbc975e4cab9460431b861353e4f18", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST25_MA.zip" + }, + "baf2020_md": { + "member": "BlockAssign_ST24_MD_CD.txt", + "sha256": "0cff392f34e07b00f2705bab056c902d67c63a32cfbbe30534777ea147d79b93", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST24_MD.zip" + }, + "baf2020_me": { + "member": "BlockAssign_ST23_ME_CD.txt", + "sha256": "431631b3352f8bab61258626a824529434e3f51e421e62890f215e47d46cda64", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST23_ME.zip" + }, + "baf2020_mi": { + "member": "BlockAssign_ST26_MI_CD.txt", + "sha256": "d819d7d0d2b5d83e0e5aab54d2674b5a625bde51add10f257a1511abd6014a88", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST26_MI.zip" + }, + "baf2020_mn": { + "member": "BlockAssign_ST27_MN_CD.txt", + "sha256": "afc53ab51e43091e4454f54e0d57b61f60a73f0d521977a4165965fefabc3e61", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST27_MN.zip" + }, + "baf2020_mo": { + "member": "BlockAssign_ST29_MO_CD.txt", + "sha256": "9d72554bbb23c0f859741641fcadd98bd2bd4858067ad010eddc881b9f14ccad", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST29_MO.zip" + }, + "baf2020_ms": { + "member": "BlockAssign_ST28_MS_CD.txt", + "sha256": "d8e4d0ef5f467650394cc902883669143724cd31699a601e52ef185d52c60b8f", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST28_MS.zip" + }, + "baf2020_mt": { + "member": "BlockAssign_ST30_MT_CD.txt", + "sha256": "7e5fde3990b536705f337bc7c3ffde2da3b2a49b7c9e58c22d6c198c7c0a93a8", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST30_MT.zip" + }, + "baf2020_nc": { + "member": "BlockAssign_ST37_NC_CD.txt", + "sha256": "9619665fc781f4ab45891a4bea3160d7bc3877d9570ba64e7b1b42224c225bf0", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST37_NC.zip" + }, + "baf2020_nd": { + "member": "BlockAssign_ST38_ND_CD.txt", + "sha256": "a89f5f58b9882aaf520c5c77c52b3b49dc198ff533b8436bf04ebfed252529fa", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST38_ND.zip" + }, + "baf2020_ne": { + "member": "BlockAssign_ST31_NE_CD.txt", + "sha256": "9065e5d1e048af09777e815294c201ae548ebeb73f02195cfb2c895aa89ae39e", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST31_NE.zip" + }, + "baf2020_nh": { + "member": "BlockAssign_ST33_NH_CD.txt", + "sha256": "c06b3a004317849b253ef32972f2a6a7e0abf9f71c496e4ddf5578c66c6b7c17", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST33_NH.zip" + }, + "baf2020_nj": { + "member": "BlockAssign_ST34_NJ_CD.txt", + "sha256": "179a04d0e2f07f516298218002afe304999fbf272b59387ebb18d550eb34e430", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST34_NJ.zip" + }, + "baf2020_nm": { + "member": "BlockAssign_ST35_NM_CD.txt", + "sha256": "e7507ee4cb1780233151c87aa2a84901dc86fede8680709e71fa14ad13b4df1e", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST35_NM.zip" + }, + "baf2020_nv": { + "member": "BlockAssign_ST32_NV_CD.txt", + "sha256": "ea0e6834f8389127a74eb4be4e28b86659fe067af7d4fa0ab91f746060d4adb9", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST32_NV.zip" + }, + "baf2020_ny": { + "member": "BlockAssign_ST36_NY_CD.txt", + "sha256": "7cf9a890beb47baca04bfa35c74db9300e1044c8a99b2cff559ade8fc1a89236", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST36_NY.zip" + }, + "baf2020_oh": { + "member": "BlockAssign_ST39_OH_CD.txt", + "sha256": "c2b0b18a5a5230572e7ca9a73ca3a0bcbab6e2ffb83505230f614cc23641c1c5", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST39_OH.zip" + }, + "baf2020_ok": { + "member": "BlockAssign_ST40_OK_CD.txt", + "sha256": "3859f1e17929d0ce91d09b750d31a3da39de398a3fc607044d2a6025452996f6", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST40_OK.zip" + }, + "baf2020_or": { + "member": "BlockAssign_ST41_OR_CD.txt", + "sha256": "ad8fe9e57a32aa434b38b92cfff510e0fccbc367721f0c7043732ba9233cc674", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST41_OR.zip" + }, + "baf2020_pa": { + "member": "BlockAssign_ST42_PA_CD.txt", + "sha256": "f61fb24709191cb1ee9f97b915fe37ef483e4b5f8bca79385790da8ad6d80bf3", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST42_PA.zip" + }, + "baf2020_ri": { + "member": "BlockAssign_ST44_RI_CD.txt", + "sha256": "0833228c38a62f9c005b0ac9cb238e2c003e270380679657f7eb2f574966c896", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST44_RI.zip" + }, + "baf2020_sc": { + "member": "BlockAssign_ST45_SC_CD.txt", + "sha256": "ee313741c4dff930cd8959855f6651cf823daf1f82f2f5e28bef17abb67d0d59", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST45_SC.zip" + }, + "baf2020_sd": { + "member": "BlockAssign_ST46_SD_CD.txt", + "sha256": "b790819eb1f0ea1d0c4fe98c87c61989230817e66804c2ae1d79c379547d8eaa", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST46_SD.zip" + }, + "baf2020_tn": { + "member": "BlockAssign_ST47_TN_CD.txt", + "sha256": "96dc990f409bbfbd02dedfd75caf6c18d65d70b8102e54100bc1b5a4e3812744", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST47_TN.zip" + }, + "baf2020_tx": { + "member": "BlockAssign_ST48_TX_CD.txt", + "sha256": "db3f1a4e06aca3d61fb15f1d3581cd08d65120588f33cf00c8265c9f043aa725", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST48_TX.zip" + }, + "baf2020_ut": { + "member": "BlockAssign_ST49_UT_CD.txt", + "sha256": "57ebe75d5353f004a466d4a7df6897207b1ef19d47849b61b188f9969c0bc2ce", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST49_UT.zip" + }, + "baf2020_va": { + "member": "BlockAssign_ST51_VA_CD.txt", + "sha256": "87ebb3582232400286fb0ff4e6a51fdfb1449221c780c9d1a3a574f752010f36", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST51_VA.zip" + }, + "baf2020_vt": { + "member": "BlockAssign_ST50_VT_CD.txt", + "sha256": "6869df8eecd778986f116f4138ce5d0f692468b944056eb58984dfa51ba3909d", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST50_VT.zip" + }, + "baf2020_wa": { + "member": "BlockAssign_ST53_WA_CD.txt", + "sha256": "58114c250c830d81ac8f499f29ae71ecae545624f484a1f9cce448182cf09001", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST53_WA.zip" + }, + "baf2020_wi": { + "member": "BlockAssign_ST55_WI_CD.txt", + "sha256": "75bed98fefe7454b6fc6c5e972744380bf99400d3c6a3b839e7a5e1fb13d093c", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST55_WI.zip" + }, + "baf2020_wv": { + "member": "BlockAssign_ST54_WV_CD.txt", + "sha256": "949ed08edf0a082f6a9a0c4300fb3c850358115cf2cc0354437116d7da7c060e", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST54_WV.zip" + }, + "baf2020_wy": { + "member": "BlockAssign_ST56_WY_CD.txt", + "sha256": "e4ce0356da93caff0870166d44e8491dc5b386580dd4e8504ad1957e76f37f09", + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST56_WY.zip" + }, + "census_block_relationship_2010_2020_nc": { + "member": "tab2010_tab2020_st37_nc.txt", + "sha256": "b8749f2c00fa4339c107433cc8ff875fbfda1d920a32cfb2f5b731cff3abe8a6", + "url": "https://www2.census.gov/geo/docs/maps-data/data/rel2020/t10t20/TAB2010_TAB2020_ST37.zip" + }, + "ncga_2016_plan": { + "member": "2016 Contingent Congressional Plan - Corrected.csv", + "sha256": "6702777cbbd13323adb8ad5be77bda9be0e9b7fe6ecf215cf0aad4a03116a628", + "url": "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2016/baf.zip" + }, + "ncga_2019_plan": { + "member": "C-Goodwin-A-1-TC.csv", + "sha256": "d669b2143e5cb44cef7a37774cc5cd2f1c5ad2ab5a59488d074fa419102dabfa", + "url": "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2019/HB1029%203rd%20Edition%20-%20Blockfile.zip" + }, + "ncga_2020_district_populations_2019_plan": { + "sha256": "7737126c3bef52144495512b780fd66deafab8c45da97d293bd9eaa86f39d5f3", + "url": "https://www.ncleg.gov/Files/GIS/Maps_Reports/Decennial_ReCalc/2020/DeviationReports/2020_Deviation_Rpt_Cong.xlsx" + } + }, + "url": "https://www2.census.gov/geo/docs/maps-data/data/baf2020/BlockAssign_ST{fips}_{usps}.zip", + "vintage": "117th_congress" + }, + "118th_congress": { + "apportionment": "2020_census", + "block_splits": { + "080010096072000": { + "districts_within_block": [ + "07", + "08" + ], + "tabulated_with": "08" + } + }, + "census_state_files": [ + "01", + "02", + "04", + "05", + "06", + "08", + "09", + "10", + "11", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "53", + "54", + "55", + "56", + "72" + ], + "notes": "Districts of the 118th Congress (2023-2025), the first plans drawn on 2020 census geography.", + "published": "2022-12-16", + "source": "Census 118th Congressional District block equivalency file (National_CD118.txt)", + "source_files": { + "118th_congress": { + "member": "National_CD118.txt", + "sha256": "a2f38d0dd7c207fa144a88b66df9f59f8a6e5c27e932fbd4a32d1bcf587c5763", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2023/118-congressional-district-bef/cd118.zip" + }, + "118th_congress_block_split_note": { + "sha256": "016d7a697d79d84187828d19c8546878598f0e745bee88942c0749ef3875cb37", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2023/118-congressional-district-bef/CD118_BlockSplits.pdf" + } + }, + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2023/118-congressional-district-bef/cd118.zip", + "vintage": "118th_congress" + }, + "119th_congress": { + "apportionment": "2020_census", + "block_splits": { + "080010096072000": { + "districts_within_block": [ + "07", + "08" + ], + "tabulated_with": "08" + } + }, + "census_state_files": [ + "01", + "13", + "22", + "36", + "37" + ], + "notes": "Districts of the 119th Congress (2025-2027); the block ladder's primary plan.", + "published": "2024-07-29", + "source": "Census 119th Congressional District block equivalency file (NationalCD119.txt)", + "source_files": { + "119th_congress": { + "member": "NationalCD119.txt", + "sha256": "1433feb5178dc7b4188ee30f5f7f715851f4400740b8fe1ce606a876c6294bd6", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2025/119-congressional-district-befs/cd119.zip" + }, + "119th_congress_block_split_note": { + "sha256": "f0b70f2bde47285f63367ef667a6080ec5c2042842af1e2457c0fb4b739dadcc", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2025/119-congressional-district-befs/CD119_BlockSplits.pdf" + } + }, + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2025/119-congressional-district-befs/cd119.zip", + "vintage": "119th_congress" + }, + "120th_congress": { + "apportionment": "2020_census", + "block_splits": { + "080010096072000": { + "districts_within_block": [ + "07", + "08" + ], + "tabulated_with": "08" + } + }, + "census_state_files": [ + "01", + "06", + "12", + "22", + "29", + "37", + "39", + "47", + "48", + "49" + ], + "known_deviations": { + "29": { + "as_of": "2026-09-27", + "evidence": [ + "https://www.census.gov/geographies/mapping-files/2027/dec/rdo/120-congressional-district-bef.html", + "https://www.supremecourt.gov/opinions/25pdf/26a388_q86b.pdf", + "https://statecourtreport.org/case-tracker/von-glahn-v-hoskins" + ], + "from_plan": "119th_congress", + "note": "Census's 120th rows for Missouri encode the 2025 HB 1 map, and Census's own 120th landing page warns they may not be the districts in place for November 2026. The Missouri Supreme Court held on 2026-09-03 (von Glahn v. Hoskins, SC101805) that HB 1 cannot take effect before a referendum on the November 2026 ballot, and the U.S. Supreme Court held on 2026-09-25 (People Not Politicians v. Onder, 26A388, per curiam) that 'the 2022 map\u2014not the 2025 map\u2014must be used in the 2026 congressional election.' Missouri's rows here are its 2022 map, taken from the 119th file (the build checks they are unchanged from the 118th)." + } + }, + "notes": "Districts for the November 3, 2026 election (120th Congress, 2027-2029), including the 2025-26 mid-decade redraws: Census's 120th Congressional District block equivalency file, except any state whose map in force differs from Census's rows (see known_deviations).", + "published": "2026-08-31", + "source": "Census 120th Congressional District block equivalency file (NationalCD120.txt)", + "source_files": { + "120th_congress": { + "member": "NationalCD120.txt", + "sha256": "f3074ac5a3b353f9da22baaeb73c47ddd1c9040fffe7f5131f947f35c25a1097", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2027/120-congressional-district-befs/cd120.zip" + }, + "120th_congress_block_split_note": { + "sha256": "722fa5d22d2d6c5c4a687686a6a039f515f0de045a4dbb16d967cc398a80a388", + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2027/120-congressional-district-befs/CD120_BlockSplits.pdf" + } + }, + "url": "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files/2027/120-congressional-district-befs/cd120.zip", + "vintage": "120th_congress" + } + }, + "plans": { + "117th_congress": { + "districts": 436, + "districts_by_state": { + "01": 7, + "02": 1, + "04": 9, + "05": 4, + "06": 53, + "08": 7, + "09": 5, + "10": 1, + "11": 1, + "12": 27, + "13": 14, + "15": 2, + "16": 2, + "17": 18, + "18": 9, + "19": 4, + "20": 4, + "21": 6, + "22": 6, + "23": 2, + "24": 8, + "25": 9, + "26": 14, + "27": 8, + "28": 4, + "29": 8, + "30": 1, + "31": 3, + "32": 4, + "33": 2, + "34": 12, + "35": 3, + "36": 27, + "37": 13, + "38": 1, + "39": 16, + "40": 5, + "41": 5, + "42": 18, + "44": 2, + "45": 7, + "46": 1, + "47": 9, + "48": 36, + "49": 4, + "50": 1, + "51": 11, + "53": 10, + "54": 3, + "55": 8, + "56": 1 + }, + "max_district_population": 1084225, + "min_district_population": 543072, + "states_differing_from_primary": [ + "01", + "04", + "05", + "06", + "08", + "09", + "12", + "13", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "39", + "40", + "41", + "42", + "44", + "45", + "47", + "48", + "49", + "51", + "53", + "54", + "55" + ] + }, + "118th_congress": { + "districts": 436, + "districts_by_state": { + "01": 7, + "02": 1, + "04": 9, + "05": 4, + "06": 52, + "08": 8, + "09": 5, + "10": 1, + "11": 1, + "12": 28, + "13": 14, + "15": 2, + "16": 2, + "17": 17, + "18": 9, + "19": 4, + "20": 4, + "21": 6, + "22": 6, + "23": 2, + "24": 8, + "25": 9, + "26": 13, + "27": 8, + "28": 4, + "29": 8, + "30": 2, + "31": 3, + "32": 4, + "33": 2, + "34": 12, + "35": 3, + "36": 26, + "37": 14, + "38": 1, + "39": 15, + "40": 5, + "41": 6, + "42": 17, + "44": 2, + "45": 7, + "46": 1, + "47": 9, + "48": 38, + "49": 4, + "50": 1, + "51": 11, + "53": 10, + "54": 2, + "55": 8, + "56": 1 + }, + "max_district_population": 989948, + "min_district_population": 542112, + "states_differing_from_primary": [ + "01", + "13", + "22", + "36", + "37" + ] + }, + "119th_congress": { + "districts": 436, + "districts_by_state": { + "01": 7, + "02": 1, + "04": 9, + "05": 4, + "06": 52, + "08": 8, + "09": 5, + "10": 1, + "11": 1, + "12": 28, + "13": 14, + "15": 2, + "16": 2, + "17": 17, + "18": 9, + "19": 4, + "20": 4, + "21": 6, + "22": 6, + "23": 2, + "24": 8, + "25": 9, + "26": 13, + "27": 8, + "28": 4, + "29": 8, + "30": 2, + "31": 3, + "32": 4, + "33": 2, + "34": 12, + "35": 3, + "36": 26, + "37": 14, + "38": 1, + "39": 15, + "40": 5, + "41": 6, + "42": 17, + "44": 2, + "45": 7, + "46": 1, + "47": 9, + "48": 38, + "49": 4, + "50": 1, + "51": 11, + "53": 10, + "54": 2, + "55": 8, + "56": 1 + }, + "max_district_population": 989948, + "min_district_population": 542112, + "states_differing_from_primary": [] + }, + "120th_congress": { + "districts": 436, + "districts_by_state": { + "01": 7, + "02": 1, + "04": 9, + "05": 4, + "06": 52, + "08": 8, + "09": 5, + "10": 1, + "11": 1, + "12": 28, + "13": 14, + "15": 2, + "16": 2, + "17": 17, + "18": 9, + "19": 4, + "20": 4, + "21": 6, + "22": 6, + "23": 2, + "24": 8, + "25": 9, + "26": 13, + "27": 8, + "28": 4, + "29": 8, + "30": 2, + "31": 3, + "32": 4, + "33": 2, + "34": 12, + "35": 3, + "36": 26, + "37": 14, + "38": 1, + "39": 15, + "40": 5, + "41": 6, + "42": 17, + "44": 2, + "45": 7, + "46": 1, + "47": 9, + "48": 38, + "49": 4, + "50": 1, + "51": 11, + "53": 10, + "54": 2, + "55": 8, + "56": 1 + }, + "max_district_population": 989948, + "min_district_population": 542112, + "states_differing_from_primary": [ + "01", + "06", + "12", + "22", + "37", + "39", + "47", + "48", + "49" + ] + } + }, + "population": 331449281, + "population_source": { + "source": "2020 Census P.L. 94-171 geographic headers, POP100", + "source_files": { + "pl94171_ak": { + "member": "akgeo2020.pl", + "sha256": "c90837c12ab2f5fe99132c558705e8d0b5174afdf2a0d28b71927f535fe63487", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Alaska/ak2020.pl.zip" + }, + "pl94171_al": { + "member": "algeo2020.pl", + "sha256": "bd3fbbf22615b166869f9119aa12017b9a7b82baf0f22a9170e92a88a397cd77", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Alabama/al2020.pl.zip" + }, + "pl94171_ar": { + "member": "argeo2020.pl", + "sha256": "940ad2916676e443dfdab98c3e145fcf62ef9dcd9f10c763ccdb195f4d5f132e", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Arkansas/ar2020.pl.zip" + }, + "pl94171_az": { + "member": "azgeo2020.pl", + "sha256": "56997e8b1b5175148fd3e1157ab57eee7c1722221cafebfd5d12fe2e687b2aff", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Arizona/az2020.pl.zip" + }, + "pl94171_ca": { + "member": "cageo2020.pl", + "sha256": "e02c83c85a9a5d69ded4bf0d4c8ebe3f4bcd2d8e5911db765e9e42915ad1f753", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/California/ca2020.pl.zip" + }, + "pl94171_co": { + "member": "cogeo2020.pl", + "sha256": "fce76008aedd10681bf0e57fa366ca788d280264edfe91e7d80a9b33ae86ab2c", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Colorado/co2020.pl.zip" + }, + "pl94171_ct": { + "member": "ctgeo2020.pl", + "sha256": "c4a2d42a79c52de10ea9397cca43db59c2df38e6d6e66b4396a0de8bd3f66780", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Connecticut/ct2020.pl.zip" + }, + "pl94171_dc": { + "member": "dcgeo2020.pl", + "sha256": "21b5189f45ce260c4607c9579f4ad52d11920133af827b14b183121b48d40ce4", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/District_of_Columbia/dc2020.pl.zip" + }, + "pl94171_de": { + "member": "degeo2020.pl", + "sha256": "2d7743cc442574dc785c728e9637c5d4648bef518beef41b5f03756eb56ea78a", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Delaware/de2020.pl.zip" + }, + "pl94171_fl": { + "member": "flgeo2020.pl", + "sha256": "fdcc98180f634b69c40ed1d71a63fe00fdda4cda3e35fbaa6382336501f4b0cf", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Florida/fl2020.pl.zip" + }, + "pl94171_ga": { + "member": "gageo2020.pl", + "sha256": "26434588ceab435fa33a138ff999e4187c923a74732fcc488c3a0597b4adebcb", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Georgia/ga2020.pl.zip" + }, + "pl94171_hi": { + "member": "higeo2020.pl", + "sha256": "b7e1ba6350c5d75267b688504052af001ad227763995f6bb26ccdd92fc45261f", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Hawaii/hi2020.pl.zip" + }, + "pl94171_ia": { + "member": "iageo2020.pl", + "sha256": "c55e8276080b684431b8629269d4225c31080ea17257a737f9d99e3cf2ac6822", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Iowa/ia2020.pl.zip" + }, + "pl94171_id": { + "member": "idgeo2020.pl", + "sha256": "89345b6d5d12ccdec8af4039b0a6f1f5ba4cd7ca86c8f73ad6250b5713e3df47", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Idaho/id2020.pl.zip" + }, + "pl94171_il": { + "member": "ilgeo2020.pl", + "sha256": "e73f82d6aea0b9b372e9f8572c6edbedfa72ae9fc2039da84faf0cdf0c478a32", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Illinois/il2020.pl.zip" + }, + "pl94171_in": { + "member": "ingeo2020.pl", + "sha256": "5f4999237a9bf3b3337865494d0d62c0a1dc620cad7e48caebaa83811f95901a", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Indiana/in2020.pl.zip" + }, + "pl94171_ks": { + "member": "ksgeo2020.pl", + "sha256": "ac6b4ae5e2af4c8d639b46748d55927ee7821756f6073cc4f8238c1fb00498ee", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Kansas/ks2020.pl.zip" + }, + "pl94171_ky": { + "member": "kygeo2020.pl", + "sha256": "252a04b760e4da69665d5654108be2b63d55d294c1c0c608bfc7e7fcf2360ac4", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Kentucky/ky2020.pl.zip" + }, + "pl94171_la": { + "member": "lageo2020.pl", + "sha256": "d8d9ea96195f2b848be70ef33dcb224d433d8d58025e11aaa0432c5211e5345c", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Louisiana/la2020.pl.zip" + }, + "pl94171_ma": { + "member": "mageo2020.pl", + "sha256": "9857362fbc1162d3943685134a77a91542a70eb667f3477559d07fd0a81e4889", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Massachusetts/ma2020.pl.zip" + }, + "pl94171_md": { + "member": "mdgeo2020.pl", + "sha256": "9af1e1f0185b6219a9432132ef78e47c2d74c474948d2ed6004c7061412393f3", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Maryland/md2020.pl.zip" + }, + "pl94171_me": { + "member": "megeo2020.pl", + "sha256": "f9bb6fe4cf13bb28b44e4d6a8d5d3e5209f9249ebc402db7ea5c24f8a673ac07", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Maine/me2020.pl.zip" + }, + "pl94171_mi": { + "member": "migeo2020.pl", + "sha256": "971bd53abeb1d905bb9b09bfe4dc1afe8514a916f24d285b289e0f66ec5cfb62", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Michigan/mi2020.pl.zip" + }, + "pl94171_mn": { + "member": "mngeo2020.pl", + "sha256": "7d30974597e73893d25ffef0975c8e673afcc1166f9919d5ff2a466a2e0e188f", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Minnesota/mn2020.pl.zip" + }, + "pl94171_mo": { + "member": "mogeo2020.pl", + "sha256": "900d8a3bcc59e2cc3c174730e2a8704bd653af1fc043c9fbd73196e1c0d7fba5", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Missouri/mo2020.pl.zip" + }, + "pl94171_ms": { + "member": "msgeo2020.pl", + "sha256": "cb64e3465cee9623dc574605f4dfcada33eaea6833062d31f769811fd20a5f26", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Mississippi/ms2020.pl.zip" + }, + "pl94171_mt": { + "member": "mtgeo2020.pl", + "sha256": "e01288ce3ae37ae732eb61d40268c6ac3ec8f9772c8609c6e9ca538bdd0bc130", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Montana/mt2020.pl.zip" + }, + "pl94171_nc": { + "member": "ncgeo2020.pl", + "sha256": "04f2f250dad877d237d94f553f9cb2209594c2a021d15f9fdeedd4404db538db", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/North_Carolina/nc2020.pl.zip" + }, + "pl94171_nd": { + "member": "ndgeo2020.pl", + "sha256": "a37d18e05a73b311a8fd7ade36dd5b17cac16ff125318c9ec6d2edd7f6968b71", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/North_Dakota/nd2020.pl.zip" + }, + "pl94171_ne": { + "member": "negeo2020.pl", + "sha256": "29298d68a5905cf9cb14cd64e35750d14727a330c5940ccf5238a8ae6e110300", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Nebraska/ne2020.pl.zip" + }, + "pl94171_nh": { + "member": "nhgeo2020.pl", + "sha256": "31852c0cfa871ea81f933bb1df904f0d45997097e38ade83db87ba58255d4468", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/New_Hampshire/nh2020.pl.zip" + }, + "pl94171_nj": { + "member": "njgeo2020.pl", + "sha256": "caf0da4a9e9d46ced3f8020a7a2bf5240cd8805ed0679716f0cb2b8fcc65e811", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/New_Jersey/nj2020.pl.zip" + }, + "pl94171_nm": { + "member": "nmgeo2020.pl", + "sha256": "77133ba4bb2fb2c28d39993bb4cff5f1c42823c9d13b75e51e8ecf986354c259", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/New_Mexico/nm2020.pl.zip" + }, + "pl94171_nv": { + "member": "nvgeo2020.pl", + "sha256": "ac7e792e0a1d6248ff1700159f4ec6a3a7b3330ebe10899bb4f47ab97c19952d", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Nevada/nv2020.pl.zip" + }, + "pl94171_ny": { + "member": "nygeo2020.pl", + "sha256": "9d046014bee6b61afa808c5dfe841a6d036c77bff70197472fbdd291a40b8ce2", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/New_York/ny2020.pl.zip" + }, + "pl94171_oh": { + "member": "ohgeo2020.pl", + "sha256": "ceef338247ad2dccc267885f2879e167ab6730c7a332d3ceb5aef9d6e2338aa0", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Ohio/oh2020.pl.zip" + }, + "pl94171_ok": { + "member": "okgeo2020.pl", + "sha256": "e95c99e6385f4ed3a5ea67962444a53ad7a731f26d1dd88df2efde504f101dff", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Oklahoma/ok2020.pl.zip" + }, + "pl94171_or": { + "member": "orgeo2020.pl", + "sha256": "f8e7fa8c331446850b1bf545e6c4aa63733373f582b11edd6b3cf31016bc4509", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Oregon/or2020.pl.zip" + }, + "pl94171_pa": { + "member": "pageo2020.pl", + "sha256": "2d33a7dab29c8dd5692bbde203d253e06eebbc44fcbaa96b1caa958d454026ae", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Pennsylvania/pa2020.pl.zip" + }, + "pl94171_ri": { + "member": "rigeo2020.pl", + "sha256": "a2a6de20d0bac0dfe91aef66d3c2dde0eb768863884fbc45d4f3df5c49f5dcd0", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Rhode_Island/ri2020.pl.zip" + }, + "pl94171_sc": { + "member": "scgeo2020.pl", + "sha256": "e8996f6873405d6c048c42e8878231fc7f152564295498ff586a041648a46bb5", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/South_Carolina/sc2020.pl.zip" + }, + "pl94171_sd": { + "member": "sdgeo2020.pl", + "sha256": "30d4edc3ccc86ad76a2a388c763ecdaebe23f43a5dcb92658587c04ca5cfa1a5", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/South_Dakota/sd2020.pl.zip" + }, + "pl94171_tn": { + "member": "tngeo2020.pl", + "sha256": "89f843dbc717409169fef44df2682548c308c06c0d825451bf2e78340234daf2", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Tennessee/tn2020.pl.zip" + }, + "pl94171_tx": { + "member": "txgeo2020.pl", + "sha256": "6fee8ca5759eb4942db25a59b8cbd3439ad4188279fccff87dac70a89ceca6ff", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Texas/tx2020.pl.zip" + }, + "pl94171_ut": { + "member": "utgeo2020.pl", + "sha256": "4e6de13cd7b5307bfdd327446dd7107e2fdb790903f59186bb8285881531d0f0", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Utah/ut2020.pl.zip" + }, + "pl94171_va": { + "member": "vageo2020.pl", + "sha256": "e33c7c1ac0aa84b5fe5ba20188d2e562c5c96b1c02f59e766e18d037b23df9e8", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Virginia/va2020.pl.zip" + }, + "pl94171_vt": { + "member": "vtgeo2020.pl", + "sha256": "6a18a3364dff5e0432b25f4b4972eaf22ac0699ea0273430509a9b6a763638e9", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Vermont/vt2020.pl.zip" + }, + "pl94171_wa": { + "member": "wageo2020.pl", + "sha256": "dde77f2db632a4af274a968c39a7ab58fac6cfd90718e4a72ec239b84980db5b", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Washington/wa2020.pl.zip" + }, + "pl94171_wi": { + "member": "wigeo2020.pl", + "sha256": "e70f80ef4a803cdeb1e1767fc42e13f7d01ce9f98566c82ca4c2379aa9809f7f", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Wisconsin/wi2020.pl.zip" + }, + "pl94171_wv": { + "member": "wvgeo2020.pl", + "sha256": "8356c49f300e9c18658de56c06e551f5e8cb591c395fc3d1748077d46322889b", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/West_Virginia/wv2020.pl.zip" + }, + "pl94171_wy": { + "member": "wygeo2020.pl", + "sha256": "98dd7d035942fea2758e0fcabe71efa0588a7ec7c9c33110bcde493fe8e8b942", + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/Wyoming/wy2020.pl.zip" + } + }, + "url": "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/{dirname}/{usps_lower}2020.pl.zip" + }, + "primary_plan": "119th_congress", + "schema_version": 1, + "states": 51 +} diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py new file mode 100644 index 000000000..b4ce3d2ba --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py @@ -0,0 +1,519 @@ +"""Example tests for the US block -> congressional-district plan registry. + +Property-based tests of the same invariants live in +``test_us_cd_plan_registry_properties.py``. The opt-in test at the bottom +checks a real built artifact when ``MICROCOSM_US_CD_PLAN_REGISTRY`` points at +one. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path + +import numpy as np +import pytest + +from microcosm.build.us_runtime.block_ladder_sources import US_STATES +from microcosm.build.us_runtime.cd_plan_registry import ( + CD_PLAN_ARRAY_PREFIX, + PRIMARY_CD_PLAN, + US_HOUSE_APPORTIONMENT, + CdBlockAssignment, + assemble_us_cd_plan_registry, + cd_block_assignment, + check_cd_plan_registry_against_block_ladder, + expected_district_counts, + load_pinned_us_cd_plan_registry, + load_us_cd_plan_registry, + packaged_us_cd_plan_registry_provenance, + parse_cd_block_assignment, + replace_state_assignments, + summarize_cd_plan_registry, +) + +SHA = "0" * 64 + + +def _spec(plan: str, apportionment: str = "2020_census") -> dict: + return { + "vintage": plan, + "source": f"test source for {plan}", + "url": f"https://example.test/{plan}.zip", + "apportionment": apportionment, + "source_files": { + plan: {"url": f"https://example.test/{plan}.zip", "sha256": SHA} + }, + } + + +# Delaware (one seat) and Rhode Island (two seats in both apportionments). +DE = [10_001_0401_00_1000 + i for i in range(3)] +RI = [44_001_0301_00_1000 + i for i in range(4)] +BLOCKS = np.asarray(DE + RI, dtype=np.int64) +POPULATION = np.asarray([10, 20, 30, 40, 50, 60, 70], dtype=np.int64) +PLAN_119 = { + **dict.fromkeys(DE, 1000), + RI[0]: 4401, + RI[1]: 4401, + RI[2]: 4402, + RI[3]: 4402, +} +PLAN_120 = {**PLAN_119, RI[1]: 4402} + + +def _payload(**overrides): + kwargs = { + "block_geoid": BLOCKS, + "population": POPULATION, + "plan_assignments": { + "119th_congress": cd_block_assignment(PLAN_119, label="119"), + "120th_congress": cd_block_assignment(PLAN_120, label="120"), + }, + "plan_sources": { + "119th_congress": _spec("119th_congress"), + "120th_congress": _spec("120th_congress"), + }, + } + kwargs.update(overrides) + return assemble_us_cd_plan_registry(**kwargs) + + +def _write(tmp_path: Path, payload, name: str = "registry.npz") -> Path: + path = tmp_path / name + np.savez_compressed(path, **payload) + return path + + +# --- parsing ------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("lines", "label"), + [ + ( + [ + "BLOCKID|DISTRICT\n", + "100010401001000|00\n", + "440010301001000|02\n", + "440010301001001|ZZ\n", + ], + "BAF CD layer", + ), + ( + [ + "GEOID, CDFP\n", + "100010401001000,00\n", + "440010301001000,02\n", + "440010301001001,ZZ\n", + ], + "118th state file", + ), + ( + [ + "GEOID,CDFP\r\n", + "100010401001000,00\r\n", + "440010301001000,02\r\n", + "440010301001001,ZZ\r\n", + ], + "119th national file", + ), + ( + [ + "GEOID,STATEFP,COUNTYFP,TRACTCE,BLOCKCE,CDFP\n", + "100010401001000,10,001,040100,1000,00\n", + "440010301001000,44,001,030100,1000,02\n", + "440010301001001,44,001,030100,1001,ZZ\n", + ], + "120th national file", + ), + ], +) +def test_parser_reads_every_census_format(lines, label): + parsed = parse_cd_block_assignment(lines, label=label) + assert parsed.block_geoid.tolist() == [100010401001000, 440010301001000] + assert parsed.district_geoid.tolist() == [1000, 4402] + + +def test_parser_maps_the_delegate_code_to_district_00(): + parsed = parse_cd_block_assignment(["GEOID,CDFP", "110010001011000,98"], label="DC") + assert parsed.district_geoid.tolist() == [1100] + + +@pytest.mark.parametrize( + ("lines", "message"), + [ + (["GEOID,CDFP", "100010401001000,00", "100010401001000,00"], "more than once"), + (["GEOID,CDFP", "10001040100100,00"], "15 digits"), + (["GEOID,CDFP", "100010401001000,00,7"], "fields"), + (["GEOID,CDFP", "100010401001000,X1"], "two-digit"), + ( + [ + "GEOID,STATEFP,COUNTYFP,TRACTCE,BLOCKCE,CDFP", + "100010401001000,44,001,040100,1000,00", + ], + "STATEFP", + ), + (["GEOID,DISTRICTS", "100010401001000,00"], "none of the columns"), + (["GEOID,CDFP", "100010401001000,ZZ"], "no district assignments"), + ([], "empty"), + ], +) +def test_parser_refuses_malformed_files(lines, message): + with pytest.raises(ValueError, match=message): + parse_cd_block_assignment(lines, label="test") + + +def test_parser_consumes_a_stream_lazily(): + def lines(): + yield "GEOID,CDFP" + for suffix in range(1000): + yield f"{100010401000000 + suffix},00" + + parsed = parse_cd_block_assignment(lines(), label="stream") + assert len(parsed) == 1000 + assert bool(np.all(np.diff(parsed.block_geoid) > 0)) + + +# --- apportionment ------------------------------------------------------- + + +def test_apportionment_tables_cover_every_state_and_sum_to_435(): + fips = {state for state, _, _ in US_STATES} + for census, seats in US_HOUSE_APPORTIONMENT.items(): + assert set(seats) == fips, census + assert sum(seats.values()) == 435, census + + +def test_expected_district_counts_give_dc_and_at_large_states_one_district(): + counts = expected_district_counts("2020_census") + assert counts["11"] == 1 # DC delegate + assert counts["56"] == 1 # Wyoming at-large + assert counts["30"] == 2 # Montana gained a seat in 2020 + assert expected_district_counts("2010_census")["30"] == 1 + assert sum(counts.values()) == 436 + with pytest.raises(ValueError, match="Unknown apportionment"): + expected_district_counts("1990_census") + + +# --- assembly and loading ------------------------------------------------- + + +def test_round_trip_preserves_every_plan_and_conserves_population(tmp_path): + registry = load_us_cd_plan_registry(_write(tmp_path, _payload())) + assert registry.plan_ids == ("119th_congress", "120th_congress") + assert registry.block_geoid.tolist() == BLOCKS.tolist() + assert registry.plans["119th_congress"].tolist() == [ + PLAN_119[b] for b in BLOCKS.tolist() + ] + assert registry.state_population() == {"10": 60, "44": 220} + assert registry.district_population("119th_congress") == { + "1000": 60, + "4401": 90, + "4402": 130, + } + assert registry.district_population("120th_congress") == { + "1000": 60, + "4401": 40, + "4402": 180, + } + summary = summarize_cd_plan_registry(registry) + assert summary["population"] == 280 + assert summary["plans"]["120th_congress"]["states_differing_from_primary"] == ["44"] + assert summary["plans"]["119th_congress"]["states_differing_from_primary"] == [] + assert registry.plan_sources["120th_congress"]["districts"] == 3 + assert registry.plan_sources["120th_congress"]["blocks_outside_universe"] == 0 + + +def test_plan_arrays_are_stored_as_int16_and_loaded_as_int64(tmp_path): + payload = _payload() + assert payload[f"{CD_PLAN_ARRAY_PREFIX}119th_congress"].dtype == np.int16 + registry = load_us_cd_plan_registry(_write(tmp_path, payload)) + assert registry.plans["119th_congress"].dtype == np.int64 + + +def test_assembly_refuses_a_populated_block_without_a_district(): + gap = {block: district for block, district in PLAN_120.items() if block != RI[3]} + with pytest.raises(ValueError, match="1 populated block"): + _payload( + plan_assignments={ + "119th_congress": cd_block_assignment(PLAN_119, label="119"), + "120th_congress": cd_block_assignment(gap, label="120"), + } + ) + + +def test_assembly_refuses_a_district_in_another_state(): + crossed = {**PLAN_120, DE[0]: 4401} + with pytest.raises(ValueError, match="another state's district"): + _payload( + plan_assignments={ + "119th_congress": cd_block_assignment(PLAN_119, label="119"), + "120th_congress": cd_block_assignment(crossed, label="120"), + } + ) + + +def test_assembly_refuses_a_district_count_off_the_apportionment(): + merged = {**PLAN_120, RI[0]: 4402} # Rhode Island down to one district + with pytest.raises(ValueError, match="apportionment"): + _payload( + plan_assignments={ + "119th_congress": cd_block_assignment(PLAN_119, label="119"), + "120th_congress": cd_block_assignment(merged, label="120"), + } + ) + + +def test_assembly_refuses_incomplete_plan_metadata(): + for key in ("vintage", "source", "url", "apportionment", "source_files"): + spec = _spec("120th_congress") + del spec[key] + with pytest.raises(ValueError, match="missing"): + _payload( + plan_sources={ + "119th_congress": _spec("119th_congress"), + "120th_congress": spec, + } + ) + mismatched = {**_spec("120th_congress"), "vintage": "119th_congress"} + with pytest.raises(ValueError, match="must match"): + _payload( + plan_sources={ + "119th_congress": _spec("119th_congress"), + "120th_congress": mismatched, + } + ) + + +def test_blocks_outside_the_universe_are_counted_and_ignored(tmp_path): + unpopulated = 44_001_0301_00_2000 + puerto_rico = 72_001_9501_00_1000 + extended = {**PLAN_120, unpopulated: 4401, puerto_rico: 7200} + registry = load_us_cd_plan_registry( + _write( + tmp_path, + _payload( + plan_assignments={ + "119th_congress": cd_block_assignment(PLAN_119, label="119"), + "120th_congress": cd_block_assignment(extended, label="120"), + } + ), + ) + ) + assert registry.plan_sources["120th_congress"]["blocks_outside_universe"] == 2 + assert unpopulated not in registry.block_geoid.tolist() + + +def test_loader_pins_the_sha256(tmp_path): + path = _write(tmp_path, _payload()) + registry = load_us_cd_plan_registry(path) + assert ( + load_us_cd_plan_registry(path, expected_sha256=registry.sha256).sha256 + == registry.sha256 + ) + with pytest.raises(ValueError, match="expected"): + load_us_cd_plan_registry(path, expected_sha256="f" * 64) + + +def test_loader_refuses_an_artifact_edited_after_the_build(tmp_path): + payload = _payload() + crossed = payload[f"{CD_PLAN_ARRAY_PREFIX}120th_congress"].copy() + crossed[0] = 4401 # a Delaware block moved into Rhode Island + with pytest.raises(ValueError, match="another state's district"): + load_us_cd_plan_registry( + _write( + tmp_path, {**payload, f"{CD_PLAN_ARRAY_PREFIX}120th_congress": crossed} + ) + ) + + metadata = json.loads(str(payload["metadata_json"])) + metadata["plans"]["120th_congress"]["districts"] = 4 + with pytest.raises(ValueError, match="records 4 districts"): + load_us_cd_plan_registry( + _write( + tmp_path, + {**payload, "metadata_json": np.asarray(json.dumps(metadata))}, + "districts.npz", + ) + ) + + unsorted = {**payload, "block_geoid": payload["block_geoid"][::-1].copy()} + with pytest.raises(ValueError, match="sorted and unique"): + load_us_cd_plan_registry(_write(tmp_path, unsorted, "unsorted.npz")) + + extra = {**payload, "unexpected": np.zeros(1)} + with pytest.raises(ValueError, match="unexpected arrays"): + load_us_cd_plan_registry(_write(tmp_path, extra, "extra.npz")) + + missing_plan = dict(payload) + del missing_plan[f"{CD_PLAN_ARRAY_PREFIX}120th_congress"] + with pytest.raises(ValueError, match="do not match its arrays"): + load_us_cd_plan_registry(_write(tmp_path, missing_plan, "missing.npz")) + + +def test_ladder_differential_accepts_agreement_and_refuses_any_difference(tmp_path): + registry = load_us_cd_plan_registry(_write(tmp_path, _payload())) + order = np.asarray([6, 0, 3, 1, 5, 2, 4]) # ladder rows in any order + ladder_cd = np.asarray([PLAN_119[b] for b in BLOCKS.tolist()], dtype=np.int64) + receipt = check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS[order], + population=POPULATION[order], + congressional_district_geoid=ladder_cd[order], + ) + assert receipt == { + "plan": PRIMARY_CD_PLAN, + "blocks": 7, + "population": 280, + "agreement": "exact", + } + + changed_cd = ladder_cd.copy() + changed_cd[4] = 4402 + with pytest.raises(ValueError, match="119th_congress district of 1 block"): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS, + population=POPULATION, + congressional_district_geoid=changed_cd, + ) + changed_population = POPULATION.copy() + changed_population[0] += 1 + with pytest.raises(ValueError, match="population of 1 block"): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS, + population=changed_population, + congressional_district_geoid=ladder_cd, + ) + with pytest.raises(ValueError, match="different blocks"): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS[1:], + population=POPULATION[1:], + congressional_district_geoid=ladder_cd[1:], + ) + + +def test_replace_state_assignments_swaps_exactly_one_state(): + base = cd_block_assignment(PLAN_120, label="base") + donor = cd_block_assignment(PLAN_119, label="donor").for_state("44") + merged = replace_state_assignments(base, donor, state_fips="44") + assert ( + dict( + zip( + merged.block_geoid.tolist(), merged.district_geoid.tolist(), strict=True + ) + ) + == PLAN_119 + ) + + with pytest.raises(ValueError, match="other states"): + replace_state_assignments( + base, cd_block_assignment(PLAN_119, label="all"), state_fips="44" + ) + partial = CdBlockAssignment(donor.block_geoid[:2], donor.district_geoid[:2]) + with pytest.raises(ValueError, match="leaves 2 block"): + replace_state_assignments(base, partial, state_fips="44") + + +def test_cd_block_assignment_refuses_duplicates_and_bad_geoids(): + with pytest.raises(ValueError, match="more than once"): + cd_block_assignment([(DE[0], 1000), (DE[0], 1000)], label="dup") + with pytest.raises(ValueError, match="15-digit"): + cd_block_assignment({12345: 1000}, label="short") + + +# --- packaged provenance --------------------------------------------------- + + +def test_packaged_provenance_is_complete_and_matches_the_apportionment(): + receipt = packaged_us_cd_plan_registry_provenance() + assert re.fullmatch(r"[0-9a-f]{64}", receipt["output_sha256"]) + assert receipt["kind"] == "us_cd_plan_registry" + assert receipt["block_vintage"] == "2020_tabulation_blocks" + assert receipt["primary_plan"] == PRIMARY_CD_PLAN + # The block ladder's universe (US_PUMA_LADDER.md): every populated block + # of the 50 states and DC, the 2020 apportionment population. + assert receipt["blocks"] == 5_769_942 + assert receipt["population"] == 331_449_281 + assert receipt["states"] == 51 + assert set(receipt["plans"]) == { + "117th_congress", + "118th_congress", + "119th_congress", + "120th_congress", + } + for plan, summary in receipt["plans"].items(): + spec = receipt["plan_sources"][plan] + assert spec["vintage"] == plan + assert summary["districts"] == 436, plan + assert summary["districts_by_state"] == expected_district_counts( + spec["apportionment"] + ), plan + for entry in spec["source_files"].values(): + assert entry["url"].startswith( + ("https://www2.census.gov/", "https://www.ncleg.gov/") + ) + assert re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) + plans = receipt["plans"] + assert plans["119th_congress"]["states_differing_from_primary"] == [] + # Census's redrawn states for the 118th -> 119th and 119th -> 120th. + assert plans["118th_congress"]["states_differing_from_primary"] == [ + "01", + "13", + "22", + "36", + "37", + ] + census_120th = receipt["plan_sources"]["120th_congress"]["census_state_files"] + assert census_120th == ["01", "06", "12", "22", "29", "37", "39", "47", "48", "49"] + # Missouri reverts to its 2022 map; every other Census redraw stands. + assert plans["120th_congress"]["states_differing_from_primary"] == [ + state for state in census_120th if state != "29" + ] + missouri = receipt["plan_sources"]["120th_congress"]["known_deviations"]["29"] + assert missouri["from_plan"] == "119th_congress" + + north_carolina = receipt["plan_sources"]["117th_congress"]["known_deviations"]["37"] + method = north_carolina["method_check"] + assert method["population_share_agreeing"] >= method["minimum_population_share"] + official = north_carolina["official_district_population_check"] + assert official["largest_share"] <= official["maximum_share"] + assert sorted(official["differences"]) == [f"{d:02d}" for d in range(1, 14)] + assert set(receipt["plan_sources"]["117th_congress"]["known_deviations"]) == {"37"} + check = receipt["block_ladder_check"] + assert check["agreement"] == "exact" + assert check["plan"] == PRIMARY_CD_PLAN + assert check["blocks"] == receipt["blocks"] + assert check["population"] == receipt["population"] + assert re.fullmatch(r"[0-9a-f]{64}", check["block_ladder_sha256"]) + + +# --- a real artifact (opt-in) ----------------------------------------------- + + +@pytest.mark.skipif( + not os.environ.get("MICROCOSM_US_CD_PLAN_REGISTRY"), + reason="set MICROCOSM_US_CD_PLAN_REGISTRY to a built registry to check it", +) +def test_built_registry_matches_its_packaged_receipt(): + registry = load_pinned_us_cd_plan_registry( + os.environ["MICROCOSM_US_CD_PLAN_REGISTRY"] + ) + receipt = packaged_us_cd_plan_registry_provenance() + summary = summarize_cd_plan_registry(registry) + for key in ("blocks", "population", "states", "primary_plan", "plans"): + assert summary[key] == receipt[key], key + ladder_path = os.environ.get("MICROCOSM_US_BLOCK_LADDER") + if ladder_path: + with np.load(ladder_path, allow_pickle=False) as ladder: + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=ladder["block_geoid"], + population=ladder["population"], + congressional_district_geoid=ladder["congressional_district_geoid"], + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_crosswalk.py b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_crosswalk.py new file mode 100644 index 000000000..edaeaaf19 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_crosswalk.py @@ -0,0 +1,189 @@ +"""Examples for carrying a 2010-block plan onto 2020 blocks. + +North Carolina's 117th-Congress plan exists only on 2010 blocks, so the +registry build carries it through the Census 2010-2020 block relationship +file. Properties of the same functions are in +``test_us_cd_plan_registry_properties.py``. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from microcosm.build.us_runtime.cd_plan_registry import ( + BlockRelationship, + CdBlockAssignment, + assignment_agreement, + cd_block_assignment, + crosswalk_plan_to_2020_blocks, + fill_unassigned_blocks, + parse_block_relationship_2010_2020, + parse_cd_block_assignment, +) + +HEADER = ( + "STATE_2010|COUNTY_2010|TRACT_2010|BLK_2010|BLKSF_2010|AREALAND_2010|" + "AREAWATER_2010|BLOCK_PART_FLAG_O|STATE_2020|COUNTY_2020|TRACT_2020|BLK_2020|" + "BLKSF_2020|AREALAND_2020|AREAWATER_2020|BLOCK_PART_FLAG_R|AREALAND_INT|" + "AREAWATER_INT" +) + + +def _row(block_2010: str, block_2020: str, land: int, water: int = 0) -> str: + def split(geoid: str) -> str: + return f"{geoid[:2]}|{geoid[2:5]}|{geoid[5:11]}|{geoid[11:]}" + + return f"{split(block_2010)}||0|0|p|{split(block_2020)}||0|0|p|{land}|{water}" + + +def test_relationship_parser_builds_geoids_and_reads_intersection_areas(): + relationship = parse_block_relationship_2010_2020( + [ + "" + HEADER + "\r\n", + _row("370010201001000", "370010201001000", 29235) + "\r\n", + _row("131110501001001", "370399306042005", 783, 5) + "\r\n", + ], + label="rel", + ) + assert relationship.block_2010.tolist() == [370010201001000, 131110501001001] + assert relationship.block_2020.tolist() == [370010201001000, 370399306042005] + assert relationship.area_land.tolist() == [29235, 783] + assert relationship.area_water.tolist() == [0, 5] + + +def test_relationship_parser_refuses_a_missing_column(): + with pytest.raises(ValueError, match="missing columns"): + parse_block_relationship_2010_2020( + [HEADER.replace("AREALAND_INT", "AREALAND")], label="rel" + ) + + +def test_parser_reads_a_legislature_plan_block_file(): + plan = parse_cd_block_assignment( + ['"Block","District"', '"371139706001037","11"', '"370010201001000","1"'], + label="C-Goodwin-A-1-TC.csv", + ) + assert plan.block_geoid.tolist() == [370010201001000, 371139706001037] + assert plan.district_geoid.tolist() == [3701, 3711] + keyed = parse_cd_block_assignment( + ['"Block_Key","District"', '"370010205012023","06"'], label="2016 plan" + ) + assert keyed.district_geoid.tolist() == [3706] + + +# 2010 blocks A, B (district 1) and C (district 2); 2020 blocks X, Y, Z, W. +A, B, C, OUT = 370010001001000, 370010001001001, 370010001002000, 450010001001000 +X, Y, Z, W = 370010001001000, 370010001001001, 370010001001002, 370010001002001 +PLAN_2010 = cd_block_assignment({A: 3701, B: 3701, C: 3702}, label="2010 plan") + + +def _relationship(rows): + return BlockRelationship( + block_2010=np.asarray([r[0] for r in rows], dtype=np.int64), + block_2020=np.asarray([r[1] for r in rows], dtype=np.int64), + area_land=np.asarray([r[2] for r in rows], dtype=np.int64), + area_water=np.asarray([r[3] for r in rows], dtype=np.int64), + ) + + +def test_crosswalk_takes_the_largest_land_area_summed_by_district(): + relationship = _relationship( + [ + (A, X, 100, 0), # X wholly in district 1 + (A, Y, 30, 0), # Y: district 1 has 30 + 30 = 60 of land... + (B, Y, 30, 0), + (C, Y, 50, 0), # ...beating district 2's single largest piece + (C, Z, 10, 0), # Z: land tie, broken by water + (A, Z, 10, 5), + (OUT, W, 99, 0), # W's only counterpart is in another state + ] + ) + carried, diagnostics = crosswalk_plan_to_2020_blocks( + PLAN_2010, relationship, state_fips="37" + ) + assert dict( + zip(carried.block_geoid.tolist(), carried.district_geoid.tolist(), strict=True) + ) == {X: 3701, Y: 3701, Z: 3701} + assert diagnostics["straddling"].tolist() == [Y, Z] + assert diagnostics["unassigned"].tolist() == [W] + assert diagnostics["blocks_2020"].tolist() == [X, Y, Z, W] + + +def test_crosswalk_breaks_a_full_tie_by_the_lower_district(): + relationship = _relationship([(C, X, 10, 0), (A, X, 10, 0)]) + carried, _ = crosswalk_plan_to_2020_blocks(PLAN_2010, relationship, state_fips="37") + assert carried.district_geoid.tolist() == [3701] + + +def test_crosswalk_refuses_a_state_the_file_does_not_cover(): + relationship = _relationship([(A, X, 1, 0)]) + with pytest.raises(ValueError, match="no 2020 blocks in state 45"): + crosswalk_plan_to_2020_blocks(PLAN_2010, relationship, state_fips="45") + + +def test_fill_uses_the_smallest_neighbourhood_with_assigned_blocks(): + # Block group 3700100010010 holds X (district 1, 5 people) and Y (district + # 2, 50 people); W sits alone in block group ...0020 of the same tract. + assigned = cd_block_assignment({X: 3701, Y: 3702}, label="assigned") + same_group = 370010001001009 + filled, fills = fill_unassigned_blocks( + assigned, + np.asarray([W, same_group]), + block_geoid=np.asarray([X, Y, W]), + population=np.asarray([5, 50, 7]), + ) + placed = dict( + zip(filled.block_geoid.tolist(), filled.district_geoid.tolist(), strict=True) + ) + assert placed[same_group] == 3702 # block group majority by population + assert placed[W] == 3702 # no assigned block in W's group: tract majority + assert fills == [ + { + "block": f"{same_group:015d}", + "district": "02", + "rule": "block_group", + "population": 0, + }, + {"block": f"{W:015d}", "district": "02", "rule": "tract", "population": 7}, + ] + + +def test_fill_refuses_a_block_with_no_assigned_county_neighbour(): + assigned = cd_block_assignment({X: 3701}, label="assigned") + with pytest.raises(ValueError, match="shares a county"): + fill_unassigned_blocks( + assigned, + np.asarray([370030001001000]), + block_geoid=np.asarray([X]), + population=np.asarray([1]), + ) + + +def test_agreement_counts_blocks_and_people_and_treats_gaps_as_disagreement(): + left = cd_block_assignment({X: 3701, Y: 3701, Z: 3702}, label="left") + right = CdBlockAssignment(np.asarray([X, Y]), np.asarray([3701, 3702])) + agreement = assignment_agreement( + left, + right, + block_geoid=np.asarray([X, Y, Z]), + population=np.asarray([10, 20, 30]), + ) + assert agreement == { + "blocks": 3, + "blocks_agreeing": 1, + "population": 60, + "population_agreeing": 10, + "population_share_agreeing": 10 / 60, + } + + +def test_crosswalk_with_no_assigned_counterpart_leaves_every_block_out(): + # Minimized counterexample: the only 2010 counterpart is outside the plan. + relationship = _relationship([(OUT, X, 5, 0)]) + carried, diagnostics = crosswalk_plan_to_2020_blocks( + PLAN_2010, relationship, state_fips="37" + ) + assert len(carried) == 0 + assert diagnostics["unassigned"].tolist() == [X] + assert diagnostics["straddling"].tolist() == [] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py new file mode 100644 index 000000000..df7cc62ce --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py @@ -0,0 +1,470 @@ +"""Property-based tests for the US block -> congressional-district plan registry. + +Each test draws a random multi-state geography (blocks, populations, and one +district assignment per plan that respects the real House apportionment of +each drawn state) and checks an invariant for every such input: + +- conservation: for every plan, district populations sum exactly to state + populations, which sum to the national total; +- partition: every block has exactly one district per plan, in its own state, + and each state has exactly its apportioned number of districts; +- determinism: the artifact's bytes depend only on the inputs' contents, not + their order; +- fail-closed: dropping, crossing or merging any district assignment is + refused at build time, and the ladder differential refuses any change; +- crosswalk: carrying a 2010-block plan onto 2020 blocks gives each 2020 + block the district with the most intersecting land (then water, then the + lower code), leaves out exactly the blocks with no assigned counterpart, + and does not depend on the relationship file's row order. +""" + +from __future__ import annotations + +import hashlib +import io +from pathlib import Path + +import numpy as np +import pytest + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import HealthCheck, given, settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from microcosm.build.us_runtime.cd_plan_registry import ( # noqa: E402 + CD_PLAN_ARRAY_PREFIX, + PRIMARY_CD_PLAN, + BlockRelationship, + assemble_us_cd_plan_registry, + cd_block_assignment, + check_cd_plan_registry_against_block_ladder, + crosswalk_plan_to_2020_blocks, + expected_district_counts, + fill_unassigned_blocks, + load_us_cd_plan_registry, + replace_state_assignments, + summarize_cd_plan_registry, +) + +#: Plans under test and the apportionment each must satisfy. +PLANS = { + "117th_congress": "2010_census", + "119th_congress": "2020_census", + "120th_congress": "2020_census", +} +#: Small states (few districts) so drawn geographies stay small. Montana and +#: West Virginia change seat counts between 2010 and 2020. +SMALL_STATES = ("10", "11", "15", "16", "23", "30", "31", "33", "35", "44", "54") +MAX_DISTRICTS = max( + expected_district_counts(apportionment)[state] + for apportionment in PLANS.values() + for state in SMALL_STATES +) + +SETTINGS = settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +def _spec(plan: str) -> dict: + url = f"https://example.test/{plan}.zip" + return { + "vintage": plan, + "source": f"test source for {plan}", + "url": url, + "apportionment": PLANS[plan], + "source_files": {plan: {"url": url, "sha256": "0" * 64}}, + } + + +@st.composite +def geographies(draw): + """Blocks, populations and a valid district map per plan.""" + + states = draw( + st.lists(st.sampled_from(SMALL_STATES), min_size=1, max_size=4, unique=True) + ) + blocks: list[int] = [] + population: list[int] = [] + plans: dict[str, dict[int, int]] = {plan: {} for plan in PLANS} + for state in sorted(states): + suffixes = draw( + st.lists( + st.integers(min_value=0, max_value=10**13 - 1), + min_size=MAX_DISTRICTS, + max_size=MAX_DISTRICTS + 25, + unique=True, + ) + ) + state_blocks = [int(state) * 10**13 + suffix for suffix in suffixes] + blocks.extend(state_blocks) + population.extend( + draw( + st.lists( + st.integers(1, 5_000), + min_size=len(state_blocks), + max_size=len(state_blocks), + ) + ) + ) + for plan, apportionment in PLANS.items(): + count = expected_district_counts(apportionment)[state] + codes = [0] if count == 1 else list(range(1, count + 1)) + order = draw(st.permutations(state_blocks)) + # The first ``count`` blocks seed one district each, so no + # apportioned district is empty; the rest land anywhere. + for position, block in enumerate(order): + code = ( + codes[position] + if position < count + else draw(st.sampled_from(codes)) + ) + plans[plan][block] = int(state) * 100 + code + return ( + np.asarray(blocks, dtype=np.int64), + np.asarray(population, dtype=np.int64), + plans, + ) + + +def _assemble(blocks, population, plans, **overrides): + kwargs = { + "block_geoid": blocks, + "population": population, + "plan_assignments": { + plan: cd_block_assignment(mapping, label=plan) + for plan, mapping in plans.items() + }, + "plan_sources": {plan: _spec(plan) for plan in plans}, + } + kwargs.update(overrides) + return assemble_us_cd_plan_registry(**kwargs) + + +def _npz_bytes(payload) -> bytes: + buffer = io.BytesIO() + np.savez_compressed(buffer, **payload) + return buffer.getvalue() + + +def _load(tmp_path: Path, payload, name: str): + path = tmp_path / f"{name}.npz" + path.write_bytes(_npz_bytes(payload)) + return load_us_cd_plan_registry(path) + + +@SETTINGS +@given(geography=geographies()) +def test_every_plan_partitions_and_conserves_state_population( + tmp_path_factory, geography +): + blocks, population, plans = geography + registry = _load( + tmp_path_factory.mktemp("r"), _assemble(blocks, population, plans), "r" + ) + + state_population: dict[str, int] = {} + for block, people in zip(blocks.tolist(), population.tolist(), strict=True): + state_population[f"{block // 10**13:02d}"] = ( + state_population.get(f"{block // 10**13:02d}", 0) + people + ) + assert registry.state_population() == state_population + assert int(registry.population.sum()) == int(population.sum()) + + for plan, mapping in plans.items(): + # Exactly one district per block, the one the source gave it. + assert registry.plans[plan].tolist() == [ + mapping[b] for b in registry.block_geoid.tolist() + ] + # Every district lies in its block's state. + assert (registry.plans[plan] // 100 == registry.block_geoid // 10**13).all() + districts = registry.district_population(plan) + by_state: dict[str, int] = {} + count_by_state: dict[str, int] = {} + for district, people in districts.items(): + by_state[district[:2]] = by_state.get(district[:2], 0) + people + count_by_state[district[:2]] = count_by_state.get(district[:2], 0) + 1 + assert by_state == state_population + expected = expected_district_counts(PLANS[plan]) + assert count_by_state == {state: expected[state] for state in state_population} + assert all(people > 0 for people in districts.values()) + + summary = summarize_cd_plan_registry(registry) + assert summary["population"] == int(population.sum()) + assert summary["blocks"] == len(blocks) + + +@SETTINGS +@given(geography=geographies(), seed=st.integers(0, 2**32 - 1)) +def test_artifact_bytes_do_not_depend_on_input_order(geography, seed): + blocks, population, plans = geography + rng = np.random.default_rng(seed) + order = rng.permutation(len(blocks)) + shuffled_plans = {} + for plan, mapping in plans.items(): + items = list(mapping.items()) + shuffled_plans[plan] = dict(items[i] for i in rng.permutation(len(items))) + + original = _npz_bytes(_assemble(blocks, population, plans)) + shuffled = _npz_bytes(_assemble(blocks[order], population[order], shuffled_plans)) + assert hashlib.sha256(original).hexdigest() == hashlib.sha256(shuffled).hexdigest() + # And building twice from the same inputs gives the same bytes. + assert _npz_bytes(_assemble(blocks, population, plans)) == original + + +@SETTINGS +@given(geography=geographies(), data=st.data()) +def test_a_block_without_a_district_is_refused(geography, data): + blocks, population, plans = geography + plan = data.draw(st.sampled_from(sorted(plans))) + dropped = data.draw(st.sampled_from(blocks.tolist())) + broken = {**plans, plan: {b: d for b, d in plans[plan].items() if b != dropped}} + with pytest.raises(ValueError, match="unassigned"): + _assemble(blocks, population, broken) + + +@SETTINGS +@given(geography=geographies(), data=st.data()) +def test_a_district_in_another_state_is_refused(geography, data): + blocks, population, plans = geography + plan = data.draw(st.sampled_from(sorted(plans))) + block = data.draw(st.sampled_from(blocks.tolist())) + other_state = data.draw( + st.sampled_from([s for s in SMALL_STATES if int(s) != block // 10**13]) + ) + broken = {**plans, plan: {**plans[plan], block: int(other_state) * 100}} + with pytest.raises(ValueError, match="another state's district"): + _assemble(blocks, population, broken) + + +@SETTINGS +@given(geography=geographies(), data=st.data()) +def test_a_merged_or_invented_district_is_refused(geography, data): + blocks, population, plans = geography + plan = data.draw(st.sampled_from(sorted(plans))) + mapping = plans[plan] + districts = sorted(set(mapping.values())) + multi = [ + d + for d in districts + if sum(1 for other in districts if other // 100 == d // 100) > 1 + ] + if multi and data.draw(st.booleans()): + # Merge one district into a neighbour: one district too few. + gone = data.draw(st.sampled_from(multi)) + keep = next(d for d in districts if d // 100 == gone // 100 and d != gone) + broken_plan = {b: (keep if d == gone else d) for b, d in mapping.items()} + else: + # Move one block into a district number the state was not given. + block = data.draw(st.sampled_from(blocks.tolist())) + broken_plan = {**mapping, block: (block // 10**13) * 100 + 99} + with pytest.raises(ValueError, match="apportionment"): + _assemble(blocks, population, {**plans, plan: broken_plan}) + + +@SETTINGS +@given(geography=geographies(), data=st.data()) +def test_ladder_differential_refuses_any_single_change( + tmp_path_factory, geography, data +): + blocks, population, plans = geography + registry = _load( + tmp_path_factory.mktemp("d"), _assemble(blocks, population, plans), "d" + ) + ladder_cd = np.asarray( + [plans[PRIMARY_CD_PLAN][b] for b in blocks.tolist()], dtype=np.int64 + ) + receipt = check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=blocks, + population=population, + congressional_district_geoid=ladder_cd, + ) + assert receipt["agreement"] == "exact" + assert receipt["population"] == int(population.sum()) + + index = data.draw(st.integers(0, len(blocks) - 1)) + change = data.draw(st.sampled_from(["district", "population", "drop"])) + changed_blocks, changed_population, changed_cd = ( + blocks, + population.copy(), + ladder_cd.copy(), + ) + if change == "district": + changed_cd[index] += 1 # any other district code + elif change == "population": + changed_population[index] += 1 + else: + keep = np.arange(len(blocks)) != index + changed_blocks, changed_population, changed_cd = ( + blocks[keep], + population[keep], + ladder_cd[keep], + ) + with pytest.raises(ValueError): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=changed_blocks, + population=changed_population, + congressional_district_geoid=changed_cd, + ) + + +@SETTINGS +@given(geography=geographies(), data=st.data()) +def test_state_substitution_changes_exactly_that_state( + tmp_path_factory, geography, data +): + blocks, population, plans = geography + state = data.draw( + st.sampled_from(sorted({f"{b // 10**13:02d}" for b in blocks.tolist()})) + ) + base = cd_block_assignment(plans["120th_congress"], label="120") + donor = cd_block_assignment(plans["119th_congress"], label="119").for_state(state) + merged = replace_state_assignments(base, donor, state_fips=state) + merged_map = dict( + zip(merged.block_geoid.tolist(), merged.district_geoid.tolist(), strict=True) + ) + for block, district in plans["120th_congress"].items(): + in_state = f"{block // 10**13:02d}" == state + expected = plans["119th_congress"][block] if in_state else district + assert merged_map[block] == expected + + # Both plans use the 2020 apportionment, so the result is a valid plan. + payload = _assemble( + blocks, + population, + {"120th_congress": merged_map, "119th_congress": plans["119th_congress"]}, + ) + registry = _load(tmp_path_factory.mktemp("s"), payload, "s") + summary = summarize_cd_plan_registry(registry) + differing = summary["plans"]["120th_congress"]["states_differing_from_primary"] + assert state not in differing + assert payload[f"{CD_PLAN_ARRAY_PREFIX}120th_congress"].dtype == np.int16 + + +@st.composite +def relationships(draw): + """A 2010 plan in one state and a relationship file to 2020 blocks.""" + + state = 37 + blocks_2010 = draw( + st.lists(st.integers(0, 10**6), min_size=1, max_size=30, unique=True) + ) + blocks_2010 = [state * 10**13 + b for b in blocks_2010] + plan = {b: state * 100 + draw(st.integers(1, 4)) for b in blocks_2010} + # Some 2010 counterparts sit outside the plan (another state's blocks). + outside = [ + 45 * 10**13 + b + for b in draw(st.lists(st.integers(0, 10**6), max_size=5, unique=True)) + ] + blocks_2020 = [ + state * 10**13 + b + for b in draw( + st.lists(st.integers(0, 10**6), min_size=1, max_size=30, unique=True) + ) + ] + rows = [] + for block_2020 in blocks_2020: + for block_2010 in draw( + st.lists( + st.sampled_from(blocks_2010 + outside), + min_size=1, + max_size=4, + unique=True, + ) + ): + rows.append( + ( + block_2010, + block_2020, + draw(st.integers(0, 50)), + draw(st.integers(0, 5)), + ) + ) + return plan, rows + + +def _relationship(rows): + return BlockRelationship( + block_2010=np.asarray([r[0] for r in rows], dtype=np.int64), + block_2020=np.asarray([r[1] for r in rows], dtype=np.int64), + area_land=np.asarray([r[2] for r in rows], dtype=np.int64), + area_water=np.asarray([r[3] for r in rows], dtype=np.int64), + ) + + +@SETTINGS +@given(case=relationships(), seed=st.integers(0, 2**32 - 1)) +def test_crosswalk_picks_the_largest_overlap_whatever_the_row_order(case, seed): + plan, rows = case + plan_2010 = cd_block_assignment(plan, label="2010 plan") + carried, diagnostics = crosswalk_plan_to_2020_blocks( + plan_2010, _relationship(rows), state_fips="37" + ) + + # Reference implementation: sum (land, water) per candidate district. + expected: dict[int, int] = {} + candidates: dict[int, dict[int, list[int]]] = {} + for block_2010, block_2020, land, water in rows: + if block_2010 not in plan: + continue + area = candidates.setdefault(block_2020, {}).setdefault( + plan[block_2010], [0, 0] + ) + area[0] += land + area[1] += water + for block_2020, by_district in candidates.items(): + expected[block_2020] = min( + by_district, key=lambda d: (-by_district[d][0], -by_district[d][1], d) + ) + got = dict( + zip(carried.block_geoid.tolist(), carried.district_geoid.tolist(), strict=True) + ) + assert got == expected + all_2020 = {r[1] for r in rows} + assert set(diagnostics["unassigned"].tolist()) == all_2020 - set(expected) + assert set(diagnostics["straddling"].tolist()) == { + b for b, by_district in candidates.items() if len(by_district) > 1 + } + + shuffled = [rows[i] for i in np.random.default_rng(seed).permutation(len(rows))] + again, _ = crosswalk_plan_to_2020_blocks( + plan_2010, _relationship(shuffled), state_fips="37" + ) + assert np.array_equal(again.block_geoid, carried.block_geoid) + assert np.array_equal(again.district_geoid, carried.district_geoid) + + +@SETTINGS +@given(case=relationships()) +def test_fill_completes_every_block_with_a_district_already_nearby(case): + plan, rows = case + carried, diagnostics = crosswalk_plan_to_2020_blocks( + cd_block_assignment(plan, label="2010 plan"), + _relationship(rows), + state_fips="37", + ) + if len(carried) == 0: + return # nothing to borrow from; the build refuses this case + filled, fills = fill_unassigned_blocks( + carried, + diagnostics["unassigned"], + block_geoid=carried.block_geoid, + population=np.ones(len(carried), dtype=np.int64), + ) + assert set(filled.block_geoid.tolist()) == set(diagnostics["blocks_2020"].tolist()) + assert len(fills) == len(diagnostics["unassigned"]) + present = set(carried.district_geoid.tolist()) + for fill in fills: + assert 3700 + int(fill["district"]) in present + # Blocks the crosswalk assigned keep their district. + kept = dict( + zip(filled.block_geoid.tolist(), filled.district_geoid.tolist(), strict=True) + ) + for block, district in zip( + carried.block_geoid.tolist(), carried.district_geoid.tolist(), strict=True + ): + assert kept[block] == district diff --git a/tools/build_us_cd_plan_registry_artifact.py b/tools/build_us_cd_plan_registry_artifact.py new file mode 100644 index 000000000..c54a854ee --- /dev/null +++ b/tools/build_us_cd_plan_registry_artifact.py @@ -0,0 +1,776 @@ +"""Build the US block -> congressional-district plan registry from Census sources. + +Downloads (with a local cache) the 2020 P.L. 94-171 geographic headers (the +populated-block universe and its populations), the 2020 Block Assignment +Files' ``CD`` layer (117th plan), and the Census 118th, 119th and 120th +Congressional District block equivalency files. It parses each plan onto +2020 blocks, applies any registered state substitutions, and writes one +national NPZ whose metadata records every source file's URL and SHA-256. + +Before writing, the build cross-checks its sources against each other: + +- the apportionment constant in ``cd_plan_registry`` against the Census + apportionment table; +- each Census BEF's state files against that state's rows in its national + file; +- the states whose blocks change between consecutive plans against the + states Census ships a state file for (its list of redrawn states); +- each Census block-split note's tabulated district against the registry. + +The artifact is then loaded back through ``load_us_cd_plan_registry`` (which +re-checks every invariant) and, with ``--block-ladder``, compared block by +block with the ladder's own 119th-Congress districts and populations. + +The method, sources and invariants are documented in +``packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md``. + +Example: + uv run python tools/build_us_cd_plan_registry_artifact.py \\ + --out build/us/us_cd_plan_registry_2020.npz \\ + --block-ladder build/us/us_block_ladder_2020.npz \\ + --provenance-json packages/microcosm-build/src/microcosm/build/\\ +us_runtime/us_cd_plan_registry.provenance.json + + # Smoke run over a few states (no ladder check, not publishable): + uv run python tools/build_us_cd_plan_registry_artifact.py \\ + --out /tmp/cd_plans_smoke.npz --states 10,29,37 +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import json +import re +import sys +import urllib.request +import zipfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import numpy as np + +from microcosm.build.us_runtime.block_ladder_sources import ( + US_STATES, + parse_pl_geo_blocks, +) +from microcosm.build.us_runtime.cd_plan_registry import ( + PRIMARY_CD_PLAN, + US_HOUSE_APPORTIONMENT, + US_HOUSE_APPORTIONMENT_SOURCE, + CdBlockAssignment, + assemble_us_cd_plan_registry, + assignment_agreement, + check_cd_plan_registry_against_block_ladder, + concat_cd_block_assignments, + crosswalk_plan_to_2020_blocks, + fill_unassigned_blocks, + load_us_cd_plan_registry, + parse_block_relationship_2010_2020, + parse_cd_block_assignment, + replace_state_assignments, + summarize_cd_plan_registry, +) + +ARTIFACT_NAME = "us_cd_plan_registry_2020.npz" + +PL94171_URL_TEMPLATE = ( + "https://www2.census.gov/programs-surveys/decennial/2020/data/" + "01-Redistricting_File--PL_94-171/{dirname}/{usps_lower}2020.pl.zip" +) +BAF2020_URL_TEMPLATE = ( + "https://www2.census.gov/geo/docs/maps-data/data/baf2020/" + "BlockAssign_ST{fips}_{usps}.zip" +) +BAF2020_CD_MEMBER_TEMPLATE = "BlockAssign_ST{fips}_{usps}_CD.txt" +_MAPPING_FILES = "https://www2.census.gov/programs-surveys/decennial/rdo/mapping-files" + +#: One entry per Census national BEF: the ZIP, its national member, the +#: pattern naming its state files (group 1 = state FIPS) and its split note. +CENSUS_BEFS: dict[str, dict[str, str]] = { + "118th_congress": { + "url": f"{_MAPPING_FILES}/2023/118-congressional-district-bef/cd118.zip", + "member": "National_CD118.txt", + "state_member_pattern": r"^(\d{2})_[A-Z]{2}_CD118\.txt$", + "block_split_note": ( + f"{_MAPPING_FILES}/2023/118-congressional-district-bef/CD118_BlockSplits.pdf" + ), + "published": "2022-12-16", + "source": "Census 118th Congressional District block equivalency file", + }, + "119th_congress": { + "url": f"{_MAPPING_FILES}/2025/119-congressional-district-befs/cd119.zip", + "member": "NationalCD119.txt", + "state_member_pattern": r"^(\d{2})_[A-Z]{2}_CD119\.txt$", + "block_split_note": ( + f"{_MAPPING_FILES}/2025/119-congressional-district-befs/CD119_BlockSplits.pdf" + ), + "published": "2024-07-29", + "source": "Census 119th Congressional District block equivalency file", + }, + "120th_congress": { + "url": f"{_MAPPING_FILES}/2027/120-congressional-district-befs/cd120.zip", + "member": "NationalCD120.txt", + "state_member_pattern": r"^CD120_(\d{2})\.txt$", + "block_split_note": ( + f"{_MAPPING_FILES}/2027/120-congressional-district-befs/CD120_BlockSplits.pdf" + ), + "published": "2026-08-31", + "source": "Census 120th Congressional District block equivalency file", + }, +} + +#: Blocks the Census split notes list as divided by a district line. For +#: tabulation Census assigns each to one district "specified to the U.S. +#: Census Bureau by the state"; the build checks the registry agrees. The +#: 118th, 119th and 120th notes each list only this block. +CENSUS_BLOCK_SPLITS: dict[str, dict[str, dict[str, Any]]] = { + plan: { + "080010096072000": { + "districts_within_block": ["07", "08"], + "tabulated_with": "08", + } + } + for plan in CENSUS_BEFS +} + +PLAN_APPORTIONMENT = { + "117th_congress": "2010_census", + "118th_congress": "2020_census", + "119th_congress": "2020_census", + "120th_congress": "2020_census", +} + +PLAN_NOTES = { + "117th_congress": ( + "Districts of the 117th Congress (2021-2023), the geography of the IRS " + "SOI congressional-district tables for tax years 2020-2022. Every state " + "but North Carolina comes from the 2020 Block Assignment Files' CD " + "layer, which carries the 116th-Congress plans on 2020 blocks; Census " + "reports North Carolina as the only state whose districts changed " + "between the 116th and 117th Congress (see known_deviations)." + ), + "118th_congress": ( + "Districts of the 118th Congress (2023-2025), the first plans drawn " + "on 2020 census geography." + ), + "119th_congress": ( + "Districts of the 119th Congress (2025-2027); the block ladder's primary plan." + ), + "120th_congress": ( + "Districts for the November 3, 2026 election (120th Congress, " + "2027-2029), including the 2025-26 mid-decade redraws: Census's 120th " + "Congressional District block equivalency file, except any state whose " + "map in force differs from Census's rows (see known_deviations)." + ), +} + +#: North Carolina is the one state whose 117th-Congress districts differ from +#: its 116th (Census, "Congressional Districts of the 117th Congress"): the +#: 2020 election used the 2019 remedial plan (HB 1029, S.L. 2019-249), which +#: Census never tabulated on 2020 blocks. The build carries the General +#: Assembly's block file for that plan (2010 blocks) onto 2020 blocks through +#: the Census 2010-2020 block relationship file, checks the method by running +#: it on the 2016 plan against Census's own 2020-block version of that plan, +#: and checks the result against the General Assembly's official 2020-census +#: district populations for the 2019 plan. +NC_FIPS = "37" +NC_117TH_SOURCES = { + "ncga_2019_plan": { + "url": ( + "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2019/" + "HB1029%203rd%20Edition%20-%20Blockfile.zip" + ), + "member": "C-Goodwin-A-1-TC.csv", + }, + "ncga_2016_plan": { + "url": "https://www.ncleg.gov/Files/GIS/Plans_Main/Congress_2016/baf.zip", + "member": "2016 Contingent Congressional Plan - Corrected.csv", + }, + "census_block_relationship_2010_2020_nc": { + "url": ( + "https://www2.census.gov/geo/docs/maps-data/data/rel2020/t10t20/" + "TAB2010_TAB2020_ST37.zip" + ), + "member": "tab2010_tab2020_st37_nc.txt", + }, + "ncga_2020_district_populations_2019_plan": { + "url": ( + "https://www.ncleg.gov/Files/GIS/Maps_Reports/Decennial_ReCalc/2020/" + "DeviationReports/2020_Deviation_Rpt_Cong.xlsx" + ), + }, +} +#: The method run on the 2016 plan must reproduce Census's 2020-block version +#: of that plan (the BAF CD layer) for at least this share of population. +NC_117TH_MIN_METHOD_AGREEMENT = 0.999 +#: Each 2019-plan district's population must be within this share of the +#: General Assembly's official 2020 figure. +NC_117TH_MAX_DISTRICT_GAP = 0.005 + +#: Registered substitutions: a state's blocks in ``plan`` taken from another +#: registered plan instead of the plan's own source, with the reason and the +#: primary evidence. Each one is recorded in the plan's ``known_deviations``. +STATE_SUBSTITUTIONS: dict[str, dict[str, dict[str, Any]]] = { + "120th_congress": { + "29": { + "from_plan": "119th_congress", + "note": ( + "Census's 120th rows for Missouri encode the 2025 HB 1 map, and " + "Census's own 120th landing page warns they may not be the " + "districts in place for November 2026. The Missouri Supreme Court " + "held on 2026-09-03 (von Glahn v. Hoskins, SC101805) that HB 1 " + "cannot take effect before a referendum on the November 2026 " + "ballot, and the U.S. Supreme Court held on 2026-09-25 (People " + "Not Politicians v. Onder, 26A388, per curiam) that 'the 2022 " + "map\u2014not the 2025 map\u2014must be used in the 2026 congressional " + "election.' Missouri's rows here are its 2022 map, taken from the " + "119th file (the build checks they are unchanged from the 118th)." + ), + "evidence": [ + "https://www.census.gov/geographies/mapping-files/2027/dec/rdo/" + "120-congressional-district-bef.html", + "https://www.supremecourt.gov/opinions/25pdf/26a388_q86b.pdf", + "https://statecourtreport.org/case-tracker/von-glahn-v-hoskins", + ], + "as_of": "2026-09-27", + } + } +} + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build the US block -> congressional-district plan registry." + ) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument( + "--cache-dir", + type=Path, + default=Path.home() / ".cache" / "populace-us-geography", + help="Download cache; re-runs reuse cached files.", + ) + parser.add_argument( + "--states", + help="Comma-separated state FIPS subset (smoke runs; not publishable).", + ) + parser.add_argument( + "--block-ladder", + type=Path, + help=( + "Block ladder NPZ to compare against: same blocks, same " + "populations, identical 119th-Congress districts." + ), + ) + parser.add_argument( + "--provenance-json", + type=Path, + help="Where to write the build receipt. Defaults beside --out.", + ) + return parser.parse_args(argv) + + +def _log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _download(url: str, cache_dir: Path) -> Path: + cache_dir.mkdir(parents=True, exist_ok=True) + destination = cache_dir / url.rsplit("/", 1)[-1] + if destination.exists() and destination.stat().st_size > 0: + return destination + _log(f" downloading {url}") + request = urllib.request.Request( + url, headers={"User-Agent": "microcosm-build (cd plan registry)"} + ) + with urllib.request.urlopen(request) as response: + payload = response.read() + if not payload: + raise RuntimeError(f"Empty download from {url}") + partial = destination.with_suffix(destination.suffix + ".partial") + partial.write_bytes(payload) + partial.replace(destination) + return destination + + +@contextmanager +def _zip_member_stream( + archive_path: Path, member: str, *, encoding: str = "latin-1" +) -> Iterator[io.TextIOWrapper]: + with zipfile.ZipFile(archive_path) as archive, archive.open(member) as raw: + yield io.TextIOWrapper(raw, encoding=encoding) + + +def _source_entry(url: str, path: Path, member: str | None = None) -> dict[str, str]: + entry = {"url": url, "sha256": _sha256(path)} + if member is not None: + entry["member"] = member + return entry + + +def _check_apportionment(cache_dir: Path) -> None: + """The packaged seat counts must equal the Census apportionment table.""" + + path = _download(US_HOUSE_APPORTIONMENT_SOURCE["url"], cache_dir) + digest = _sha256(path) + if digest != US_HOUSE_APPORTIONMENT_SOURCE["sha256"]: + raise SystemExit( + f"Census apportionment table changed: sha256 {digest}, pinned " + f"{US_HOUSE_APPORTIONMENT_SOURCE['sha256']}. Re-verify the constant." + ) + name_to_fips = {dirname.replace("_", " "): fips for fips, _, dirname in US_STATES} + table: dict[str, dict[str, int]] = {} + with path.open(encoding="utf-8-sig") as stream: + for row in csv.DictReader(stream): + year = row["Year"] + if row["Geography Type"] != "State" or row["Name"] not in name_to_fips: + continue + seats = row["Number of Representatives"].strip() + table.setdefault(f"{year}_census", {})[name_to_fips[row["Name"]]] = ( + int(seats) if seats else 0 + ) + for census, seats in US_HOUSE_APPORTIONMENT.items(): + if table.get(census) != dict(seats): + raise SystemExit( + f"US_HOUSE_APPORTIONMENT[{census!r}] disagrees with the Census table." + ) + _log(" apportionment constant matches the Census table") + + +def _universe( + states: list[tuple[str, str, str]], cache_dir: Path, sources: dict[str, Any] +) -> tuple[np.ndarray, np.ndarray]: + blocks: list[np.ndarray] = [] + populations: list[np.ndarray] = [] + for fips, usps, dirname in states: + url = PL94171_URL_TEMPLATE.format(dirname=dirname, usps_lower=usps.lower()) + path = _download(url, cache_dir) + member = f"{usps.lower()}geo2020.pl" + sources[f"pl94171_{usps.lower()}"] = _source_entry(url, path, member) + with _zip_member_stream(path, member) as stream: + state_blocks = parse_pl_geo_blocks(stream, state_fips=fips) + keys = np.fromiter(state_blocks.keys(), dtype=np.int64, count=len(state_blocks)) + values = np.fromiter( + state_blocks.values(), dtype=np.int64, count=len(state_blocks) + ) + blocks.append(keys) + populations.append(values) + _log(f" {usps}: {len(keys):,} populated blocks, {int(values.sum()):,} people") + return np.concatenate(blocks), np.concatenate(populations) + + +def _plan_117th( + states: list[tuple[str, str, str]], cache_dir: Path +) -> tuple[CdBlockAssignment, dict[str, Any]]: + parts: list[CdBlockAssignment] = [] + source_files: dict[str, Any] = {} + for fips, usps, _ in states: + url = BAF2020_URL_TEMPLATE.format(fips=fips, usps=usps) + path = _download(url, cache_dir) + member = BAF2020_CD_MEMBER_TEMPLATE.format(fips=fips, usps=usps) + source_files[f"baf2020_{usps.lower()}"] = _source_entry(url, path, member) + with _zip_member_stream(path, member) as stream: + parts.append(parse_cd_block_assignment(stream, label=member)) + assignment = concat_cd_block_assignments(parts, label="2020 BAF CD layer") + spec = { + "source": "Census 2020 Block Assignment Files, CD layer", + "url": BAF2020_URL_TEMPLATE, + "source_files": source_files, + } + return assignment, spec + + +def _ncga_2020_district_populations(path: Path) -> dict[int, int]: + """District -> 2020 population from the General Assembly's deviation report.""" + + import openpyxl # dev dependency, as for the block ladder's CBSA workbook + + workbook = openpyxl.load_workbook(path, read_only=True) + populations: dict[int, int] = {} + for row in workbook.worksheets[0].iter_rows(values_only=True): + if row and isinstance(row[0], int) and isinstance(row[2], int): + populations[row[0]] = row[2] + if sorted(populations) != list(range(1, 14)): + raise SystemExit( + f"NCGA 2020 deviation report: expected districts 1-13, got {sorted(populations)}." + ) + return populations + + +def _nc_117th( + census_116th: CdBlockAssignment, + block_geoid: np.ndarray, + population: np.ndarray, + cache_dir: Path, +) -> tuple[CdBlockAssignment, dict[str, Any], dict[str, Any]]: + """North Carolina's 2019 plan on 2020 blocks, its deviation record and sources.""" + + paths = { + key: _download(spec["url"], cache_dir) for key, spec in NC_117TH_SOURCES.items() + } + source_files = { + key: _source_entry(spec["url"], paths[key], spec.get("member")) + for key, spec in NC_117TH_SOURCES.items() + } + in_state = block_geoid // 10**13 == int(NC_FIPS) + nc_blocks, nc_population = block_geoid[in_state], population[in_state] + + relationship_spec = NC_117TH_SOURCES["census_block_relationship_2010_2020_nc"] + with _zip_member_stream( + paths["census_block_relationship_2010_2020_nc"], + relationship_spec["member"], + encoding="utf-8-sig", + ) as stream: + relationship = parse_block_relationship_2010_2020( + stream, label=relationship_spec["member"] + ) + + def carry(key: str) -> tuple[CdBlockAssignment, dict[str, Any]]: + spec = NC_117TH_SOURCES[key] + with _zip_member_stream( + paths[key], spec["member"], encoding="utf-8-sig" + ) as stream: + plan_2010 = parse_cd_block_assignment(stream, label=spec["member"]) + carried, diagnostics = crosswalk_plan_to_2020_blocks( + plan_2010, relationship, state_fips=NC_FIPS + ) + filled, fills = fill_unassigned_blocks( + carried, + diagnostics["unassigned"], + block_geoid=nc_blocks, + population=nc_population, + ) + straddling = np.isin(nc_blocks, diagnostics["straddling"]) + return filled, { + "straddling_blocks": int(len(diagnostics["straddling"])), + "straddling_populated_blocks": int(straddling.sum()), + "straddling_population": int(nc_population[straddling].sum()), + "filled_blocks": fills, + } + + plan_2019, diagnostics_2019 = carry("ncga_2019_plan") + plan_2016, _ = carry("ncga_2016_plan") + census_nc = census_116th.for_state(NC_FIPS) + + method_check = assignment_agreement( + plan_2016, census_nc, block_geoid=nc_blocks, population=nc_population + ) + if method_check["population_share_agreeing"] < NC_117TH_MIN_METHOD_AGREEMENT: + raise SystemExit( + "NC 117th: the crosswalk method reproduces Census's 2020-block 2016 " + f"plan for only {method_check['population_share_agreeing']:.5f} of " + f"the population (minimum {NC_117TH_MIN_METHOD_AGREEMENT})." + ) + + official = _ncga_2020_district_populations( + paths["ncga_2020_district_populations_2019_plan"] + ) + index = np.searchsorted(plan_2019.block_geoid, nc_blocks) + districts = plan_2019.district_geoid[index] % 100 + if not np.array_equal(plan_2019.block_geoid[index], nc_blocks): + raise SystemExit("NC 117th: the 2019 plan leaves populated blocks unassigned.") + ours = { + int(d): int(nc_population[districts == d].sum()) for d in np.unique(districts) + } + gaps = {f"{d:02d}": ours.get(d, 0) - official[d] for d in sorted(official)} + worst = max(abs(gap) / official[int(d)] for d, gap in gaps.items()) + if worst > NC_117TH_MAX_DISTRICT_GAP: + raise SystemExit( + f"NC 117th: district populations differ from the General Assembly's " + f"2020 figures by up to {worst:.4%}: {gaps}." + ) + + deviation = { + "note": ( + "North Carolina is the one state whose 117th districts differ from " + "its 116th. Its 117th districts are the 2019 remedial plan (HB 1029 " + "3rd edition, S.L. 2019-249, 'C-Goodwin-A-1-TC'), which Census never " + "tabulated on 2020 blocks, so this state's rows are not from the " + "2020 BAF CD layer (the 2016 plan). The General Assembly's block " + "file for the 2019 plan (2010 blocks) is carried onto 2020 blocks " + "through the Census 2010-2020 tabulation block relationship file." + ), + "method": ( + "Each 2020 block takes the 2019-plan district whose 2010 blocks cover " + "the most of its land area (then water area, then the lower district " + "code). A 2020 block with no North Carolina 2010 counterpart takes the " + "district holding most of its block group's population (else its " + "tract's, else its county's)." + ), + **diagnostics_2019, + "method_check": { + "description": ( + "The same method run on the 2016 plan, compared with Census's own " + "2020-block version of that plan (the 2020 BAF CD layer), over " + "populated blocks." + ), + "minimum_population_share": NC_117TH_MIN_METHOD_AGREEMENT, + **method_check, + }, + "official_district_population_check": { + "description": ( + "District populations of the carried 2019 plan minus the General " + "Assembly's official 2020-census populations for that plan." + ), + "maximum_share": NC_117TH_MAX_DISTRICT_GAP, + "differences": gaps, + "largest_share": round(worst, 6), + }, + "agreement_with_the_116th_plan": assignment_agreement( + plan_2019, census_nc, block_geoid=nc_blocks, population=nc_population + ), + "source_files": source_files, + } + return plan_2019, deviation, source_files + + +def _plan_from_census_bef( + plan: str, + cache_dir: Path, + selected: set[str], +) -> tuple[CdBlockAssignment, dict[str, Any], set[str]]: + """Parse a Census national BEF and check its state files against it. + + Returns the assignment restricted to ``selected`` states, the plan's + source spec, and the states Census shipped a state file for. + """ + + bef = CENSUS_BEFS[plan] + path = _download(bef["url"], cache_dir) + split_note = _download(bef["block_split_note"], cache_dir) + with _zip_member_stream(path, bef["member"]) as stream: + national = parse_cd_block_assignment(stream, label=bef["member"]) + national = concat_cd_block_assignments( + [national.for_state(state) for state in sorted(selected & national.states())], + label=bef["member"], + ) + pattern = re.compile(bef["state_member_pattern"]) + with zipfile.ZipFile(path) as archive: + state_members = { + match.group(1): name + for name in archive.namelist() + if (match := pattern.match(name)) + } + for state in sorted(set(state_members) & selected): + with _zip_member_stream(path, state_members[state]) as stream: + state_file = parse_cd_block_assignment(stream, label=state_members[state]) + national_rows = national.for_state(state) + if not ( + np.array_equal(state_file.block_geoid, national_rows.block_geoid) + and np.array_equal(state_file.district_geoid, national_rows.district_geoid) + ): + raise SystemExit( + f"{plan}: state file {state_members[state]} disagrees with " + f"{bef['member']} for state {state}." + ) + spec = { + "source": f"{bef['source']} ({bef['member']})", + "url": bef["url"], + "published": bef["published"], + "source_files": { + plan: _source_entry(bef["url"], path, bef["member"]), + f"{plan}_block_split_note": _source_entry( + bef["block_split_note"], split_note + ), + }, + "census_state_files": sorted(state_members), + "block_splits": CENSUS_BLOCK_SPLITS[plan], + } + return national, spec, set(state_members) + + +def _changed_states(before: CdBlockAssignment, after: CdBlockAssignment) -> set[str]: + """States with any block whose district differs between two plans.""" + + common, before_index, after_index = np.intersect1d( + before.block_geoid, after.block_geoid, assume_unique=True, return_indices=True + ) + differs = before.district_geoid[before_index] != after.district_geoid[after_index] + return {f"{state:02d}" for state in np.unique(common[differs] // 10**13).tolist()} + + +def main(argv: list[str] | None = None) -> None: + args = _parse_args(argv) + selected_fips = ( + {value.strip() for value in args.states.split(",")} if args.states else None + ) + if selected_fips is not None: + unknown = selected_fips - {fips for fips, _, _ in US_STATES} + if unknown: + raise SystemExit(f"Unknown state FIPS in --states: {sorted(unknown)}") + states = [s for s in US_STATES if selected_fips is None or s[0] in selected_fips] + selected = {fips for fips, _, _ in states} + _log(f"Building the US CD plan registry for {len(states)} state(s)") + + _check_apportionment(args.cache_dir) + + population_sources: dict[str, Any] = {} + block_geoid, population = _universe(states, args.cache_dir, population_sources) + _log(f" universe: {len(block_geoid):,} blocks, {int(population.sum()):,} people") + + assignments: dict[str, CdBlockAssignment] = {} + specs: dict[str, dict[str, Any]] = {} + assignments["117th_congress"], specs["117th_congress"] = _plan_117th( + states, args.cache_dir + ) + if NC_FIPS in selected: + _log(" carrying North Carolina's 2019 plan onto 2020 blocks") + nc_plan, nc_deviation, nc_sources = _nc_117th( + assignments["117th_congress"], block_geoid, population, args.cache_dir + ) + assignments["117th_congress"] = replace_state_assignments( + assignments["117th_congress"], nc_plan, state_fips=NC_FIPS + ) + specs["117th_congress"].setdefault("known_deviations", {})[NC_FIPS] = ( + nc_deviation + ) + specs["117th_congress"]["source_files"].update(nc_sources) + _log( + " NC 117th: method check " + f"{nc_deviation['method_check']['population_share_agreeing']:.5f}, " + "largest district gap " + f"{nc_deviation['official_district_population_check']['largest_share']:.4%}" + ) + census_state_files: dict[str, set[str]] = {} + for plan in CENSUS_BEFS: + _log(f" parsing {plan}") + assignments[plan], specs[plan], census_state_files[plan] = ( + _plan_from_census_bef(plan, args.cache_dir, selected) + ) + + # Census ships a state file exactly for the states it lists as redrawn; + # the blocks must agree with that list. + for previous, plan in ( + ("118th_congress", "119th_congress"), + ("119th_congress", "120th_congress"), + ): + changed = _changed_states(assignments[previous], assignments[plan]) + expected = census_state_files[plan] & selected + if changed != expected: + raise SystemExit( + f"{plan}: states changed from {previous} are {sorted(changed)}, " + f"but Census ships state files for {sorted(expected)}." + ) + + for plan, substitutions in STATE_SUBSTITUTIONS.items(): + deviations = specs[plan].setdefault("known_deviations", {}) + for state, substitution in sorted(substitutions.items()): + if state not in selected: + continue + donor = assignments[substitution["from_plan"]].for_state(state) + assignments[plan] = replace_state_assignments( + assignments[plan], donor, state_fips=state + ) + deviations[state] = substitution + _log(f" {plan}: state {state} taken from {substitution['from_plan']}") + + plan_sources = { + plan: { + "vintage": plan, + "apportionment": PLAN_APPORTIONMENT[plan], + "notes": PLAN_NOTES[plan], + **specs[plan], + } + for plan in sorted(assignments) + } + metadata = { + "universe": ( + "2020 tabulation blocks with P.L. 94-171 POP100 > 0 in the 50 states and DC" + ), + "states": sorted(selected), + "population_source": { + "source": "2020 Census P.L. 94-171 geographic headers, POP100", + "url": PL94171_URL_TEMPLATE, + "source_files": population_sources, + }, + "apportionment_source": dict(US_HOUSE_APPORTIONMENT_SOURCE), + } + payload = assemble_us_cd_plan_registry( + block_geoid=block_geoid, + population=population, + plan_assignments=assignments, + plan_sources=plan_sources, + metadata=metadata, + ) + args.out.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(args.out, **payload) + + registry = load_us_cd_plan_registry(args.out) # self-check: re-validates + for plan in CENSUS_BEFS: + for geoid, split in CENSUS_BLOCK_SPLITS[plan].items(): + block = int(geoid) + if f"{block // 10**13:02d}" not in selected: + continue + index = int(np.searchsorted(registry.block_geoid, block)) + if index >= len(registry) or registry.block_geoid[index] != block: + raise SystemExit(f"{plan}: split block {geoid} is not in the universe.") + district = int(registry.plans[plan][index]) % 100 + if f"{district:02d}" != split["tabulated_with"]: + raise SystemExit( + f"{plan}: split block {geoid} is in district {district:02d}, " + f"Census tabulates it with {split['tabulated_with']}." + ) + + summary = summarize_cd_plan_registry(registry) + ladder_check = None + if args.block_ladder is not None: + with np.load(args.block_ladder, allow_pickle=False) as ladder: + ladder_check = check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=ladder["block_geoid"], + population=ladder["population"], + congressional_district_geoid=ladder["congressional_district_geoid"], + plan=PRIMARY_CD_PLAN, + ) + ladder_check["block_ladder_sha256"] = _sha256(args.block_ladder) + _log(f" block ladder agrees exactly ({ladder_check['blocks']:,} blocks)") + + receipt = { + "artifact": ARTIFACT_NAME, + "kind": registry.metadata["kind"], + "schema_version": registry.metadata["schema_version"], + "block_vintage": registry.metadata["block_vintage"], + "output_sha256": registry.sha256, + **summary, + "block_ladder_check": ladder_check, + "plan_sources": { + plan: { + key: value + for key, value in spec.items() + if key not in {"districts", "blocks_outside_universe"} + } + for plan, spec in registry.plan_sources.items() + }, + "population_source": metadata["population_source"], + "apportionment_source": metadata["apportionment_source"], + } + receipt_path = ( + args.provenance_json + if args.provenance_json is not None + else args.out.with_suffix(".provenance.json") + ) + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") + _log(f" wrote {args.out} (sha256 {registry.sha256}) and {receipt_path}") + print( + json.dumps( + {k: receipt[k] for k in ("output_sha256", "blocks", "population")}, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() From fd9a02261f44eb0475c19f7cac179c42170031c6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 27 Sep 2026 21:03:59 -0400 Subject: [PATCH 2/2] Address review: strict array shapes, pinned split notes, narrower claims - Loader and ladder differential refuse arrays that are not 1-D and aligned (a (n, 1) population or trailing ladder values used to pass). - The builder pins the reviewed Census block-split notes by SHA-256 and refuses a changed note, since CENSUS_BLOCK_SPLITS transcribes them. - Docs and docstrings: the loader's checks are structural; national completeness comes from the build, the ladder agreement and the pin. "Exact block sums" is qualified for North Carolina's carried plan. - The 120th note says "the 2026 elections" (Louisiana's Nov 3 is a primary). District data unchanged; artifact sha256 6b16fc21... - Tests: tamper cases for shapes, misaligned ladder arrays, a lazy-parse test that fails if the stream is materialized, split-note pin tests, and a fill property over several counties, tracts and block groups checked against a reference implementation. - Classify cd_plan_registry.py as a reviewed non-registry runtime module. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/US_CD_PLAN_REGISTRY.md | 41 ++++--- .../build/us_runtime/cd_plan_registry.py | 76 +++++++++---- .../us_cd_plan_registry.provenance.json | 4 +- .../us/test_us_cd_plan_registry.py | 104 ++++++++++++++++++ .../us/test_us_cd_plan_registry_properties.py | 82 ++++++++++++++ .../engine_free/us/test_us_spine_blindness.py | 4 + tools/build_us_cd_plan_registry_artifact.py | 22 +++- 7 files changed, 294 insertions(+), 39 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md b/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md index 30f580c73..58b755b47 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/US_CD_PLAN_REGISTRY.md @@ -5,8 +5,11 @@ geography is a lookup from that block. Congressional districts are the one layer with several live versions on the same blocks, so this registry records, for every populated 2020 block, its district under each registered plan. A block draw can then derive every plan's district from the chosen block, and -SOI's 117th-Congress district tables become exact block sums with no -crosswalk. +SOI's 117th-Congress district tables become exact sums of blocks under the +registry's assignments, with no crosswalk on the consumer's side. (The +assignments themselves follow Census's whole-block tabulation; North +Carolina's 117th rows are carried from 2010 blocks, below, so they +approximate the official boundaries.) Code: `cd_plan_registry.py`. Build: `tools/build_us_cd_plan_registry_artifact.py`. Receipt (artifact SHA-256, sources, per-plan totals, every check's result): @@ -19,7 +22,7 @@ Receipt (artifact SHA-256, sources, per-plan totals, every check's result): | `117th_congress` | 2021–2023; the geography of the IRS SOI congressional-district tables for tax years 2020–2022 | Census 2020 Block Assignment Files, `CD` layer; North Carolina carried from its 2019 plan (below) | 2010 | | `118th_congress` | 2023–2025 | Census 118th Congressional District block equivalency file | 2020 | | `119th_congress` | 2025–2027; the block ladder's primary plan | Census 119th Congressional District block equivalency file | 2020 | -| `120th_congress` | the November 3, 2026 election, including the 2025–26 mid-decade redraws | Census 120th Congressional District block equivalency file (published 2026-08-31); Missouri from its 2022 map (below) | 2020 | +| `120th_congress` | the 2026 elections (the 120th Congress), including the 2025–26 mid-decade redraws | Census 120th Congressional District block equivalency file (published 2026-08-31); Missouri from its 2022 map (below) | 2020 | District geoids are `state_fips * 100 + district`. At-large states and the DC delegate are district `00`, the repo-wide convention. Each plan's metadata @@ -70,23 +73,31 @@ the court record as of 2026-09-27; a later ruling would mean a rebuild. ## Invariants -Checked when the artifact is built (`assemble_us_cd_plan_registry`) and again -on every load (`load_us_cd_plan_registry`): +Structural invariants, checked when the artifact is built +(`assemble_us_cd_plan_registry`) and again on every load +(`load_us_cd_plan_registry`): -- **Universe.** The blocks are exactly the 2020 P.L. 94-171 blocks with - positive population in the 50 states and DC (5,769,942 blocks, 331,449,281 - people), sorted and unique: the block ladder's universe. +- **Shape.** The block, population and plan arrays are one-dimensional and + aligned; the blocks are 15-digit geoids, sorted and unique, each with + positive population. - **Partition.** Every block has exactly one district under every plan, and that district lies in the block's state. A populated block that a plan's source leaves unassigned is a build error, never a gap. -- **Apportionment.** Each state's districts under a plan are exactly its House - apportionment for that plan's census: `00` for an at-large state or DC, +- **Apportionment.** For every state present, its districts under a plan are + exactly its House apportionment for that plan's census: `00` for an at-large state or DC, otherwise `01` through `n`. So every apportioned district contains a populated block. - **Conservation.** Therefore, for every plan, district populations sum exactly to state populations, and those sum to the national total. `summarize_cd_plan_registry` re-checks this. +These are structural checks, so a registry for a subset of states (a +`--states` smoke build) passes them too. That the published artifact covers +exactly the national universe, every 2020 P.L. 94-171 block with positive +population in the 50 states and DC (5,769,942 blocks, 331,449,281 people), is +established by the build, by its exact agreement with the block ladder, and +by the SHA-256 pin that `load_pinned_us_cd_plan_registry` enforces. + The build also cross-checks its sources against each other and refuses on any disagreement: @@ -97,7 +108,9 @@ disagreement: 119th to the 120th, against the states Census ships a state file for (its list of redrawn states); - the one block Census lists as split by a district line (Colorado - 080010096072000) against the district Census tabulates it with; + 080010096072000) against the district Census tabulates it with. The table + is a transcription of Census's split notes, so the build pins the reviewed + notes' SHA-256 and refuses a changed note; - with `--block-ladder`, the registry against the ladder: the same blocks, the same populations, and the same 119th district on every block. @@ -118,8 +131,10 @@ uv run python tools/build_us_cd_plan_registry_artifact.py \ The tool downloads and caches its sources, builds the NPZ (about 18 MB), loads it back through the validating loader, and writes the receipt. The output is -byte-reproducible from the pinned sources. A national build takes about two -minutes and 2.4 GB of memory. +byte-reproducible from the pinned sources: two independent national builds +produced the same SHA-256. On the machine used for this PR (an Apple-silicon +laptop, measured with `/usr/bin/time -l`), a national build took about two +minutes with a 2.4 GB peak resident set; the receipt does not record this. ## Use diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py index 4b57dd372..ac20eaf8e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/cd_plan_registry.py @@ -10,7 +10,7 @@ (see ``congressional_district_vintage``). - ``118th_congress``: the first post-2020-apportionment plan. - ``119th_congress``: the current plan and the block ladder's primary one. -- ``120th_congress``: the plans in effect for the November 2026 election, +- ``120th_congress``: the plans for the 2026 elections (120th Congress), including the 2025-26 mid-decade redraws. Each plan is one ``int16`` array aligned to the registry's sorted @@ -24,22 +24,30 @@ deviations. The build is byte-reproducible, and its SHA-256 is pinned in the packaged ``us_cd_plan_registry.provenance.json``. -Invariants, checked by :func:`assemble_us_cd_plan_registry` when the artifact -is built and again by :func:`load_us_cd_plan_registry` on every load: +Structural invariants, checked by :func:`assemble_us_cd_plan_registry` when +the artifact is built and again by :func:`load_us_cd_plan_registry` on every +load: -- The block universe is exactly the 2020 P.L. 94-171 blocks with positive - population (the block ladder's universe), sorted and unique. +- ``block_geoid``, ``population`` and every plan array are one-dimensional + and aligned; the blocks are 15-digit geoids, sorted and unique, each with + positive population. - Every block has exactly one district under every plan. A populated block a plan's source leaves unassigned is a build error, never a gap. - Every district lies in its block's state. -- Each state's districts under a plan are exactly its House apportionment - for that plan's census: ``00`` for an at-large state or the DC delegate, - otherwise ``01`` through ``n``. So every apportioned district contains at - least one populated block. +- For every state present, the districts under a plan are exactly its House + apportionment for that plan's census: ``00`` for an at-large state or the + DC delegate, otherwise ``01`` through ``n``. So every apportioned district + contains at least one populated block. Summing block population by any plan's district therefore reproduces every state's population exactly: the plans are partitions of the same blocks. +These checks are structural: a registry for a subset of states passes them. +That the published artifact covers exactly the national universe (every 2020 +P.L. 94-171 block with positive population in the 50 states and DC) is +established by the build, by its exact agreement with the block ladder, and +by pinning the artifact's SHA-256 (:func:`load_pinned_us_cd_plan_registry`). + This module is pure apart from the NPZ read. Nothing here assigns households; the block draw derives every plan's district from the chosen block. """ @@ -160,8 +168,13 @@ class CdBlockAssignment: district_geoid: np.ndarray def __post_init__(self) -> None: - if self.block_geoid.shape != self.district_geoid.shape: - raise ValueError("block_geoid and district_geoid must be aligned.") + if ( + self.block_geoid.ndim != 1 + or self.block_geoid.shape != self.district_geoid.shape + ): + raise ValueError( + "block_geoid and district_geoid must be aligned 1-D arrays." + ) if len(self.block_geoid) and not bool(np.all(np.diff(self.block_geoid) > 0)): raise ValueError("CdBlockAssignment block_geoid must be sorted and unique.") @@ -753,15 +766,19 @@ def load_us_cd_plan_registry( f"{sorted(plans_metadata)} do not match its arrays {sorted(plan_arrays)}." ) + _require_aligned_1d( + {"block_geoid": block_geoid, "population": population} + | { + f"{CD_PLAN_ARRAY_PREFIX}{plan}": values + for plan, values in plan_arrays.items() + }, + label="US CD plan registry", + ) _validate_block_geoids(block_geoid, label="block_geoid") if len(block_geoid) == 0: raise ValueError("US CD plan registry has zero blocks.") if not bool(np.all(np.diff(block_geoid) > 0)): raise ValueError("US CD plan registry block_geoid must be sorted and unique.") - if len(population) != len(block_geoid): - raise ValueError( - "US CD plan registry population is not aligned to block_geoid." - ) if (population <= 0).any(): raise ValueError( "US CD plan registry population must be positive for every block." @@ -771,10 +788,6 @@ def load_us_cd_plan_registry( for plan in sorted(plan_arrays): spec = _validated_plan_spec(plan, plans_metadata[plan]) values = plan_arrays[plan] - if len(values) != len(block_geoid): - raise ValueError( - f"CD plan {plan!r} has {len(values)} values for {len(block_geoid)} blocks." - ) counts = _validate_plan_districts( block_geoid, values, plan=plan, apportionment=spec["apportionment"] ) @@ -833,6 +846,14 @@ def check_cd_plan_registry_against_block_ladder( difference; returns a small receipt on agreement. """ + _require_aligned_1d( + { + "block_geoid": np.asarray(block_geoid), + "population": np.asarray(population), + "congressional_district_geoid": np.asarray(congressional_district_geoid), + }, + label="Block ladder", + ) ladder_blocks = _int64_array(np.asarray(block_geoid), label="ladder block_geoid") order = np.argsort(ladder_blocks, kind="stable") ladder_blocks = ladder_blocks[order] @@ -935,8 +956,10 @@ def _sorted_assignment( ) -> CdBlockAssignment: blocks = _int64_array(np.asarray(blocks), label=f"{label} block_geoid") districts = _int64_array(np.asarray(districts), label=f"{label} district_geoid") - if blocks.shape != districts.shape: - raise ValueError(f"{label}: block and district arrays are not aligned.") + if blocks.ndim != 1 or blocks.shape != districts.shape: + raise ValueError( + f"{label}: block and district arrays must be aligned 1-D arrays." + ) _validate_block_geoids(blocks, label=label) order = np.argsort(blocks, kind="stable") blocks = blocks[order] @@ -1043,6 +1066,17 @@ def _validate_plan_districts( return counts +def _require_aligned_1d(arrays: Mapping[str, np.ndarray], *, label: str) -> None: + """Refuse any array that is not 1-D or not the length of the others.""" + + shapes = {name: tuple(np.shape(values)) for name, values in arrays.items()} + not_1d = {name: shape for name, shape in shapes.items() if len(shape) != 1} + if not_1d: + raise ValueError(f"{label} arrays must be one-dimensional; got {not_1d}.") + if len(set(shapes.values())) > 1: + raise ValueError(f"{label} arrays are not aligned: {shapes}.") + + def _validate_block_geoids(blocks: np.ndarray, *, label: str) -> None: bad = (blocks < _MIN_BLOCK_GEOID) | (blocks >= _MAX_BLOCK_GEOID) if bad.any(): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json b/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json index 058c52aee..5dc2395c1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_cd_plan_registry.provenance.json @@ -14,7 +14,7 @@ "block_vintage": "2020_tabulation_blocks", "blocks": 5769942, "kind": "us_cd_plan_registry", - "output_sha256": "aa2535a9585a6eb4c4a862d6cc65702d0fa1923476c7b0cf8be6fe95f3480c88", + "output_sha256": "6b16fc2141060be312eae6ac5e5bd6b2985e45816bcfcab8024f7488be75168f", "plan_sources": { "117th_congress": { "apportionment": "2010_census", @@ -543,7 +543,7 @@ "note": "Census's 120th rows for Missouri encode the 2025 HB 1 map, and Census's own 120th landing page warns they may not be the districts in place for November 2026. The Missouri Supreme Court held on 2026-09-03 (von Glahn v. Hoskins, SC101805) that HB 1 cannot take effect before a referendum on the November 2026 ballot, and the U.S. Supreme Court held on 2026-09-25 (People Not Politicians v. Onder, 26A388, per curiam) that 'the 2022 map\u2014not the 2025 map\u2014must be used in the 2026 congressional election.' Missouri's rows here are its 2022 map, taken from the 119th file (the build checks they are unchanged from the 118th)." } }, - "notes": "Districts for the November 3, 2026 election (120th Congress, 2027-2029), including the 2025-26 mid-decade redraws: Census's 120th Congressional District block equivalency file, except any state whose map in force differs from Census's rows (see known_deviations).", + "notes": "Districts for the 2026 elections (120th Congress, 2027-2029), including the 2025-26 mid-decade redraws: Census's 120th Congressional District block equivalency file, except any state whose map in force differs from Census's rows (see known_deviations).", "published": "2026-08-31", "source": "Census 120th Congressional District block equivalency file (NationalCD120.txt)", "source_files": { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py index b4ce3d2ba..9fae4afc5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry.py @@ -8,6 +8,7 @@ from __future__ import annotations +import importlib.util import json import os import re @@ -33,8 +34,10 @@ replace_state_assignments, summarize_cd_plan_registry, ) +from test_support.paths import paths_for SHA = "0" * 64 +_TEST_PATHS = paths_for("microcosm-build") def _spec(plan: str, apportionment: str = "2020_census") -> dict: @@ -177,6 +180,19 @@ def lines(): assert bool(np.all(np.diff(parsed.block_geoid) > 0)) +def test_parser_stops_at_a_bad_record_without_reading_ahead(): + # A parser that materialized the stream first would hit the tail's + # AssertionError instead of reporting the bad record. + def lines(): + yield "GEOID,CDFP" + yield "100010401001000,00" + yield "10001040100100X,00" + raise AssertionError("the parser read past the bad record") + + with pytest.raises(ValueError, match="15 digits"): + parse_cd_block_assignment(lines(), label="stream") + + # --- apportionment ------------------------------------------------------- @@ -354,6 +370,54 @@ def test_loader_refuses_an_artifact_edited_after_the_build(tmp_path): load_us_cd_plan_registry(_write(tmp_path, missing_plan, "missing.npz")) +@pytest.mark.parametrize( + ("key", "change", "message"), + [ + ("population", lambda a: a.reshape(-1, 1), "one-dimensional"), + ( + "population", + lambda a: np.concatenate([a, a]).reshape(2, -1), + "one-dimensional", + ), + ("block_geoid", lambda a: a[::-1].reshape(-1, 1).copy(), "one-dimensional"), + ("population", lambda a: np.append(a, 5), "not aligned"), + ( + f"{CD_PLAN_ARRAY_PREFIX}120th_congress", + lambda a: a.reshape(-1, 1), + "one-dimensional", + ), + (f"{CD_PLAN_ARRAY_PREFIX}120th_congress", lambda a: a[:-1], "not aligned"), + ], +) +def test_loader_refuses_arrays_that_are_not_aligned_and_one_dimensional( + tmp_path, key, change, message +): + payload = _payload() + tampered = {**payload, key: change(payload[key])} + with pytest.raises(ValueError, match=message): + load_us_cd_plan_registry(_write(tmp_path, tampered)) + + +def test_ladder_differential_refuses_misaligned_ladder_arrays(tmp_path): + # Extra trailing values must not be silently dropped by the reordering. + registry = load_us_cd_plan_registry(_write(tmp_path, _payload())) + ladder_cd = np.asarray([PLAN_119[b] for b in BLOCKS.tolist()], dtype=np.int64) + with pytest.raises(ValueError, match="not aligned"): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS, + population=np.append(POPULATION, 999), + congressional_district_geoid=np.append(ladder_cd, 9999), + ) + with pytest.raises(ValueError, match="one-dimensional"): + check_cd_plan_registry_against_block_ladder( + registry, + block_geoid=BLOCKS.reshape(-1, 1), + population=POPULATION.reshape(-1, 1), + congressional_district_geoid=ladder_cd.reshape(-1, 1), + ) + + def test_ladder_differential_accepts_agreement_and_refuses_any_difference(tmp_path): registry = load_us_cd_plan_registry(_write(tmp_path, _payload())) order = np.asarray([6, 0, 3, 1, 5, 2, 4]) # ladder rows in any order @@ -427,6 +491,46 @@ def test_cd_block_assignment_refuses_duplicates_and_bad_geoids(): cd_block_assignment({12345: 1000}, label="short") +# --- builder --------------------------------------------------------------- + + +def _load_builder(): + path = _TEST_PATHS.repository / "tools" / "build_us_cd_plan_registry_artifact.py" + spec = importlib.util.spec_from_file_location("build_us_cd_plan_registry", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_builder_pins_every_census_split_note_it_transcribes(): + builder = _load_builder() + assert ( + set(builder.CENSUS_BEFS) + == set(builder.CENSUS_BLOCK_SPLITS) + == set(builder.CENSUS_BLOCK_SPLIT_NOTE_SHA256) + ) + receipt = packaged_us_cd_plan_registry_provenance() + for plan, digest in builder.CENSUS_BLOCK_SPLIT_NOTE_SHA256.items(): + note = receipt["plan_sources"][plan]["source_files"][f"{plan}_block_split_note"] + assert note["sha256"] == digest + assert ( + receipt["plan_sources"][plan]["block_splits"] + == (builder.CENSUS_BLOCK_SPLITS[plan]) + ) + + +def test_builder_refuses_a_split_note_that_is_not_the_reviewed_one( + tmp_path, monkeypatch +): + builder = _load_builder() + error_page = tmp_path / "CD119_BlockSplits.pdf" + error_page.write_text("503 Service Unavailable") + monkeypatch.setattr(builder, "_download", lambda url, cache_dir: error_page) + with pytest.raises(SystemExit, match="not the reviewed"): + builder._plan_from_census_bef("119th_congress", tmp_path, {"10"}) + + # --- packaged provenance --------------------------------------------------- diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py index df7cc62ce..93d9c2e90 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_cd_plan_registry_properties.py @@ -468,3 +468,85 @@ def test_fill_completes_every_block_with_a_district_already_nearby(case): carried.block_geoid.tolist(), carried.district_geoid.tolist(), strict=True ): assert kept[block] == district + + +@st.composite +def neighbourhoods(draw): + """Blocks spread over counties, tracts and block groups, some unassigned. + + Returns assigned {block: district}, unassigned blocks, and populations + (small values so ties are common). + """ + + blocks = set() + for _ in range(draw(st.integers(2, 25))): + county = draw(st.sampled_from([1, 3])) + tract = draw(st.sampled_from([100, 200, 999999])) + group = draw(st.integers(1, 3)) + block = draw(st.integers(0, 999)) + blocks.add(int(f"37{county:03d}{tract:06d}{group}{block:03d}")) + blocks = sorted(blocks) + assigned_mask = draw( + st.lists(st.booleans(), min_size=len(blocks), max_size=len(blocks)) + ) + assigned = { + block: 3700 + draw(st.integers(1, 3)) + for block, keep in zip(blocks, assigned_mask, strict=True) + if keep + } + # Every county with an unassigned block needs an assigned neighbour. + counties = {block // 10**10 for block in assigned} + unassigned = [ + block + for block in blocks + if block not in assigned and block // 10**10 in counties + ] + population = {block: draw(st.integers(0, 3)) for block in blocks} + return assigned, unassigned, population + + +@SETTINGS +@given(case=neighbourhoods()) +def test_fill_takes_the_smallest_neighbourhood_majority(case): + assigned, unassigned, population = case + if not assigned: + return + filled, fills = fill_unassigned_blocks( + cd_block_assignment(assigned, label="assigned"), + np.asarray(unassigned, dtype=np.int64), + block_geoid=np.asarray(list(population), dtype=np.int64), + population=np.asarray(list(population.values()), dtype=np.int64), + ) + + # Reference: block group, then tract, then county; most people, then most + # blocks, then the lower district; only already-assigned blocks vote. + expected = [] + for block in sorted(unassigned): + for rule, divisor in ( + ("block_group", 10**3), + ("tract", 10**4), + ("county", 10**10), + ): + voters = [b for b in assigned if b // divisor == block // divisor] + if not voters: + continue + tally = {} + for voter in voters: + people, count = tally.get(assigned[voter], (0, 0)) + tally[assigned[voter]] = (people + population[voter], count + 1) + district = min(tally, key=lambda d: (-tally[d][0], -tally[d][1], d)) + expected.append( + { + "block": f"{block:015d}", + "district": f"{district % 100:02d}", + "rule": rule, + "population": population[block], + } + ) + break + assert fills == expected + placed = dict( + zip(filled.block_geoid.tolist(), filled.district_geoid.tolist(), strict=True) + ) + for block, district in assigned.items(): + assert placed[block] == district diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 2c5a35548..692633b85 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -251,6 +251,10 @@ "block_ladder_sources.py", "capital_gain_distributions.py", "casualty_losses.py", + # Block -> congressional-district plan registry: artifact assembly, + # validating loader and 2010->2020 plan crosswalk. Assigns no + # households and applies no population treatment. + "cd_plan_registry.py", # Reviewed Chronicle feed pin loader; no population treatment. Remains # subject to the all-runtime source-identity scan. "chronicle_feed.py", diff --git a/tools/build_us_cd_plan_registry_artifact.py b/tools/build_us_cd_plan_registry_artifact.py index c54a854ee..239d0a24a 100644 --- a/tools/build_us_cd_plan_registry_artifact.py +++ b/tools/build_us_cd_plan_registry_artifact.py @@ -127,7 +127,9 @@ #: Blocks the Census split notes list as divided by a district line. For #: tabulation Census assigns each to one district "specified to the U.S. #: Census Bureau by the state"; the build checks the registry agrees. The -#: 118th, 119th and 120th notes each list only this block. +#: 118th, 119th and 120th notes each list only this block. The table is a +#: transcription of the notes, so the build pins the reviewed notes' bytes +#: (:data:`CENSUS_BLOCK_SPLIT_NOTE_SHA256`) and refuses a changed note. CENSUS_BLOCK_SPLITS: dict[str, dict[str, dict[str, Any]]] = { plan: { "080010096072000": { @@ -138,6 +140,12 @@ for plan in CENSUS_BEFS } +CENSUS_BLOCK_SPLIT_NOTE_SHA256 = { + "118th_congress": "016d7a697d79d84187828d19c8546878598f0e745bee88942c0749ef3875cb37", + "119th_congress": "f0b70f2bde47285f63367ef667a6080ec5c2042842af1e2457c0fb4b739dadcc", + "120th_congress": "722fa5d22d2d6c5c4a687686a6a039f515f0de045a4dbb16d967cc398a80a388", +} + PLAN_APPORTIONMENT = { "117th_congress": "2010_census", "118th_congress": "2020_census", @@ -162,8 +170,8 @@ "Districts of the 119th Congress (2025-2027); the block ladder's primary plan." ), "120th_congress": ( - "Districts for the November 3, 2026 election (120th Congress, " - "2027-2029), including the 2025-26 mid-decade redraws: Census's 120th " + "Districts for the 2026 elections (120th Congress, 2027-2029), " + "including the 2025-26 mid-decade redraws: Census's 120th " "Congressional District block equivalency file, except any state whose " "map in force differs from Census's rows (see known_deviations)." ), @@ -550,6 +558,14 @@ def _plan_from_census_bef( bef = CENSUS_BEFS[plan] path = _download(bef["url"], cache_dir) split_note = _download(bef["block_split_note"], cache_dir) + split_note_sha256 = _sha256(split_note) + if split_note_sha256 != CENSUS_BLOCK_SPLIT_NOTE_SHA256[plan]: + raise SystemExit( + f"{plan}: Census block-split note {bef['block_split_note']} has " + f"sha256 {split_note_sha256}, not the reviewed " + f"{CENSUS_BLOCK_SPLIT_NOTE_SHA256[plan]}. Re-read the note and update " + "CENSUS_BLOCK_SPLITS and CENSUS_BLOCK_SPLIT_NOTE_SHA256 together." + ) with _zip_member_stream(path, bef["member"]) as stream: national = parse_cd_block_assignment(stream, label=bef["member"]) national = concat_cd_block_assignments(