diff --git a/changelog.d/761-uk-rowwise-pilot.added.md b/changelog.d/761-uk-rowwise-pilot.added.md new file mode 100644 index 0000000000..7db3a1b12e --- /dev/null +++ b/changelog.d/761-uk-rowwise-pilot.added.md @@ -0,0 +1 @@ +The UK rowwise pilot machinery (#761, #495 increment 5) lands: the local-areas line is renamed `uk-local-*` and its `uk/local` Logbook scope is ratified (migration + mirror + README together); the rowwise dataset driver gains fail-closed `--input-sha256` and `--ladder-sha256` pins with a committed-crosswalk cross-check; the pool-only source-lineage modulus is refused on spine inputs carrying explicit lineage columns, which now report their own additive lineage block at pre-clone grain; the ladder clone path refuses pre-assigned geography columns; `expected_uk_ladder_area_support` computes the exact analytic per-constituency and per-LA expectation from the sampler's own stage weights; the per-area support diagnostics (ESS, distinct source households) lift off the solve matrix into `uk_area_support_summary` / `uk_ladder_area_support_summary` plus `uk_region_mix`, wired into the ladder manifest, dry-run plan, and a new `area_support_summary.csv` sidecar; and `--candidate-clone-counts` plans rows, bytes, and support at every candidate K in one dry-run. diff --git a/experiments/761-uk-rowwise-pilot-receipts.md b/experiments/761-uk-rowwise-pilot-receipts.md new file mode 100644 index 0000000000..ccfcfb7393 --- /dev/null +++ b/experiments/761-uk-rowwise-pilot-receipts.md @@ -0,0 +1,172 @@ +# K=1 rowwise pilot receipts (#761, #495 increment 5) + +Round A: spine-i, 2026-08-29, licensed machine. Round B (spine-j, after the +#804 licensed phase re-cuts the acceptance receipt) re-runs the same two +commands and appends its section below; per María's 2026-08-28 ruling the K +adjudication cites Round B, with Round A as the control. + +## R0 — input identity (Round A) + +| item | value | +|---|---| +| input | `spine-i.h5`, sha256 `0cce03992207b4e16d96483642e2edd943fd25112a6425208d748b31c7742416` (= the committed `uk/spine_candidate_acceptance.json` candidate pin), 153,184,127 bytes | +| pin arm | `--input-sha256` verified fail-closed; manifest `inputs.dataset.pin_verified: true` | +| weight kind | `importance` in, `importance` out; input mass log 6 records | +| ladder | `uk_oa_ladder_2021.npz`, sha256 `9c6d56b90d2e975d750106b175020a54c5ec6acf42ef8909d304a9d7fc3868a7`, `matches_local_area_crosswalk_pin: true` | +| code | `652d2891` (`uk-rowwise-pilot-761` rung-8 tip, on the merged #795+#801 main) | +| receipts | dry-run plan sha `a44db1464e3a50ed…`, build manifest sha `b1a1294afeecff6e…`, evidence dir `data/ukds/acceptance/761-rowwise-pilot/spine-i/` (licensed-side) | + +## R1 — the multi-K dry-run plan + +One invocation (`--n-clones 1 --candidate-clone-counts 1,2,4,10 --dry-run`, +seed 42). Every candidate K runs the identical fenced clone → weight-divide → +ladder-assign → gate pipeline at the build seed; `expected` is the analytic +expectation from the sampler's own stage weights. Bytes are the declared +input×K lower bound; the real K=1 build measured ×1.044 over it (added +geography columns), giving the calibrated ceiling column. + +### Constituency grain (650 areas, all assigned at every K) + +| K | households | H5 est. (calibrated) | rows min / med (realized) | rows min / med (expected) | ESS min / med | distinct sources min / med | +|--:|--:|--:|--:|--:|--:|--:| +| 1 | 52,846 | 153 MB (≈160 MB) | 28 / 72 | 27.3 / 71.7 | 15.3 / 62.2 | 28 / 71 | +| 2 | 105,692 | 306 MB (≈319 MB) | 42 / 144 | 54.6 / 143.5 | 28.8 / 120.7 | 42 / 137 | +| 4 | 211,384 | 613 MB (≈640 MB) | 101 / 287 | 109.3 / 286.9 | 86.4 / 240.4 | 97 / 256 | +| 10 | 528,460 | 1,532 MB (≈1,600 MB) | 301 / 714.5 | 273.2 / 717.3 | 245.0 / 592.3 | 276 / 552.5 | + +### Local-authority grain (361 areas, all assigned at every K) + +| K | rows min / med (realized) | rows min / med (expected) | ESS min / med | distinct sources min / med | +|--:|--:|--:|--:|--:| +| 1 | 1 / 106 | 1.7 / 106.6 | 1.0 / 89.4 | 1 / 102 | +| 2 | 3 / 215 | 3.3 / 213.2 | 2.4 / 182.5 | 3 / 201 | +| 4 | 7 / 420 | 6.7 / 426.4 | 6.8 / 355.4 | 7 / 364 | +| 10 | 17 / 1,064 | 16.6 / 1,066.1 | 12.1 / 892.2 | 17 / 721 | + +The LA minimum is the Isles of Scilly (`E06000053`; ~0.05% of national +households), then the City of London (`E09000001`): population-proportional +assignment gives them almost nothing at any affordable K — a support floor, +if one is wanted there, is target-side or assignment-side work, not a clone +multiplier. The constituency floor is Na h-Eileanan an Iar (`S14000027`), +exactly the area the epic predicted (~240 rows at the old pool's K=1 scale ⇒ +~24 at the spine's, measured 28). + +The analytic expectation tracks the realized draw at every K on both grains +(largest relative gap at the thin tail, e.g. K=2 constituency min 54.6 +expected vs 42 realized — a single seed's draw at a 42-row floor). Full +per-K stats: `experiments/761-uk-rowwise-pilot-support.json`. + +Basis note (post review round): the stats block now reads +`nonzero_households` (`rows_basis` recorded), so a zero-weight row can never +count as support. Round A measured `assigned == nonzero` for all 1,011 areas +— the spine's SPI channel carries positive importance mass, unlike the old +pool's zero-weight convention — so every figure above stands unchanged; +Round B emits the labelled shape natively. + +## R2 — the real K=1 build + +`--n-clones 1 --seed 42`, both pins armed, 2026-08-29T00:26Z: + +| check | result | +|---|---| +| `uk_geography_ladder` gate | **passed** — constituency/ward/ITL nonempty weighted shares 1.0, London weighted share 0.1261 ∈ (0.08, 0.20) | +| assignment | 650/650 constituencies, 361/361 local authorities, `missing_geography_rows: 0` | +| mass conservation | `abs_delta: 0.0` (input = output = 29,247,433.0), rtol 1e-9, **exact** | +| weight-kind chain | `importance` → `importance`; mass log 6 → 7 records (the single clone record) | +| lineage | `explicit_lineage_columns` basis: 16,288 distinct `source_household_id` (frs 16,288 / spi 10,000 — the SPI channel's sources nest inside the FRS set); flags: spi_synthetic 20,085, cgt_clone 26,420, band_donor 270; `pool: null`, no modulus | +| duplicate source×constituency pairs | 1,454 (K=1: SPI/CGT copies of one source landing in its own constituency — the honest support discount the lineage columns exist to expose) | +| cost | 17.3 s wall, **1.51 GB peak RSS**, output H5 159,984,959 bytes (sha `8bb2b5fa5f28e03e…`) | +| Logbook | pipeline `uk-local-rowwise`, rung `f100`, build id `…-20260829T002607Z-42440bbd`, row digest `a4d2c0b2b356493c…` — the **genesis row of the ratified `uk/local` chain** (`logbook/uk/local.jsonl`, chain validates) | + +Memory ceiling at K, linear in the clone factor from the measured K=1 point: +≈1.5 GB (K=1) → ≈3 GB (K=2) → ≈6 GB (K=4) → ≈15 GB (K=10) peak RSS, with +H5 sizes per R1. All of K ∈ {1,2,4} fit ordinary hardware; K=10 remains +feasible on the licensed machine. + +## R3 — grain (the #688 evidence) + +Pre-clone: 52,846 households = (16,288 raw FRS + 10,000 SPI-synthetic) × 2 ++ 270 CGT band donors, carrying 16,288 distinct source households. Post-clone +at K=1: row counts unchanged (52,846 / 61,211 / 113,649), ids untouched +(`_remap_ids` is the identity at clone 0), weights divided by 1, one mass +record appended. K is a declared build parameter: `parameters.n_clones` in +the manifest, `candidates.clone_counts` in the plan — the declaration #688's +open half asked for. The old `household_id mod 10^8` lineage rule does not +apply to this spine and is structurally refused; distinct-source accounting +reads `source_household_id` directly. + +## R4 — rung ruling + +The Logbook rung stays `f100` at every K: rungs are sample-fraction tokens +(`DESIGN.md` scale ladder: f001 = 1% smoke, f010 = 10% development, f100 = +full; `logbook.py` `_validate_rung`: "#624 fraction token"), and the pilot +consumes 100% of the spine regardless of the clone multiplier. K lives in +`parameters.n_clones`, not the rung. + +## R5 — K recommendation (adjudication pending Round B) + +Measured support says the epic's "first published candidate at K=2–4" range +survives the spine swap, at the top of the range: + +- **K=1 is a pilot, not a candidate**: constituency min-ESS 15.3 means the + thinnest areas calibrate on ~15 effective households. +- **K=4 is the recommendation for #762's first calibrated candidate**: + constituency floor 101 rows / ESS 86 / 97 distinct sources, median 287 + rows — between the US sparse and dense arms per area — at 640 MB and + ~6 GB solve headroom. LA-grain thinness (Scilly, City of London) is + unfixable by K and should be handled target-side. +- **K=10 buys diminishing returns** (median 714 ≈ US-dense 775, but 1.6 GB + H5 and ~15 GB RSS) and adds zero information; hold it unless #762's solve + diagnostics show support-limited misses at K=4. + +**Adjudication (María, 2026-08-30): K=4 approved** for #762's first +calibrated rowwise candidate, on the measurement above — constituency floor +101 rows / ESS 86.4 / 97 distinct sources, median 287 rows, at ≈640 MB and +≈6 GB solve headroom — confirmed unchanged on spine-k (Round B: support +inputs measured-preserved, tables reproduce exactly). K=10 stays held unless +#762's solve diagnostics show support-limited misses at K=4; the LA-grain +floor (Scilly, City of London) is target-side work at any K. The ruled K +becomes #762's declared `--n-clones`, recorded in that build's manifest. + +## Round B — spine-k (2026-08-30) + +The spine line moved past spine-i in the recert lane before any acceptance +re-cut: spine-j (`spine-j-remeasure`) and then **spine-k** +(`data/ukds/acceptance/spine-k-stack/spine-k.h5`, built 2026-08-30, release +id `uk-frs-spine-20260830T120239Z`, **14/14 spine gates passed**, importance +weights, mass total 29,247,433.0, the same 52,846/61,211/113,649 **row-count +roster** — the artifact bytes differ, and the Logbook `identity_digest` is +re-derived for this run, never carried over). Round B ran against spine-k, +sha-pinned at the artifact: +`b4403ea4b2d345de06e9f297fdeb9f144519407e98d653b09cf77b16c1e436cb`. The +committed `spine_candidate_acceptance.json` still names spine-i — the +acceptance re-mint is the recert lane's step; this pilot pins bytes, not the +receipt, and records that status honestly. + +Same two commands as Round A (multi-K dry-run at seed 42; real K=1 build, +Logbook row chained onto the genesis digest `a4d2c0b2…`): + +- **Support is bit-identical to Round A at every K on both grains** — every + min/median rows, ESS, and distinct-sources figure in R1's tables + reproduces exactly. Read precisely, this establishes two things and + entails a third. Measured: (a) spine-k **preserves the support inputs** — + household roster, region mix, and the importance-weight vector — which was + not knowable a priori (the recert changes could have moved any of them); + (b) the seeded two-stage draw is **reproducible** and the assignment + surface is **value-column-independent**. Entailed, once (a) holds: the + support tables reproduce bit-for-bit — the metric is a function of those + inputs only, so identity here is arithmetic, not an independent check. + The K adjudication is defined over the support surface (#761 items 6–7), + so it carries to spine-k because its inputs are measured-unchanged. + Anything value-dependent — whether the solve at the ruled K fits spine-k's + revised wealth columns — is invisible to this metric by design and belongs + to #762's fit diagnostics, not this pilot. +- Build receipts: gate passed 650/650 + 361/361, `missing_geography_rows: 0`, + mass `abs_delta: 0.0`, `importance` carried, mass log 6 → 7, lineage + identical (16,288 distinct sources; frs 16,288 / spi 10,000; flags + 20,085 / 26,420 / 270). 11.4 s wall, 1.51 GB peak RSS. +- Receipts: dry-run plan sha `5edc378eeac7e1ab…`, build manifest sha + `ded07a6122a3c36e…`, evidence `…/761-rowwise-pilot/spine-k/`; Logbook row + `15f6b09944f749cd…` (chain tail; `logbook/uk/local.jsonl` validates at 2 + rows). Stats emitted in the post-review `rows_basis: nonzero_households` + shape; `assigned == nonzero` for all 1,011 areas here too. diff --git a/experiments/761-uk-rowwise-pilot-support.json b/experiments/761-uk-rowwise-pilot-support.json new file mode 100644 index 0000000000..08a99a0aed --- /dev/null +++ b/experiments/761-uk-rowwise-pilot-support.json @@ -0,0 +1,2300 @@ +{ + "artifact_kind": "uk_rowwise_pilot_support_stats", + "candidates": [ + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "E14001454", + "ess": 15.27149392611927 + }, + { + "area_code": "E14001132", + "ess": 20.095695257804277 + }, + { + "area_code": "E14001515", + "ess": 20.63481711963163 + }, + { + "area_code": "E14001508", + "ess": 21.63829872776318 + }, + { + "area_code": "E14001455", + "ess": 23.0322856228197 + }, + { + "area_code": "E14001360", + "ess": 23.140344869850303 + }, + { + "area_code": "S14000027", + "ess": 24.16263812992307 + }, + { + "area_code": "E14001463", + "ess": 24.317601510355406 + }, + { + "area_code": "E14001186", + "ess": 24.654670866710497 + }, + { + "area_code": "E14001195", + "ess": 24.66316425668849 + }, + { + "area_code": "E14001388", + "ess": 26.286214083844957 + }, + { + "area_code": "E14001550", + "ess": 26.793894830865497 + }, + { + "area_code": "E14001303", + "ess": 29.111499596347695 + }, + { + "area_code": "E14001524", + "ess": 29.16543602979089 + }, + { + "area_code": "E14001189", + "ess": 29.751497224394736 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 28 + }, + { + "area_code": "E14001454", + "rows": 32 + }, + { + "area_code": "E14001309", + "rows": 36 + }, + { + "area_code": "E14001089", + "rows": 38 + }, + { + "area_code": "E14001123", + "rows": 41 + }, + { + "area_code": "E14001270", + "rows": 41 + }, + { + "area_code": "E14001204", + "rows": 42 + }, + { + "area_code": "E14001187", + "rows": 43 + }, + { + "area_code": "E14001276", + "rows": 43 + }, + { + "area_code": "E14001137", + "rows": 44 + }, + { + "area_code": "E14001374", + "rows": 44 + }, + { + "area_code": "E14001448", + "rows": 44 + }, + { + "area_code": "E14001124", + "rows": 45 + }, + { + "area_code": "E14001261", + "rows": 45 + }, + { + "area_code": "E14001360", + "rows": 45 + } + ], + "median_distinct_sources": 71.0, + "median_ess": 62.151628242143616, + "median_rows": 72.0, + "min_distinct_sources": 28, + "min_ess": 15.27149392611927, + "min_rows": 28, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 1.0 + }, + { + "area_code": "E09000001", + "ess": 8.844673393420043 + }, + { + "area_code": "S12000027", + "ess": 17.798842023607254 + }, + { + "area_code": "E07000035", + "ess": 21.016318762353905 + }, + { + "area_code": "E07000133", + "ess": 22.18918146591543 + }, + { + "area_code": "S12000023", + "ess": 23.44623761223246 + }, + { + "area_code": "E06000017", + "ess": 23.94465696601676 + }, + { + "area_code": "S12000013", + "ess": 24.16263812992307 + }, + { + "area_code": "E07000124", + "ess": 29.030121672784347 + }, + { + "area_code": "E07000135", + "ess": 31.560810193240044 + }, + { + "area_code": "E07000142", + "ess": 34.00462465185576 + }, + { + "area_code": "E07000136", + "ess": 35.037877189449375 + }, + { + "area_code": "E07000237", + "ess": 35.25903544081407 + }, + { + "area_code": "E07000218", + "ess": 35.410416897750636 + }, + { + "area_code": "E07000223", + "ess": 35.42512690790566 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 1 + }, + { + "area_code": "E09000001", + "rows": 10 + }, + { + "area_code": "S12000027", + "rows": 21 + }, + { + "area_code": "E07000133", + "rows": 26 + }, + { + "area_code": "S12000023", + "rows": 27 + }, + { + "area_code": "E06000017", + "rows": 28 + }, + { + "area_code": "S12000013", + "rows": 28 + }, + { + "area_code": "E07000124", + "rows": 33 + }, + { + "area_code": "E07000135", + "rows": 37 + }, + { + "area_code": "E07000223", + "rows": 39 + }, + { + "area_code": "E07000218", + "rows": 40 + }, + { + "area_code": "E07000136", + "rows": 42 + }, + { + "area_code": "E07000235", + "rows": 44 + }, + { + "area_code": "E07000074", + "rows": 47 + }, + { + "area_code": "E07000120", + "rows": 49 + } + ], + "median_distinct_sources": 102.0, + "median_ess": 89.3840331485377, + "median_rows": 106.0, + "min_distinct_sources": 1, + "min_ess": 1.0, + "min_rows": 1, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 27.323142724462524 + }, + { + "area_code": "E14001213", + "rows": 43.96348004983817 + }, + { + "area_code": "E14001089", + "rows": 44.79852868761248 + }, + { + "area_code": "S14000051", + "rows": 45.50555403862137 + }, + { + "area_code": "E14001454", + "rows": 45.69263309892105 + }, + { + "area_code": "E14001270", + "rows": 45.81192576146024 + }, + { + "area_code": "E14001300", + "rows": 46.08830678160054 + }, + { + "area_code": "E14001417", + "rows": 46.34933330062193 + }, + { + "area_code": "E14001137", + "rows": 46.3989401503907 + }, + { + "area_code": "E14001187", + "rows": 46.85485072207512 + }, + { + "area_code": "E14001414", + "rows": 46.90918203372663 + }, + { + "area_code": "E14001301", + "rows": 47.16784632180665 + }, + { + "area_code": "E14001153", + "rows": 47.9922649203448 + }, + { + "area_code": "E14001123", + "rows": 48.6076260805717 + }, + { + "area_code": "E14001333", + "rows": 48.61353165792513 + } + ], + "max_rows": 361.91626312250185, + "mean_rows": 81.30153846153846, + "median_rows": 71.72847581522676, + "min_rows": 27.323142724462524, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 1.6627178705139922 + }, + { + "area_code": "E09000001", + "rows": 5.004371002424983 + }, + { + "area_code": "S12000023", + "rows": 22.242684068920017 + }, + { + "area_code": "S12000027", + "rows": 23.262869969701352 + }, + { + "area_code": "S12000013", + "rows": 27.323142724462524 + }, + { + "area_code": "E06000017", + "rows": 27.638664317142478 + }, + { + "area_code": "E07000133", + "rows": 35.05453232779583 + }, + { + "area_code": "E07000218", + "rows": 37.007407555320576 + }, + { + "area_code": "E07000135", + "rows": 37.140158454497374 + }, + { + "area_code": "E07000124", + "rows": 39.36028899235197 + }, + { + "area_code": "E07000199", + "rows": 43.96197631917495 + }, + { + "area_code": "E07000047", + "rows": 44.49059696774796 + }, + { + "area_code": "E07000074", + "rows": 46.41700376343651 + }, + { + "area_code": "E07000223", + "rows": 46.75337968963706 + }, + { + "area_code": "E07000125", + "rows": 46.8050199035923 + } + ], + "max_rows": 1091.7357576570837, + "mean_rows": 146.38781163434902, + "median_rows": 106.60903552779729, + "min_rows": 1.6627178705139922, + "n_areas": 361 + } + }, + "household_rows": 52846, + "n_clones": 1, + "output_bytes_estimate": 153184127, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 28.0 + }, + { + "area_code": "E14001454", + "rows": 32.0 + }, + { + "area_code": "E14001309", + "rows": 36.0 + }, + { + "area_code": "E14001089", + "rows": 38.0 + }, + { + "area_code": "E14001123", + "rows": 41.0 + }, + { + "area_code": "E14001270", + "rows": 41.0 + }, + { + "area_code": "E14001204", + "rows": 42.0 + }, + { + "area_code": "E14001187", + "rows": 43.0 + }, + { + "area_code": "E14001276", + "rows": 43.0 + }, + { + "area_code": "E14001137", + "rows": 44.0 + }, + { + "area_code": "E14001374", + "rows": 44.0 + }, + { + "area_code": "E14001448", + "rows": 44.0 + }, + { + "area_code": "E14001124", + "rows": 45.0 + }, + { + "area_code": "E14001261", + "rows": 45.0 + }, + { + "area_code": "E14001360", + "rows": 45.0 + } + ], + "max_rows": 354.0, + "mean_rows": 81.30153846153846, + "median_rows": 72.0, + "min_rows": 28.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 1.0 + }, + { + "area_code": "E09000001", + "rows": 10.0 + }, + { + "area_code": "S12000027", + "rows": 21.0 + }, + { + "area_code": "E07000133", + "rows": 26.0 + }, + { + "area_code": "S12000023", + "rows": 27.0 + }, + { + "area_code": "E06000017", + "rows": 28.0 + }, + { + "area_code": "S12000013", + "rows": 28.0 + }, + { + "area_code": "E07000124", + "rows": 33.0 + }, + { + "area_code": "E07000135", + "rows": 37.0 + }, + { + "area_code": "E07000223", + "rows": 39.0 + }, + { + "area_code": "E07000218", + "rows": 40.0 + }, + { + "area_code": "E07000136", + "rows": 42.0 + }, + { + "area_code": "E07000235", + "rows": 44.0 + }, + { + "area_code": "E07000074", + "rows": 47.0 + }, + { + "area_code": "E07000120", + "rows": 49.0 + } + ], + "max_rows": 1037.0, + "mean_rows": 146.38781163434902, + "median_rows": 106.0, + "min_rows": 1.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 28.772766859888733 + }, + { + "area_code": "E14001241", + "ess": 48.49213048209239 + }, + { + "area_code": "E14001354", + "ess": 51.87001349651396 + }, + { + "area_code": "E14001537", + "ess": 53.66603237929316 + }, + { + "area_code": "E14001421", + "ess": 54.07387209590336 + }, + { + "area_code": "E14001353", + "ess": 54.25201107246176 + }, + { + "area_code": "E14001575", + "ess": 55.46707664746293 + }, + { + "area_code": "E14001221", + "ess": 57.28923069544969 + }, + { + "area_code": "E14001339", + "ess": 57.53554831458898 + }, + { + "area_code": "E14001574", + "ess": 58.58463195974067 + }, + { + "area_code": "E14001132", + "ess": 59.28504402194878 + }, + { + "area_code": "E14001513", + "ess": 60.17316928039642 + }, + { + "area_code": "E14001445", + "ess": 61.57417297964269 + }, + { + "area_code": "E14001213", + "ess": 61.73672943178649 + }, + { + "area_code": "E14001153", + "ess": 62.18794773807564 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 42 + }, + { + "area_code": "E14001153", + "rows": 74 + }, + { + "area_code": "E14001414", + "rows": 79 + }, + { + "area_code": "E14001221", + "rows": 81 + }, + { + "area_code": "E14001300", + "rows": 81 + }, + { + "area_code": "E14001123", + "rows": 83 + }, + { + "area_code": "E14001187", + "rows": 84 + }, + { + "area_code": "E14001270", + "rows": 84 + }, + { + "area_code": "E14001167", + "rows": 87 + }, + { + "area_code": "E14001089", + "rows": 90 + }, + { + "area_code": "E14001257", + "rows": 91 + }, + { + "area_code": "E14001316", + "rows": 91 + }, + { + "area_code": "E14001333", + "rows": 91 + }, + { + "area_code": "E14001086", + "rows": 92 + }, + { + "area_code": "E14001526", + "rows": 92 + } + ], + "median_distinct_sources": 137.0, + "median_ess": 120.74144833002467, + "median_rows": 144.0, + "min_distinct_sources": 42, + "min_ess": 28.772766859888733, + "min_rows": 42, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 2.4156784854913016 + }, + { + "area_code": "E09000001", + "ess": 6.880763051494756 + }, + { + "area_code": "S12000013", + "ess": 28.772766859888733 + }, + { + "area_code": "S12000023", + "ess": 38.23722968460105 + }, + { + "area_code": "E06000017", + "ess": 39.626883904637666 + }, + { + "area_code": "E07000047", + "ess": 41.21513110073454 + }, + { + "area_code": "E07000133", + "ess": 41.55872805737255 + }, + { + "area_code": "S12000027", + "ess": 45.27906261453043 + }, + { + "area_code": "E07000218", + "ess": 48.75943362865257 + }, + { + "area_code": "E07000124", + "ess": 61.65635714775404 + }, + { + "area_code": "E07000223", + "ess": 68.0240614605536 + }, + { + "area_code": "E07000136", + "ess": 68.82740412657078 + }, + { + "area_code": "E07000135", + "ess": 69.31006171494359 + }, + { + "area_code": "E07000035", + "ess": 73.36159173306376 + }, + { + "area_code": "E07000142", + "ess": 74.38243109597174 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 3 + }, + { + "area_code": "E09000001", + "rows": 7 + }, + { + "area_code": "S12000013", + "rows": 42 + }, + { + "area_code": "S12000023", + "rows": 43 + }, + { + "area_code": "E06000017", + "rows": 46 + }, + { + "area_code": "S12000027", + "rows": 53 + }, + { + "area_code": "E07000133", + "rows": 64 + }, + { + "area_code": "E07000218", + "rows": 65 + }, + { + "area_code": "E07000124", + "rows": 68 + }, + { + "area_code": "E07000223", + "rows": 76 + }, + { + "area_code": "E07000136", + "rows": 81 + }, + { + "area_code": "E07000135", + "rows": 83 + }, + { + "area_code": "E07000125", + "rows": 89 + }, + { + "area_code": "E07000235", + "rows": 92 + }, + { + "area_code": "E07000120", + "rows": 95 + } + ], + "median_distinct_sources": 201.0, + "median_ess": 182.48559250642253, + "median_rows": 215.0, + "min_distinct_sources": 3, + "min_ess": 2.4156784854913016, + "min_rows": 3, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 54.64628544892505 + }, + { + "area_code": "E14001213", + "rows": 87.92696009967634 + }, + { + "area_code": "E14001089", + "rows": 89.59705737522496 + }, + { + "area_code": "S14000051", + "rows": 91.01110807724274 + }, + { + "area_code": "E14001454", + "rows": 91.3852661978421 + }, + { + "area_code": "E14001270", + "rows": 91.62385152292048 + }, + { + "area_code": "E14001300", + "rows": 92.17661356320107 + }, + { + "area_code": "E14001417", + "rows": 92.69866660124386 + }, + { + "area_code": "E14001137", + "rows": 92.7978803007814 + }, + { + "area_code": "E14001187", + "rows": 93.70970144415024 + }, + { + "area_code": "E14001414", + "rows": 93.81836406745326 + }, + { + "area_code": "E14001301", + "rows": 94.3356926436133 + }, + { + "area_code": "E14001153", + "rows": 95.9845298406896 + }, + { + "area_code": "E14001123", + "rows": 97.2152521611434 + }, + { + "area_code": "E14001333", + "rows": 97.22706331585026 + } + ], + "max_rows": 723.8325262450037, + "mean_rows": 162.6030769230769, + "median_rows": 143.45695163045352, + "min_rows": 54.64628544892505, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 3.3254357410279844 + }, + { + "area_code": "E09000001", + "rows": 10.008742004849966 + }, + { + "area_code": "S12000023", + "rows": 44.485368137840034 + }, + { + "area_code": "S12000027", + "rows": 46.525739939402705 + }, + { + "area_code": "S12000013", + "rows": 54.64628544892505 + }, + { + "area_code": "E06000017", + "rows": 55.277328634284956 + }, + { + "area_code": "E07000133", + "rows": 70.10906465559167 + }, + { + "area_code": "E07000218", + "rows": 74.01481511064115 + }, + { + "area_code": "E07000135", + "rows": 74.28031690899475 + }, + { + "area_code": "E07000124", + "rows": 78.72057798470394 + }, + { + "area_code": "E07000199", + "rows": 87.9239526383499 + }, + { + "area_code": "E07000047", + "rows": 88.98119393549592 + }, + { + "area_code": "E07000074", + "rows": 92.83400752687302 + }, + { + "area_code": "E07000223", + "rows": 93.50675937927411 + }, + { + "area_code": "E07000125", + "rows": 93.6100398071846 + } + ], + "max_rows": 2183.4715153141674, + "mean_rows": 292.77562326869804, + "median_rows": 213.21807105559458, + "min_rows": 3.3254357410279844, + "n_areas": 361 + } + }, + "household_rows": 105692, + "n_clones": 2, + "output_bytes_estimate": 306368254, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 42.0 + }, + { + "area_code": "E14001153", + "rows": 74.0 + }, + { + "area_code": "E14001414", + "rows": 79.0 + }, + { + "area_code": "E14001221", + "rows": 81.0 + }, + { + "area_code": "E14001300", + "rows": 81.0 + }, + { + "area_code": "E14001123", + "rows": 83.0 + }, + { + "area_code": "E14001187", + "rows": 84.0 + }, + { + "area_code": "E14001270", + "rows": 84.0 + }, + { + "area_code": "E14001167", + "rows": 87.0 + }, + { + "area_code": "E14001089", + "rows": 90.0 + }, + { + "area_code": "E14001257", + "rows": 91.0 + }, + { + "area_code": "E14001316", + "rows": 91.0 + }, + { + "area_code": "E14001333", + "rows": 91.0 + }, + { + "area_code": "E14001086", + "rows": 92.0 + }, + { + "area_code": "E14001526", + "rows": 92.0 + } + ], + "max_rows": 715.0, + "mean_rows": 162.6030769230769, + "median_rows": 144.0, + "min_rows": 42.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 3.0 + }, + { + "area_code": "E09000001", + "rows": 7.0 + }, + { + "area_code": "S12000013", + "rows": 42.0 + }, + { + "area_code": "S12000023", + "rows": 43.0 + }, + { + "area_code": "E06000017", + "rows": 46.0 + }, + { + "area_code": "S12000027", + "rows": 53.0 + }, + { + "area_code": "E07000133", + "rows": 64.0 + }, + { + "area_code": "E07000218", + "rows": 65.0 + }, + { + "area_code": "E07000124", + "rows": 68.0 + }, + { + "area_code": "E07000223", + "rows": 76.0 + }, + { + "area_code": "E07000136", + "rows": 81.0 + }, + { + "area_code": "E07000135", + "rows": 83.0 + }, + { + "area_code": "E07000125", + "rows": 89.0 + }, + { + "area_code": "E07000235", + "rows": 92.0 + }, + { + "area_code": "E07000120", + "rows": 95.0 + } + ], + "max_rows": 2176.0, + "mean_rows": 292.77562326869804, + "median_rows": 215.0, + "min_rows": 3.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 86.35718423510014 + }, + { + "area_code": "E14001231", + "ess": 108.2130074468227 + }, + { + "area_code": "E14001164", + "ess": 113.10723834593558 + }, + { + "area_code": "E14001417", + "ess": 114.49610093904711 + }, + { + "area_code": "E14001240", + "ess": 115.46871900751577 + }, + { + "area_code": "E14001536", + "ess": 118.08752459304661 + }, + { + "area_code": "E14001399", + "ess": 119.73863447434024 + }, + { + "area_code": "E14001371", + "ess": 125.82891686500399 + }, + { + "area_code": "E14001309", + "ess": 126.42553709302008 + }, + { + "area_code": "E14001454", + "ess": 127.31501498907119 + }, + { + "area_code": "E14001133", + "ess": 128.58281864500086 + }, + { + "area_code": "E14001485", + "ess": 129.49044303779488 + }, + { + "area_code": "E14001434", + "ess": 130.07713437572616 + }, + { + "area_code": "E14001388", + "ess": 130.93269449992587 + }, + { + "area_code": "E14001213", + "ess": 133.52997153756021 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 101 + }, + { + "area_code": "E14001213", + "rows": 154 + }, + { + "area_code": "E14001270", + "rows": 165 + }, + { + "area_code": "S14000051", + "rows": 173 + }, + { + "area_code": "E14001454", + "rows": 174 + }, + { + "area_code": "E14001089", + "rows": 179 + }, + { + "area_code": "E14001534", + "rows": 181 + }, + { + "area_code": "E14001417", + "rows": 182 + }, + { + "area_code": "E14001441", + "rows": 182 + }, + { + "area_code": "E14001558", + "rows": 183 + }, + { + "area_code": "E14001271", + "rows": 184 + }, + { + "area_code": "E14001301", + "rows": 185 + }, + { + "area_code": "E14001556", + "rows": 189 + }, + { + "area_code": "E14001189", + "rows": 191 + }, + { + "area_code": "E14001073", + "rows": 192 + } + ], + "median_distinct_sources": 256.0, + "median_ess": 240.42002819931716, + "median_rows": 287.0, + "min_distinct_sources": 97, + "min_ess": 86.35718423510014, + "min_rows": 101, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 6.803346397579617 + }, + { + "area_code": "E09000001", + "ess": 11.966201758605846 + }, + { + "area_code": "S12000023", + "ess": 71.25580669891772 + }, + { + "area_code": "S12000027", + "ess": 76.45656052703967 + }, + { + "area_code": "S12000013", + "ess": 86.35718423510014 + }, + { + "area_code": "E07000133", + "ess": 87.98764186217531 + }, + { + "area_code": "E06000017", + "ess": 91.98533042323214 + }, + { + "area_code": "E07000080", + "ess": 111.00058075828959 + }, + { + "area_code": "E07000236", + "ess": 129.4452253171101 + }, + { + "area_code": "E07000235", + "ess": 130.5025540814119 + }, + { + "area_code": "E07000135", + "ess": 131.41704492562806 + }, + { + "area_code": "E07000124", + "ess": 134.8185493449547 + }, + { + "area_code": "E07000199", + "ess": 139.0479421625425 + }, + { + "area_code": "E07000047", + "ess": 146.70768863130988 + }, + { + "area_code": "E07000119", + "ess": 150.24866836485344 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 7 + }, + { + "area_code": "E09000001", + "rows": 14 + }, + { + "area_code": "S12000023", + "rows": 86 + }, + { + "area_code": "S12000027", + "rows": 87 + }, + { + "area_code": "S12000013", + "rows": 101 + }, + { + "area_code": "E06000017", + "rows": 107 + }, + { + "area_code": "E07000133", + "rows": 142 + }, + { + "area_code": "E07000124", + "rows": 150 + }, + { + "area_code": "E07000236", + "rows": 151 + }, + { + "area_code": "E07000135", + "rows": 157 + }, + { + "area_code": "E07000199", + "rows": 164 + }, + { + "area_code": "E07000047", + "rows": 172 + }, + { + "area_code": "E07000218", + "rows": 179 + }, + { + "area_code": "E07000074", + "rows": 182 + }, + { + "area_code": "E07000125", + "rows": 186 + } + ], + "median_distinct_sources": 364.0, + "median_ess": 355.4371598394059, + "median_rows": 420.0, + "min_distinct_sources": 7, + "min_ess": 6.803346397579617, + "min_rows": 7, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 109.2925708978501 + }, + { + "area_code": "E14001213", + "rows": 175.85392019935267 + }, + { + "area_code": "E14001089", + "rows": 179.19411475044993 + }, + { + "area_code": "S14000051", + "rows": 182.02221615448548 + }, + { + "area_code": "E14001454", + "rows": 182.7705323956842 + }, + { + "area_code": "E14001270", + "rows": 183.24770304584095 + }, + { + "area_code": "E14001300", + "rows": 184.35322712640215 + }, + { + "area_code": "E14001417", + "rows": 185.39733320248772 + }, + { + "area_code": "E14001137", + "rows": 185.5957606015628 + }, + { + "area_code": "E14001187", + "rows": 187.41940288830048 + }, + { + "area_code": "E14001414", + "rows": 187.63672813490652 + }, + { + "area_code": "E14001301", + "rows": 188.6713852872266 + }, + { + "area_code": "E14001153", + "rows": 191.9690596813792 + }, + { + "area_code": "E14001123", + "rows": 194.4305043222868 + }, + { + "area_code": "E14001333", + "rows": 194.45412663170052 + } + ], + "max_rows": 1447.6650524900074, + "mean_rows": 325.2061538461538, + "median_rows": 286.91390326090703, + "min_rows": 109.2925708978501, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 6.650871482055969 + }, + { + "area_code": "E09000001", + "rows": 20.01748400969993 + }, + { + "area_code": "S12000023", + "rows": 88.97073627568007 + }, + { + "area_code": "S12000027", + "rows": 93.05147987880541 + }, + { + "area_code": "S12000013", + "rows": 109.2925708978501 + }, + { + "area_code": "E06000017", + "rows": 110.55465726856991 + }, + { + "area_code": "E07000133", + "rows": 140.21812931118333 + }, + { + "area_code": "E07000218", + "rows": 148.0296302212823 + }, + { + "area_code": "E07000135", + "rows": 148.5606338179895 + }, + { + "area_code": "E07000124", + "rows": 157.44115596940787 + }, + { + "area_code": "E07000199", + "rows": 175.8479052766998 + }, + { + "area_code": "E07000047", + "rows": 177.96238787099185 + }, + { + "area_code": "E07000074", + "rows": 185.66801505374605 + }, + { + "area_code": "E07000223", + "rows": 187.01351875854823 + }, + { + "area_code": "E07000125", + "rows": 187.2200796143692 + } + ], + "max_rows": 4366.943030628335, + "mean_rows": 585.5512465373961, + "median_rows": 426.43614211118916, + "min_rows": 6.650871482055969, + "n_areas": 361 + } + }, + "household_rows": 211384, + "n_clones": 4, + "output_bytes_estimate": 612736508, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 101.0 + }, + { + "area_code": "E14001213", + "rows": 154.0 + }, + { + "area_code": "E14001270", + "rows": 165.0 + }, + { + "area_code": "S14000051", + "rows": 173.0 + }, + { + "area_code": "E14001454", + "rows": 174.0 + }, + { + "area_code": "E14001089", + "rows": 179.0 + }, + { + "area_code": "E14001534", + "rows": 181.0 + }, + { + "area_code": "E14001417", + "rows": 182.0 + }, + { + "area_code": "E14001441", + "rows": 182.0 + }, + { + "area_code": "E14001558", + "rows": 183.0 + }, + { + "area_code": "E14001271", + "rows": 184.0 + }, + { + "area_code": "E14001301", + "rows": 185.0 + }, + { + "area_code": "E14001556", + "rows": 189.0 + }, + { + "area_code": "E14001189", + "rows": 191.0 + }, + { + "area_code": "E14001073", + "rows": 192.0 + } + ], + "max_rows": 1465.0, + "mean_rows": 325.2061538461538, + "median_rows": 287.0, + "min_rows": 101.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 7.0 + }, + { + "area_code": "E09000001", + "rows": 14.0 + }, + { + "area_code": "S12000023", + "rows": 86.0 + }, + { + "area_code": "S12000027", + "rows": 87.0 + }, + { + "area_code": "S12000013", + "rows": 101.0 + }, + { + "area_code": "E06000017", + "rows": 107.0 + }, + { + "area_code": "E07000133", + "rows": 142.0 + }, + { + "area_code": "E07000124", + "rows": 150.0 + }, + { + "area_code": "E07000236", + "rows": 151.0 + }, + { + "area_code": "E07000135", + "rows": 157.0 + }, + { + "area_code": "E07000199", + "rows": 164.0 + }, + { + "area_code": "E07000047", + "rows": 172.0 + }, + { + "area_code": "E07000218", + "rows": 179.0 + }, + { + "area_code": "E07000074", + "rows": 182.0 + }, + { + "area_code": "E07000125", + "rows": 186.0 + } + ], + "max_rows": 4396.0, + "mean_rows": 585.5512465373961, + "median_rows": 420.0, + "min_rows": 7.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 245.00661031168204 + }, + { + "area_code": "E14001454", + "ess": 309.58798378255244 + }, + { + "area_code": "E14001186", + "ess": 325.0021849429837 + }, + { + "area_code": "E14001300", + "ess": 336.04748855009973 + }, + { + "area_code": "E14001189", + "ess": 338.31029112850126 + }, + { + "area_code": "E14001276", + "ess": 343.75592586340593 + }, + { + "area_code": "E14001126", + "ess": 344.12611223537334 + }, + { + "area_code": "E14001548", + "ess": 345.93438568379673 + }, + { + "area_code": "E14001417", + "ess": 346.9780892361871 + }, + { + "area_code": "E14001429", + "ess": 348.2148890152362 + }, + { + "area_code": "E14001301", + "ess": 348.4567826735376 + }, + { + "area_code": "E14001187", + "ess": 349.9808726586294 + }, + { + "area_code": "E14001460", + "ess": 355.6390207205338 + }, + { + "area_code": "E14001550", + "ess": 355.8382451779587 + }, + { + "area_code": "E14001448", + "ess": 360.1587775232958 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 301 + }, + { + "area_code": "E14001213", + "rows": 443 + }, + { + "area_code": "E14001417", + "rows": 446 + }, + { + "area_code": "E14001270", + "rows": 447 + }, + { + "area_code": "E14001333", + "rows": 449 + }, + { + "area_code": "E14001301", + "rows": 450 + }, + { + "area_code": "E14001448", + "rows": 454 + }, + { + "area_code": "E14001454", + "rows": 454 + }, + { + "area_code": "E14001089", + "rows": 455 + }, + { + "area_code": "E14001187", + "rows": 459 + }, + { + "area_code": "E14001300", + "rows": 459 + }, + { + "area_code": "E14001271", + "rows": 479 + }, + { + "area_code": "S14000051", + "rows": 487 + }, + { + "area_code": "E14001550", + "rows": 491 + }, + { + "area_code": "E14001189", + "rows": 493 + } + ], + "median_distinct_sources": 552.5, + "median_ess": 592.328299460505, + "median_rows": 714.5, + "min_distinct_sources": 276, + "min_ess": 245.00661031168204, + "min_rows": 301, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 12.113919922730993 + }, + { + "area_code": "E09000001", + "ess": 42.0438939071751 + }, + { + "area_code": "E06000017", + "ess": 180.75401191345054 + }, + { + "area_code": "S12000023", + "ess": 197.0121926202139 + }, + { + "area_code": "S12000027", + "ess": 198.28988727221227 + }, + { + "area_code": "E07000133", + "ess": 222.9318295202106 + }, + { + "area_code": "E07000135", + "ess": 244.23156569686483 + }, + { + "area_code": "S12000013", + "ess": 245.00661031168204 + }, + { + "area_code": "E07000218", + "ess": 319.2242286745908 + }, + { + "area_code": "E07000223", + "ess": 332.43048742715064 + }, + { + "area_code": "E07000124", + "ess": 344.6432429352619 + }, + { + "area_code": "E07000199", + "ess": 374.24511694345773 + }, + { + "area_code": "E07000235", + "ess": 374.33815974358714 + }, + { + "area_code": "E07000047", + "ess": 374.8395900190861 + }, + { + "area_code": "S12000005", + "ess": 379.3890742905147 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 17 + }, + { + "area_code": "E09000001", + "rows": 49 + }, + { + "area_code": "S12000027", + "rows": 238 + }, + { + "area_code": "S12000023", + "rows": 249 + }, + { + "area_code": "E06000017", + "rows": 287 + }, + { + "area_code": "S12000013", + "rows": 301 + }, + { + "area_code": "E07000135", + "rows": 350 + }, + { + "area_code": "E07000133", + "rows": 357 + }, + { + "area_code": "E07000218", + "rows": 389 + }, + { + "area_code": "E07000124", + "rows": 398 + }, + { + "area_code": "E07000199", + "rows": 422 + }, + { + "area_code": "E07000047", + "rows": 447 + }, + { + "area_code": "E07000074", + "rows": 456 + }, + { + "area_code": "E07000125", + "rows": 456 + }, + { + "area_code": "E07000223", + "rows": 463 + } + ], + "median_distinct_sources": 721.0, + "median_ess": 892.1532218859292, + "median_rows": 1064.0, + "min_distinct_sources": 17, + "min_ess": 12.113919922730993, + "min_rows": 17, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 273.23142724462525 + }, + { + "area_code": "E14001213", + "rows": 439.63480049838165 + }, + { + "area_code": "E14001089", + "rows": 447.9852868761248 + }, + { + "area_code": "S14000051", + "rows": 455.05554038621364 + }, + { + "area_code": "E14001454", + "rows": 456.9263309892105 + }, + { + "area_code": "E14001270", + "rows": 458.11925761460236 + }, + { + "area_code": "E14001300", + "rows": 460.88306781600534 + }, + { + "area_code": "E14001417", + "rows": 463.49333300621925 + }, + { + "area_code": "E14001137", + "rows": 463.98940150390695 + }, + { + "area_code": "E14001187", + "rows": 468.54850722075116 + }, + { + "area_code": "E14001414", + "rows": 469.0918203372663 + }, + { + "area_code": "E14001301", + "rows": 471.6784632180665 + }, + { + "area_code": "E14001153", + "rows": 479.92264920344803 + }, + { + "area_code": "E14001123", + "rows": 486.076260805717 + }, + { + "area_code": "E14001333", + "rows": 486.13531657925125 + } + ], + "max_rows": 3619.1626312250182, + "mean_rows": 813.0153846153846, + "median_rows": 717.2847581522676, + "min_rows": 273.23142724462525, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 16.627178705139926 + }, + { + "area_code": "E09000001", + "rows": 50.043710024249826 + }, + { + "area_code": "S12000023", + "rows": 222.42684068920013 + }, + { + "area_code": "S12000027", + "rows": 232.62869969701347 + }, + { + "area_code": "S12000013", + "rows": 273.23142724462525 + }, + { + "area_code": "E06000017", + "rows": 276.3866431714248 + }, + { + "area_code": "E07000133", + "rows": 350.54532327795835 + }, + { + "area_code": "E07000218", + "rows": 370.0740755532057 + }, + { + "area_code": "E07000135", + "rows": 371.4015845449737 + }, + { + "area_code": "E07000124", + "rows": 393.6028899235196 + }, + { + "area_code": "E07000199", + "rows": 439.61976319174954 + }, + { + "area_code": "E07000047", + "rows": 444.90596967747956 + }, + { + "area_code": "E07000074", + "rows": 464.17003763436514 + }, + { + "area_code": "E07000223", + "rows": 467.5337968963706 + }, + { + "area_code": "E07000125", + "rows": 468.0501990359231 + } + ], + "max_rows": 10917.35757657084, + "mean_rows": 1463.8781163434903, + "median_rows": 1066.0903552779728, + "min_rows": 16.627178705139926, + "n_areas": 361 + } + }, + "household_rows": 528460, + "n_clones": 10, + "output_bytes_estimate": 1531841270, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 301.0 + }, + { + "area_code": "E14001213", + "rows": 443.0 + }, + { + "area_code": "E14001417", + "rows": 446.0 + }, + { + "area_code": "E14001270", + "rows": 447.0 + }, + { + "area_code": "E14001333", + "rows": 449.0 + }, + { + "area_code": "E14001301", + "rows": 450.0 + }, + { + "area_code": "E14001448", + "rows": 454.0 + }, + { + "area_code": "E14001454", + "rows": 454.0 + }, + { + "area_code": "E14001089", + "rows": 455.0 + }, + { + "area_code": "E14001187", + "rows": 459.0 + }, + { + "area_code": "E14001300", + "rows": 459.0 + }, + { + "area_code": "E14001271", + "rows": 479.0 + }, + { + "area_code": "S14000051", + "rows": 487.0 + }, + { + "area_code": "E14001550", + "rows": 491.0 + }, + { + "area_code": "E14001189", + "rows": 493.0 + } + ], + "max_rows": 3634.0, + "mean_rows": 813.0153846153846, + "median_rows": 714.5, + "min_rows": 301.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 17.0 + }, + { + "area_code": "E09000001", + "rows": 49.0 + }, + { + "area_code": "S12000027", + "rows": 238.0 + }, + { + "area_code": "S12000023", + "rows": 249.0 + }, + { + "area_code": "E06000017", + "rows": 287.0 + }, + { + "area_code": "S12000013", + "rows": 301.0 + }, + { + "area_code": "E07000135", + "rows": 350.0 + }, + { + "area_code": "E07000133", + "rows": 357.0 + }, + { + "area_code": "E07000218", + "rows": 389.0 + }, + { + "area_code": "E07000124", + "rows": 398.0 + }, + { + "area_code": "E07000199", + "rows": 422.0 + }, + { + "area_code": "E07000047", + "rows": 447.0 + }, + { + "area_code": "E07000074", + "rows": 456.0 + }, + { + "area_code": "E07000125", + "rows": 456.0 + }, + { + "area_code": "E07000223", + "rows": 463.0 + } + ], + "max_rows": 10929.0, + "mean_rows": 1463.8781163434903, + "median_rows": 1064.0, + "min_rows": 17.0, + "n_areas": 361 + } + } + } + ], + "input_sha256": "0cce03992207b4e16d96483642e2edd943fd25112a6425208d748b31c7742416", + "ladder_sha256": "9c6d56b90d2e975d750106b175020a54c5ec6acf42ef8909d304a9d7fc3868a7", + "plan_git_commit": "652d2891670bbfe090874905a52c9860825a24b6", + "region_mix": [ + { + "region_code": "E12000001", + "row_share": 0.04087348143662718, + "rows": 2160, + "weight_share": 0.04132138137843869 + }, + { + "region_code": "E12000002", + "row_share": 0.09105703364493055, + "rows": 4812, + "weight_share": 0.11135883275636532 + }, + { + "region_code": "E12000003", + "row_share": 0.09870188850622563, + "rows": 5216, + "weight_share": 0.08260608717353075 + }, + { + "region_code": "E12000004", + "row_share": 0.061291299246868255, + "rows": 3239, + "weight_share": 0.07270689362721167 + }, + { + "region_code": "E12000005", + "row_share": 0.062067138477841274, + "rows": 3280, + "weight_share": 0.08658163834982258 + }, + { + "region_code": "E12000006", + "row_share": 0.08346894750785301, + "rows": 4411, + "weight_share": 0.09236710335114426 + }, + { + "region_code": "E12000007", + "row_share": 0.07652424024524089, + "rows": 4044, + "weight_share": 0.12610049116675187 + }, + { + "region_code": "E12000008", + "row_share": 0.12224198614843129, + "rows": 6460, + "weight_share": 0.13547538798818118 + }, + { + "region_code": "E12000009", + "row_share": 0.0822578813912122, + "rows": 4347, + "weight_share": 0.08746195492324632 + }, + { + "region_code": "N99999999", + "row_share": 0.1072171971388563, + "rows": 5666, + "weight_share": 0.027166840020911694 + }, + { + "region_code": "S99999999", + "row_share": 0.10222154940771298, + "rows": 5402, + "weight_share": 0.08843631963643898 + }, + { + "region_code": "W99999999", + "row_share": 0.07207735684820044, + "rows": 3809, + "weight_share": 0.04841706962795675 + } + ], + "round": "spine-i", + "seed": 42 +} diff --git a/logbook/README.md b/logbook/README.md index ff96975853..a148dd142a 100644 --- a/logbook/README.md +++ b/logbook/README.md @@ -22,7 +22,7 @@ the digest and destroy the evidence the chain exists to preserve. Every new archive uses `logbook//.jsonl`. The dataset token names the line's base data, not an epic and not a build mechanism: `frs` is the FRS-derived survey-microdata line (persons, benunits, and households — -its spine, staging, and every imputation stage share one chain), `locals` is +its spine, staging, and every imputation stage share one chain), `local` is the local-areas product line, and a future firm line will carry its own base-data name. Different base data need different scopes, and scope is also where builds serialize: if two lines must append concurrently, they need @@ -30,13 +30,12 @@ separate chains. ## The vocabulary is closed-world -The ratified scopes are exactly `us` and `uk/frs` — deliberately minimal: a -scope is ratified when its line starts archiving, not speculatively. The -`uk-locals-*` drivers already follow the naming convention, and their rows -spool locally regardless (recording never consults this list); `uk/locals` -gets ratified when the local-areas line first archives. A pipeline whose -name derives any unratified scope is refused — even at genesis. Opening a -scope is a reviewed decision made in three places together: the +The ratified scopes are exactly `us`, `uk/frs`, and `uk/local` — deliberately +minimal: a scope is ratified when its line starts archiving, not speculatively. +The `uk/local` scope was ratified for the local-areas line's first archived run +in #761. A pipeline whose name derives any unratified scope is refused — even +at genesis. Opening a scope is a reviewed decision made in three places +together: the `logbook.scope_declared` migration, the `DECLARED_SCOPES` mirror in `tools/logbook.py`, and this list. It is never a side effect of a well-formed pipeline name: without the list, a typo'd `uk-huseholds-...` @@ -102,7 +101,7 @@ The database keeps `builds_unique_predecessor` global because two rows claiming one predecessor is a fork wherever it happens. Genesis, tail discovery, and advisory locking are scoped: `logbook.chain_scope(pipeline)` maps the three legacy US pipeline names to `us`, and maps new pipelines like -`uk-frs-staging` or `uk-locals-rowwise` to `uk/frs` and `uk/locals`; +`uk-frs-staging` or `uk-local-rowwise` to `uk/frs` and `uk/local`; `logbook.scope_declared` then refuses any scope outside the ratified list. The per-scope advisory lock lets independent scopes append concurrently while appends within one scope still serialize. diff --git a/logbook/uk/local.jsonl b/logbook/uk/local.jsonl new file mode 100644 index 0000000000..d6bbe4143b --- /dev/null +++ b/logbook/uk/local.jsonl @@ -0,0 +1,2 @@ +{"artifact_location":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/uk-2024-locals-rowwise-pilot-k1.h5","build_id":"uk-local-rowwise-ladder-f100-s42-20260829T002607Z-42440bbd","code_pin":"652d2891670bbfe090874905a52c9860825a24b6","cost_usd":null,"disposition":"iterating","gate_verdicts":{"uk_geography_ladder":{"receipt":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/rowwise_build_manifest.json#/rowwise_dataset/gate","verdict":"passed"}},"identity_digest":"9547050e6d16833fba9572a1517c4fad36bbeb0ccdbf4261c435884125c3fb1f","input_pins_digest":"eeb124ca3665982bd41e3ebc2820bfc6dd00c54097d4a55d1ff508f04d89dcf6","phases_reached":["attempt_started","configured","inputs_pinned","cloned","manifest_written"],"pipeline":"uk-local-rowwise","prediction_id":null,"prev_row_digest":null,"row_digest":"a4d2c0b2b356493cefb30aca6aa35ba33cfeff584b6946e09164a1cadff4c2de","rung":"f100","seed":42,"ts":"2026-08-29T00:26:07.897707Z","wall_seconds":10.081080500036478} +{"artifact_location":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-k/k1/uk-2024-locals-rowwise-pilot-k1.h5","build_id":"uk-local-rowwise-ladder-f100-s42-20260830T181120Z-3e7bd722","code_pin":"5422de7a00391e961239bc9258c0994ca144d8ee","cost_usd":null,"disposition":"iterating","gate_verdicts":{"uk_geography_ladder":{"receipt":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-k/k1/rowwise_build_manifest.json#/rowwise_dataset/gate","verdict":"passed"}},"identity_digest":"0f474ce0fbfc525a8bffac6e8ca1080bde75aaab7f269ee56660a664533447cc","input_pins_digest":"8d4c069b1350cdda9b0abfd63ab05b9d1bd7294fcc4273dd88434a96f46731c2","phases_reached":["attempt_started","configured","inputs_pinned","cloned","manifest_written"],"pipeline":"uk-local-rowwise","prediction_id":null,"prev_row_digest":"a4d2c0b2b356493cefb30aca6aa35ba33cfeff584b6946e09164a1cadff4c2de","row_digest":"15f6b09944f749cd2954a38daa78f52c9a24a9f61bbb9ab768af5b4133d22fcc","rung":"f100","seed":42,"ts":"2026-08-30T18:11:20.735779Z","wall_seconds":6.7020839168690145} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 4d0ba469e7..a43624a88a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -144,9 +144,11 @@ UK_OA_LADDER_SCHEMA_VERSION, UkOaLadder, assign_uk_geography_ladder, + expected_uk_ladder_area_support, load_uk_oa_ladder, uk_geography_ladder_assignment_summary, uk_geography_ladder_gate, + uk_region_mix, ) from microcosm.build.uk_runtime.geography_sources import ( ENGLAND_LAD_REGION_URL, @@ -275,6 +277,8 @@ rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, + uk_ladder_area_support_summary, ) from microcosm.build.uk_runtime.local_target_census import ( CENSUS_KIND, @@ -723,6 +727,7 @@ "build_uk_rowwise_local_matrix", "clone_entity_frame", "clone_uk_dataset_tables_with_ladder_geography", + "expected_uk_ladder_area_support", "clone_uk_dataset_tables_with_rowwise_geography", "clone_uk_dataset_with_ladder_geography", "clone_uk_dataset_with_rowwise_geography", @@ -817,6 +822,9 @@ "uk_legacy_jsa_policy", "uk_geography_ladder_assignment_summary", "uk_geography_ladder_gate", + "uk_region_mix", + "uk_area_support_summary", + "uk_ladder_area_support_summary", "uk_release_input_coverage_gate", "uk_release_input_coverage_required_columns", "uk_release_input_coverage_reviewed_exclusions", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index 024d6b4eaf..b5015c4b17 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -365,18 +365,11 @@ def assign_uk_geography_ladder( f"({expected_constituency_vintage!r})." ) - region_codes = _household_region_codes( - household[region_column], label=region_column + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, ) - ladder_regions = set(np.unique(ladder.region_code).tolist()) - missing_regions = sorted(set(region_codes.tolist()) - ladder_regions) - if missing_regions: - raise ValueError( - "UK OA ladder has no output areas for household region(s): " - f"{missing_regions}. The household regions and the ladder artifact " - "must share coverage (Scotland and Northern Ireland rungs are not " - "in an England-&-Wales ladder)." - ) assigned_index = _sample_oa_indices( region_codes.to_numpy(), @@ -419,6 +412,128 @@ def assign_uk_geography_ladder( return assigned +def expected_uk_ladder_area_support( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + n_clones: int = 1, + region_column: str = "region", +) -> pd.DataFrame: + """Return analytic expected rows per constituency and local authority. + + The expectation follows the assignment sampler's two stages exactly: + constituency household-count shares within each input region, followed by + OA population shares within each chosen constituency. Every constituency + and local authority in the ladder is returned, including zero-support + areas. + """ + + if isinstance(n_clones, bool) or not isinstance(n_clones, int) or n_clones <= 0: + raise ValueError("n_clones must be a positive integer.") + + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, + ) + constituency_expected = { + str(code): 0.0 for code in np.unique(ladder.constituency_code).tolist() + } + local_authority_expected = { + str(code): 0.0 for code in np.unique(ladder.local_authority_code).tolist() + } + frame = _ladder_sampling_frame(ladder) + region_counts = region_codes.value_counts(sort=False).sort_index() + + for region_code, n_region in region_counts.items(): + region_rows, constituency_weight = _constituency_household_weights( + frame, + region_code=str(region_code), + ) + region_households = float(constituency_weight.sum()) + for constituency_code, household_count in constituency_weight.items(): + expected_rows = ( + n_clones + * int(n_region) + * float(household_count) + / region_households + ) + constituency_key = str(constituency_code) + constituency_expected[constituency_key] += expected_rows + if expected_rows == 0.0: + continue + + oa_rows, oa_weights = _constituency_oa_population_weights( + region_rows, + region_code=str(region_code), + constituency_code=constituency_key, + ) + constituency_population = float(oa_weights.sum()) + local_authority_population = oa_rows.groupby( + "local_authority_code", sort=True + )["population"].sum() + for local_authority_code, population in local_authority_population.items(): + local_authority_expected[str(local_authority_code)] += ( + expected_rows * float(population) / constituency_population + ) + + rows = [ + { + "area_type": area_type, + "area_code": area_code, + "expected_rows": expected_rows, + } + for area_type, expected_by_area in ( + ("constituency", constituency_expected), + ("la", local_authority_expected), + ) + for area_code, expected_rows in expected_by_area.items() + ] + return pd.DataFrame( + rows, + columns=("area_type", "area_code", "expected_rows"), + ) + + +def uk_region_mix( + household: pd.DataFrame, + *, + region_column: str = "region", + weight_column: str = "household_weight", +) -> pd.DataFrame: + """Summarize household row and weight shares by normalized UK region.""" + + if region_column not in household.columns: + raise ValueError(f"household table must contain region column {region_column!r}.") + if weight_column not in household.columns: + raise ValueError(f"household table must contain weight column {weight_column!r}.") + + region_codes = _household_region_codes( + household[region_column], + label=region_column, + ) + weights = pd.to_numeric(household[weight_column], errors="raise").to_numpy( + dtype=np.float64 + ) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError(f"{weight_column} must be finite and non-negative.") + if len(weights) == 0: + raise ValueError("household table must contain at least one row for region mix.") + total_weight = float(weights.sum()) + if total_weight <= 0: + raise ValueError(f"{weight_column} must carry positive total weight.") + + frame = pd.DataFrame({"region_code": region_codes, "weight": weights}) + mix = ( + frame.groupby("region_code", sort=True) + .agg(rows=("weight", "size"), weight_sum=("weight", "sum")) + .reset_index() + ) + mix["row_share"] = mix["rows"] / len(frame) + mix["weight_share"] = mix["weight_sum"] / total_weight + return mix[["region_code", "rows", "row_share", "weight_share"]] + + def uk_geography_ladder_assignment_summary( household: pd.DataFrame, ladder: UkOaLadder, @@ -659,32 +774,18 @@ def _sample_oa_indices( n = len(region_codes) assigned_index = np.full(n, -1, dtype=np.int64) - frame = pd.DataFrame( - { - "constituency_code": ladder.constituency_code, - "region_code": ladder.region_code, - "population": ladder.population, - "households": ladder.households, - "ladder_index": np.arange(len(ladder), dtype=np.int64), - } - ) + frame = _ladder_sampling_frame(ladder) household_positions = pd.Series(np.arange(n, dtype=np.int64)) for region_code, region_positions in household_positions.groupby( pd.Series(region_codes), sort=True ): - region_rows = frame[frame["region_code"] == region_code] - # Stage one: constituency household-count weights within the region. - constituency_weight = region_rows.groupby("constituency_code", sort=True)[ - "households" - ].sum() + region_rows, constituency_weight = _constituency_household_weights( + frame, + region_code=str(region_code), + ) constituencies = constituency_weight.index.to_numpy() weights = constituency_weight.to_numpy(dtype=np.float64) - if weights.sum() <= 0: - raise ValueError( - f"region {region_code!r} has zero household weight for the " - "constituency draw." - ) positions = region_positions.to_numpy() chosen_constituency = rng.choice( constituencies, @@ -697,9 +798,12 @@ def _sample_oa_indices( for constituency_code, local_positions in pd.Series(positions).groupby( chosen_series, sort=True ): - oa_rows = region_rows[region_rows["constituency_code"] == constituency_code] + oa_rows, oa_weights = _constituency_oa_population_weights( + region_rows, + region_code=str(region_code), + constituency_code=str(constituency_code), + ) oa_indices = oa_rows["ladder_index"].to_numpy() - oa_weights = oa_rows["population"].to_numpy(dtype=np.float64) drawn = rng.choice( oa_indices, size=len(local_positions), @@ -713,6 +817,58 @@ def _sample_oa_indices( return assigned_index +def _ladder_sampling_frame(ladder: UkOaLadder) -> pd.DataFrame: + """Aligned ladder columns used by both sampling and its expectation.""" + + return pd.DataFrame( + { + "constituency_code": ladder.constituency_code, + "region_code": ladder.region_code, + "local_authority_code": ladder.local_authority_code, + "population": ladder.population, + "households": ladder.households, + "ladder_index": np.arange(len(ladder), dtype=np.int64), + } + ) + + +def _constituency_household_weights( + frame: pd.DataFrame, + *, + region_code: str, +) -> tuple[pd.DataFrame, pd.Series]: + """Stage-one constituency weights for one household region.""" + + region_rows = frame[frame["region_code"] == region_code] + constituency_weight = region_rows.groupby("constituency_code", sort=True)[ + "households" + ].sum() + if constituency_weight.sum() <= 0: + raise ValueError( + f"region {region_code!r} has zero household weight for the " + "constituency draw." + ) + return region_rows, constituency_weight + + +def _constituency_oa_population_weights( + region_rows: pd.DataFrame, + *, + region_code: str, + constituency_code: str, +) -> tuple[pd.DataFrame, np.ndarray]: + """Stage-two OA weights for one region-constituency pair.""" + + oa_rows = region_rows[region_rows["constituency_code"] == constituency_code] + oa_weights = oa_rows["population"].to_numpy(dtype=np.float64) + if oa_weights.sum() <= 0: + raise ValueError( + f"constituency {constituency_code!r} in region {region_code!r} has " + "zero population weight for the OA draw." + ) + return oa_rows, oa_weights + + def _metadata_from_scalar(value: np.ndarray) -> dict[str, Any]: if value.shape != (): raise ValueError("metadata_json must be a scalar JSON string.") @@ -841,6 +997,34 @@ def _household_region_codes(values: Any, *, label: str) -> pd.Series: return mapped +def _validated_household_ladder_region_codes( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + region_column: str, +) -> pd.Series: + """Normalize household regions and require matching ladder coverage.""" + + if region_column not in household.columns: + raise ValueError( + f"household table must contain {region_column!r} before geography-" + "ladder assignment (assign regions first)." + ) + region_codes = _household_region_codes( + household[region_column], label=region_column + ) + ladder_regions = set(np.unique(ladder.region_code).tolist()) + missing_regions = sorted(set(region_codes.tolist()) - ladder_regions) + if missing_regions: + raise ValueError( + "UK OA ladder has no output areas for household region(s): " + f"{missing_regions}. The household regions and the ladder artifact " + "must share coverage (Scotland and Northern Ireland rungs are not " + "in an England-&-Wales ladder)." + ) + return region_codes + + def _region_code_for_value(value: Any) -> str | None: text = str(value).strip() if not text: @@ -874,7 +1058,9 @@ def _region_code_for_value(value: Any) -> str | None: "UK_OA_LADDER_SCHEMA_VERSION", "UkOaLadder", "assign_uk_geography_ladder", + "expected_uk_ladder_area_support", "load_uk_oa_ladder", + "uk_region_mix", "uk_geography_ladder_assignment_summary", "uk_geography_ladder_gate", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index c25e285243..fa0b5a52e5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -62,6 +62,8 @@ "build_uk_rowwise_local_matrix", "past_cap_census", "require_adjudicated_uk_local_binding", + "uk_area_support_summary", + "uk_ladder_area_support_summary", "rowwise_calibration_mass_reason", "rowwise_area_support_summary", "solve_uk_rowwise_weights_under_doctrine", @@ -940,50 +942,154 @@ def rowwise_area_support_summary( ) -> pd.DataFrame: """Per-area support of a rowwise weight vector, all target areas included.""" + return uk_area_support_summary( + problem.assigned_areas, + weights, + area_codes=problem.area_codes, + source_household_ids=( + problem.household_ids + if source_household_ids is None + else source_household_ids + ), + ) + + +def uk_area_support_summary( + assigned_areas: Sequence[Any], + weights: Sequence[float], + *, + area_codes: Sequence[Any], + source_household_ids: Sequence[Any], +) -> pd.DataFrame: + """Summarize row and weighted support for a declared roster of areas.""" + values = np.asarray(weights, dtype=np.float64) if values.ndim != 1: raise ValueError(f"weights must be one-dimensional, got shape {values.shape}.") - if len(values) != problem.n_households: + assigned = np.asarray(assigned_areas, dtype=object) + if assigned.ndim != 1: + raise ValueError( + f"assigned_areas must be one-dimensional, got shape {assigned.shape}." + ) + if len(values) != len(assigned): raise ValueError( "weights must align with households, got " - f"{len(values)} weights for {problem.n_households} households." + f"{len(values)} weights for {len(assigned)} households." ) if not np.isfinite(values).all() or (values < 0).any(): raise ValueError("weights must be finite and non-negative.") - sources_list = ( - list(problem.household_ids) - if source_household_ids is None - else list(source_household_ids) - ) - if len(sources_list) != problem.n_households: + sources_list = list(source_household_ids) + if len(sources_list) != len(assigned): raise ValueError( "source_household_ids must align with households, got " - f"{len(sources_list)} for {problem.n_households}." + f"{len(sources_list)} for {len(assigned)}." ) - sources = np.empty(problem.n_households, dtype=object) + sources = np.empty(len(assigned), dtype=object) for index, source in enumerate(sources_list): sources[index] = source - assigned = np.asarray(problem.assigned_areas, dtype=object) - rows: list[dict[str, Any]] = [] - for area_code in problem.area_codes: - members = np.flatnonzero(assigned == area_code) - member_weights = values[members] - positive = member_weights > 0 - weight_sum = float(member_weights.sum()) - square_sum = float(np.square(member_weights).sum()) - rows.append( - { - "area_code": area_code, - "assigned_households": int(len(members)), - "nonzero_households": int(positive.sum()), - "nonzero_source_households": int( - len(set(sources[members][positive].tolist())) - ), - "weight_sum": weight_sum, - "max_weight": (float(member_weights.max()) if len(members) else 0.0), - "effective_sample_size": ( - weight_sum**2 / square_sum if square_sum > 0 else 0.0 - ), - } - ) - return pd.DataFrame(rows) + area_roster = np.asarray(area_codes, dtype=object) + if area_roster.ndim != 1: + raise ValueError( + f"area_codes must be one-dimensional, got shape {area_roster.shape}." + ) + + source_codes, _ = pd.factorize( + pd.Series(sources, dtype=object), + sort=False, + use_na_sentinel=False, + ) + positive = values > 0 + rows = pd.DataFrame( + { + "area_code": assigned, + "weight": values, + "weight_squared": np.square(values), + "positive": positive, + "positive_source_code": np.where(positive, source_codes, -1), + } + ) + grouped = rows.groupby("area_code", sort=False, dropna=False).agg( + assigned_households=("weight", "size"), + nonzero_households=("positive", "sum"), + nonzero_source_households=( + "positive_source_code", + lambda codes: int(codes[codes >= 0].nunique()), + ), + weight_sum=("weight", "sum"), + max_weight=("weight", "max"), + weight_square_sum=("weight_squared", "sum"), + ) + support = pd.DataFrame({"area_code": area_roster}).merge( + grouped, + how="left", + left_on="area_code", + right_index=True, + sort=False, + ) + count_columns = ( + "assigned_households", + "nonzero_households", + "nonzero_source_households", + ) + support[list(count_columns)] = ( + support[list(count_columns)].fillna(0).astype(np.int64) + ) + support[["weight_sum", "max_weight", "weight_square_sum"]] = support[ + ["weight_sum", "max_weight", "weight_square_sum"] + ].fillna(0.0) + effective_sample_size = np.zeros(len(support), dtype=np.float64) + np.divide( + np.square(support["weight_sum"].to_numpy(dtype=np.float64)), + support["weight_square_sum"].to_numpy(dtype=np.float64), + out=effective_sample_size, + where=support["weight_square_sum"].to_numpy(dtype=np.float64) > 0, + ) + support["effective_sample_size"] = effective_sample_size + return support[ + [ + "area_code", + "assigned_households", + "nonzero_households", + "nonzero_source_households", + "weight_sum", + "max_weight", + "effective_sample_size", + ] + ] + + +def uk_ladder_area_support_summary( + household: pd.DataFrame, + ladder: Any, + *, + weight_column: str = "household_weight", + source_column: str = "source_household_id", +) -> dict[str, pd.DataFrame]: + """Return matrix-free constituency and LA support on the ladder roster.""" + + if source_column not in household.columns: + raise ValueError( + f"household table must contain source column {source_column!r}; " + "distinct-source honesty requires it (pass source_column=" + "'household_id' explicitly for row-grain sources)." + ) + if weight_column not in household.columns: + raise ValueError(f"household table must contain weight column {weight_column!r}.") + + summaries: dict[str, pd.DataFrame] = {} + for area_type, assigned_column, ladder_codes in ( + ("constituency", "constituency_code", ladder.constituency_code), + ("la", "local_authority_code", ladder.local_authority_code), + ): + if assigned_column not in household.columns: + raise ValueError( + f"household table must contain assigned area column " + f"{assigned_column!r}." + ) + summaries[area_type] = uk_area_support_summary( + household[assigned_column], + household[weight_column], + area_codes=np.unique(ladder_codes), + source_household_ids=household[source_column], + ) + return summaries diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 86fb5544d0..86f67c7026 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -51,6 +51,17 @@ #: staging H5 already carries. POOL_SOURCE_LINEAGE_COLUMN = "pool_source_household_id" +#: Explicit lineage carried by accepted UK spine inputs. These columns are +#: authoritative for the spine id scheme and make the pool-only modulus rule +#: inapplicable. +UK_SPINE_LINEAGE_COLUMNS = ( + "household_support_channel", + "household_support_clone_index", + "household_is_spi_synthetic", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", +) + PERSON_ID_COLUMNS = ( "person_id", "person_household_id", @@ -76,6 +87,29 @@ def ladder_clone_index_column(entity: str) -> str: return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" +def _refuse_preassigned_geography( + table: pd.DataFrame, + *, + label: str, +) -> None: + preassigned_columns = [ + column + for column in ( + *UK_GEOGRAPHY_LADDER_COLUMNS, + "la_code_oa", + "constituency_code_oa", + "region_code_oa", + ) + if column in table.columns + ] + if preassigned_columns: + raise ValueError( + f"{label} carries pre-assigned geography column(s) " + f"{preassigned_columns!r}; pre-assigned geography cannot be " + "silently overwritten and stale *_oa columns cannot ride through." + ) + + @dataclass(frozen=True) class UKRowwiseDatasetResult: """Cloned UK single-year tables and row-wise geography metadata. @@ -296,6 +330,7 @@ def clone_uk_dataset_tables_with_ladder_geography( household_id_column="household_id", household_weight_column="household_weight", ) + _refuse_preassigned_geography(household_frame, label="household") # The artifact's clone_index names the rowwise clone dimension. A # candidate-tier clone_index inherited from the staging input (SPI/pool # lineage) is replaced, exactly as the pre-Frame clone overwrote the @@ -892,15 +927,15 @@ def apply_uk_source_lineage_modulus( ) -> pd.DataFrame: """Derive pool-grain lineage as ``pool_source_household_id``. - The certified UK pool encodes its 10x clone tiers as - ``household_id = tier * 10**8 + base``, so a modulus of ``10**8`` recovers - the enhanced-FRS pool source. The derived column is a *distinct lineage - layer*: any immediate-layer ``source_household_id``/``source_household_key`` - the input carries (the national staging H5 does) is left untouched. The - modulus is only meaningful for rows whose ids follow the pool's tier - scheme — on a seam output, channel-rebuilt households (e.g. the rebuilt - SPI channel) carry ids outside that scheme, so pool-lineage diagnostics - must be read per support channel. + The pool scheme is ``household_id = tier * 10**8 + base``; a modulus of + ``10**8`` recovers the enhanced-FRS pool source as a *distinct lineage + layer*, leaving any immediate-layer ``source_household_id`` or + ``source_household_key`` untouched. + + Spine inputs use a different scheme: ``household_id = sernum + + 1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * 10**(d+2)``. + Their explicit spine lineage columns are the source of truth, so applying + the pool modulus would mint a false second lineage layer and is refused. The mapping is refused when it would be ambiguous (the pool column already exists) or vacuous (no id reaches the modulus, making it the @@ -915,6 +950,17 @@ def apply_uk_source_lineage_modulus( f"household already carries {POOL_SOURCE_LINEAGE_COLUMN!r}; " "applying source_lineage_modulus would be ambiguous." ) + spine_lineage_columns = [ + column for column in UK_SPINE_LINEAGE_COLUMNS if column in household.columns + ] + if spine_lineage_columns: + raise ValueError( + "household carries explicit spine lineage column(s) " + f"{spine_lineage_columns!r}; explicit spine lineage columns are " + "the source of truth. The source_lineage_modulus rule was written " + "for the pool's tier·10^8+base scheme and would mint a false " + "second lineage layer." + ) numeric = pd.to_numeric(household[household_id_column], errors="raise").to_numpy( dtype=np.float64 ) diff --git a/packages/microcosm-build/tests/test_logbook_adoption.py b/packages/microcosm-build/tests/test_logbook_adoption.py index 0432d5879e..e0bafe7db5 100644 --- a/packages/microcosm-build/tests/test_logbook_adoption.py +++ b/packages/microcosm-build/tests/test_logbook_adoption.py @@ -189,7 +189,7 @@ def test_error_receipt_and_placeholder_drop_only_when_sole_key(tmp_path: Path) - path = write_error_receipt( error_receipt_path(tmp_path, build_id=state.build_id), state=state, - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", error=RuntimeError("boom"), ) apply_error_verdict( @@ -205,7 +205,7 @@ def test_error_receipt_and_placeholder_drop_only_when_sole_key(tmp_path: Path) - } assert state.phases_reached == ["attempt_started", "error"] payload = json.loads(path.read_text(encoding="utf-8")) - assert payload["pipeline"] == "uk-locals-rowwise" + assert payload["pipeline"] == "uk-local-rowwise" assert payload["error_type"] == "builtins.RuntimeError" assert payload["message"] == "boom" @@ -240,7 +240,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec started_at=1.0, now=lambda: 3.5, started_ts=datetime(2026, 8, 13, 12, 0, tzinfo=UTC), - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", rung="f100", seed=42, code_pin="a" * 40, @@ -253,7 +253,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec row = load_logbook_row(spool_path) assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS assert row.build_id == "uk-fixture-attempt" - assert row.pipeline == "uk-locals-rowwise" + assert row.pipeline == "uk-local-rowwise" assert row.phases_reached == ("attempt_started", "configured") assert row.wall_seconds == 2.5 @@ -263,7 +263,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec started_at=1.0, now=lambda: 4.0, started_ts=datetime(2026, 8, 13, 12, 0, tzinfo=UTC), - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", rung="f100", seed=42, code_pin="a" * 40, diff --git a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py index 32bddc801c..1230c84cd9 100644 --- a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py +++ b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py @@ -30,7 +30,10 @@ ROOT = Path(__file__).resolve().parents[3] MIGRATIONS = ROOT / "supabase/migrations" -NEW_MIGRATION = MIGRATIONS / "20260818000000_logbook_chain_scopes.sql" +CHAIN_SCOPES_MIGRATION = MIGRATIONS / "20260818000000_logbook_chain_scopes.sql" +UK_LOCAL_SCOPE_MIGRATION = ( + MIGRATIONS / "20260829000000_logbook_uk_local_scope.sql" +) ROWS = ROOT / "logbook/us.jsonl" BASE_MIGRATIONS = [ "20260805000000_logbook.sql", @@ -172,7 +175,8 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None _insert(connection, row) assert _count_builds(connection) == 28 - _apply_sql(connection, NEW_MIGRATION.read_text(encoding="utf-8")) + _apply_sql(connection, CHAIN_SCOPES_MIGRATION.read_text(encoding="utf-8")) + _apply_sql(connection, UK_LOCAL_SCOPE_MIGRATION.read_text(encoding="utf-8")) us_tail = _digest_of(connection, archived[-1]["build_id"]) _insert( @@ -198,17 +202,14 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None predecessor=None, ), ) - # Vocabulary is closed-world and minimal: uk/locals and uk/firms both - # derive cleanly but are not ratified yet, so even their geneses are - # refused until a migration adds them (locals rows keep spooling locally - # in the meantime — recording never consults the allowlist). Cross-scope + # Vocabulary is closed-world and minimal: uk/local is ratified for #761, + # while uk/firms still derives cleanly but remains unratified. Cross-scope # independence is already proven above: uk/frs opened while us had rows. - _refuses( + _insert( connection, _build_row( - "uk-locals-genesis", pipeline="uk-locals-rowwise", predecessor=None + "uk-local-genesis", pipeline="uk-local-rowwise", predecessor=None ), - "not in the ratified scope list", ) _refuses( connection, diff --git a/packages/microcosm-build/tests/test_logbook_cli.py b/packages/microcosm-build/tests/test_logbook_cli.py index 15c3a56f2d..64444560f8 100644 --- a/packages/microcosm-build/tests/test_logbook_cli.py +++ b/packages/microcosm-build/tests/test_logbook_cli.py @@ -112,13 +112,13 @@ def test_cli_defaults_to_the_ratified_root_paths() -> None: def test_committed_archives_are_scoped_by_country() -> None: - # One chain per country: the US pool lineage and the UK migration - # chain never share an archive (microcosm#665). + # One chain per scope: the US pool lineage, and each UK line under its + # ratified scope path (microcosm#665; uk/local opened by #761). archives = sorted( path.relative_to(ROOT / "logbook").as_posix() for path in (ROOT / "logbook").rglob("*.jsonl") ) - assert archives == ["us.jsonl"] + assert archives == ["uk/local.jsonl", "us.jsonl"] assert (ROOT / "logbook" / "README.md").is_file() @@ -246,14 +246,14 @@ def test_cli_local_export_refuses_wrong_scope_rows( capsys: pytest.CaptureFixture[str], ) -> None: # The adversarial finding on this PR: the chain verifier authenticates - # payloads, never filenames, so without this gate a uk-locals spool + # payloads, never filenames, so without this gate a uk-local spool # exported into the uk/frs archive would chain validly and # permanently mis-scope lineage. Local exports get the same scope # discipline as the remote branch. archive = tmp_path / "logbook" / "uk" / "frs.jsonl" archive.parent.mkdir(parents=True) source = tmp_path / "source.jsonl" - _write_jsonl(source, _chain(pipeline="uk-locals-rowwise")) + _write_jsonl(source, _chain(pipeline="uk-local-rowwise")) cli = _load_cli() exit_code = cli.main( @@ -263,7 +263,7 @@ def test_cli_local_export_refuses_wrong_scope_rows( assert exit_code == 1 err = capsys.readouterr().err assert "outside scope uk/frs" in err - assert "uk-locals-rowwise" in err + assert "uk-local-rowwise" in err assert not archive.exists() @@ -289,6 +289,26 @@ def test_cli_export_refuses_an_unratified_scope_archive( assert not archive.exists() +def test_cli_export_accepts_uk_local_scope_archive( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + archive = tmp_path / "logbook" / "uk" / "local.jsonl" + archive.parent.mkdir(parents=True) + source = tmp_path / "source.jsonl" + rows = _chain(pipeline="uk-local-rowwise") + _write_jsonl(source, rows) + cli = _load_cli() + + exit_code = cli.main( + ["export", "--archive", str(archive), "--source", str(source)] + ) + + assert exit_code == 0 + assert "exported 3 new Logbook rows" in capsys.readouterr().out + assert load_logbook_file(archive) == rows + + def test_cli_export_chain_orders_a_spool_directory( tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -557,7 +577,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - rows = _chain(pipeline="uk-locals-rowwise") + rows = _chain(pipeline="uk-local-rowwise") archive = tmp_path / "logbook" / "uk" / "frs.jsonl" cli = _load_cli() monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") @@ -574,7 +594,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( assert result == 1 err = capsys.readouterr().err assert "outside scope uk/frs" in err - assert "uk-locals-rowwise" in err + assert "uk-local-rowwise" in err @pytest.mark.parametrize( @@ -584,7 +604,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( ("us-pool-inc2", "us"), ("us-stacked-pool", "us"), ("uk-frs-staging", "uk/frs"), - ("uk-locals-rowwise", "uk/locals"), + ("uk-local-rowwise", "uk/local"), ("us-pool-inc3", "us/pool"), ("mystery-pipeline", None), ], diff --git a/packages/microcosm-build/tests/test_uk_geography_ladder.py b/packages/microcosm-build/tests/test_uk_geography_ladder.py index 759d5cfb0c..5131be258b 100644 --- a/packages/microcosm-build/tests/test_uk_geography_ladder.py +++ b/packages/microcosm-build/tests/test_uk_geography_ladder.py @@ -21,10 +21,12 @@ UK_GEOGRAPHY_LADDER_COLUMNS, assemble_uk_oa_ladder, assign_uk_geography_ladder, + expected_uk_ladder_area_support, join_uk_oa_ladder_layers, load_uk_oa_ladder, uk_geography_ladder_assignment_summary, uk_geography_ladder_gate, + uk_region_mix, ) @@ -269,6 +271,121 @@ def test_assignment_is_deterministic_and_two_stage_weighted(tmp_path) -> None: assert dominant > 0.95 +def test_expected_support_matches_hand_computed_two_stage_result(tmp_path) -> None: + ladder = load_uk_oa_ladder( + _write_ladder( + tmp_path / "ladder.npz", + local_authority_code=np.asarray( + ["E09000001", "E09000002", "E09000002", "W06000001"] + ), + ) + ) + household = pd.DataFrame( + {"household_id": [1, 2], "region": ["LONDON", "London"]} + ) + + support = expected_uk_ladder_area_support(household, ladder, n_clones=2) + + assert support.columns.tolist() == ["area_type", "area_code", "expected_rows"] + assert set(support["area_type"]) == {"constituency", "la"} + expected = { + (row.area_type, row.area_code): row.expected_rows + for row in support.itertuples(index=False) + } + assert expected == pytest.approx( + { + ("constituency", "E14000001"): 1.0, + ("constituency", "E14000002"): 3.0, + ("constituency", "W07000041"): 0.0, + ("la", "E09000001"): 0.99, + ("la", "E09000002"): 3.01, + ("la", "W06000001"): 0.0, + } + ) + + +@pytest.mark.parametrize("n_clones", [0, -1, True, 1.5, "2"]) +def test_expected_support_requires_positive_integer_n_clones( + tmp_path, n_clones +) -> None: + ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) + + with pytest.raises(ValueError, match="n_clones must be a positive integer"): + expected_uk_ladder_area_support( + _household(), + ladder, + n_clones=n_clones, + ) + + +def test_expected_support_matches_assignment_uncovered_region_refusal(tmp_path) -> None: + ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) + household = pd.DataFrame({"household_id": [1], "region": ["SCOTLAND"]}) + + with pytest.raises(ValueError) as assignment_error: + assign_uk_geography_ladder(household, ladder) + with pytest.raises(ValueError) as expectation_error: + expected_uk_ladder_area_support(household, ladder) + + assert str(expectation_error.value) == str(assignment_error.value) + + +def test_expected_support_agrees_with_large_realized_assignment(tmp_path) -> None: + ladder = load_uk_oa_ladder( + _write_ladder( + tmp_path / "ladder.npz", + population=np.asarray([300.0, 100.0, 300.0, 400.0]), + local_authority_code=np.asarray( + ["E09000001", "E09000002", "E09000002", "W06000001"] + ), + ) + ) + household = pd.DataFrame( + {"household_id": range(20_000), "region": ["LONDON"] * 20_000} + ) + + expected = expected_uk_ladder_area_support(household, ladder) + assigned = assign_uk_geography_ladder(household, ladder, seed=91) + actual = { + (area_type, code): float(count) + for area_type, column in ( + ("constituency", "constituency_code"), + ("la", "local_authority_code"), + ) + for code, count in assigned[column].value_counts().items() + } + + for row in expected[expected["expected_rows"] > 0].itertuples(index=False): + assert actual[(row.area_type, row.area_code)] == pytest.approx( + row.expected_rows, + rel=0.03, + ) + + +def test_region_mix_row_and_weight_shares_sum_to_one() -> None: + household = pd.DataFrame( + { + "region": ["LONDON", "London", "WALES"], + "household_weight": [1.0, 2.0, 7.0], + } + ) + + mix = uk_region_mix(household) + + assert mix.columns.tolist() == [ + "region_code", + "rows", + "row_share", + "weight_share", + ] + assert mix["row_share"].sum() == pytest.approx(1.0) + assert mix["weight_share"].sum() == pytest.approx(1.0) + london = mix.set_index("region_code").loc["E12000007"] + assert london["rows"] == 2 + assert london["row_share"] == pytest.approx(2 / 3) + assert london["weight_share"] == pytest.approx(0.3) + + def test_assignment_requires_region(tmp_path) -> None: ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) household = pd.DataFrame({"household_id": [1]}) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 1144019452..6f7934c744 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -97,6 +97,17 @@ def _household_frame() -> pd.DataFrame: ) +def _spine_household_frame() -> pd.DataFrame: + return _household_frame().assign( + source_household_id=[10, 20, 10, 30], + household_support_channel=["frs", "spi", "frs", "spi"], + household_support_clone_index=[0, 1, 0, 0], + household_is_spi_synthetic=[False, True, False, True], + household_is_capital_gains_clone=[False, False, True, False], + household_is_cgt_band_donor=[False, False, False, True], + ) + + def _person_frame() -> pd.DataFrame: return pd.DataFrame( { @@ -250,6 +261,19 @@ def test_ladder_clone_refuses_vintage_mismatch(toy_ladder) -> None: ) +def test_ladder_clone_refuses_preassigned_geography(toy_ladder) -> None: + ladder, _ = toy_ladder + preassigned = _seam_frame( + household=_household_frame().assign(oa_code="stale-oa"), + ) + + with pytest.raises( + ValueError, + match="pre-assigned geography cannot be silently overwritten", + ): + clone_uk_dataset_with_ladder_geography(preassigned, ladder, n_clones=1) + + def test_ladder_clone_refuses_uncovered_region(tmp_path) -> None: frame = _ladder_frame() england_only = frame[frame["region_code"].str.startswith("E")] @@ -348,14 +372,14 @@ def _load_builder_module(): return module -def _write_seam_h5(path) -> None: +def _write_seam_h5(path, *, household: pd.DataFrame | None = None) -> None: from microcosm.build.uk_runtime import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_national_frame dataset = uk_national_frame( person=_person_frame(), benunit=_benunit_frame(), - household=_household_frame(), + household=_household_frame() if household is None else household, time_period="2023", weight_kind=WeightKind.IMPORTANCE, mass_log=( @@ -377,7 +401,7 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) import json import sys - _, ladder_path = toy_ladder + ladder, ladder_path = toy_ladder builder = _load_builder_module() input_h5 = tmp_path / "staging.h5" _write_seam_h5(input_h5) @@ -404,12 +428,61 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) assert manifest["parameters"]["assignment_route"] == "ladder" assert manifest["inputs"]["ladder"]["sha256"] + assert manifest["inputs"]["dataset"]["pin_verified"] is False + assert manifest["inputs"]["ladder"]["pin_verified"] is False + assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False summary = manifest["rowwise_dataset"] assert summary["gate"]["passed"] is True assert summary["missing_geography_rows"] == 0 assert summary["assigned_constituencies"] >= 4 assert summary["weights"]["household_weight_kind"] == "importance" assert summary["weights"]["mass_conservation"]["passed"] is True + assert summary["source_lineage"]["explicit"] is None + assert summary["area_support"]["source_basis"] == "source_household_id" + assert summary["area_support"]["constituency"]["n_areas"] == len( + np.unique(ladder.constituency_code) + ) + assert summary["area_support"]["la"]["n_areas"] == len( + np.unique(ladder.local_authority_code) + ) + assert set(summary["area_support"]["constituency"]) == { + "n_areas", + "rows_basis", + "min_rows", + "median_rows", + "min_ess", + "median_ess", + "min_distinct_sources", + "median_distinct_sources", + "bottom_by_rows", + "bottom_by_ess", + } + assert sum(row["row_share"] for row in summary["region_mix"]) == pytest.approx( + 1.0 + ) + assert sum( + row["weight_share"] for row in summary["region_mix"] + ) == pytest.approx(1.0) + area_support_path = output_dir / builder.AREA_SUPPORT_FILENAME + assert area_support_path.exists() + assert manifest["outputs"]["area_support_summary"]["path"] == str( + area_support_path + ) + area_support = pd.read_csv(area_support_path) + assert area_support.columns.tolist() == [ + "area_type", + "area_code", + "assigned_households", + "nonzero_households", + "nonzero_source_households", + "weight_sum", + "max_weight", + "effective_sample_size", + ] + assert area_support["area_type"].tolist() == [ + *(["constituency"] * len(np.unique(ladder.constituency_code))), + *(["la"] * len(np.unique(ladder.local_authority_code))), + ] assert (output_dir / "staging_rowwise.h5").exists() @@ -427,13 +500,19 @@ def test_driver_ladder_dry_run_matches_real_assignment( _write_seam_h5(input_h5) plan_dir = tmp_path / "plan" build_dir = tmp_path / "build" + input_sha256 = builder._sha256(input_h5) + ladder_sha256 = builder._sha256(ladder_path) base_argv = [ "build_uk_rowwise_dataset.py", "--input-h5", str(input_h5), + "--input-sha256", + input_sha256, "--ladder", str(ladder_path), + "--ladder-sha256", + ladder_sha256, "--n-clones", "2", "--seed", @@ -443,10 +522,35 @@ def test_driver_ladder_dry_run_matches_real_assignment( assert builder.main() == 0 plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) assert not (plan_dir / "staging_rowwise.h5").exists() + expected_support = plan["expected_support"] + assert expected_support["basis"] == ( + "analytic expectation: constituency household-count share within region x " + "the input's region mix x n_clones; OA population shares within " + "constituency for LA support" + ) + for area_type in ("constituency", "la"): + area_support = expected_support[area_type] + # Same-named stats keys carry their semantics: an expected-mass + # figure must never be silently compared against a realized count. + assert area_support["rows_basis"] == "expected_rows" + assert plan["realized_support"][area_type]["rows_basis"] == "assigned_rows" + assert area_support["n_areas"] <= builder.EXPECTED_SUPPORT_BOTTOM_AREAS + assert len(area_support["bottom"]) == area_support["n_areas"] + assert sum(row["rows"] for row in area_support["bottom"]) == pytest.approx( + plan["plan"]["rows"]["household"] + ) monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) assert builder.main() == 0 manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) + assert plan["input"]["dataset"]["pin_verified"] is True + assert plan["input"]["ladder"]["pin_verified"] is True + assert plan["input"]["ladder"]["matches_local_area_crosswalk_pin"] is False + assert manifest["inputs"]["dataset"]["pin_verified"] is True + assert manifest["inputs"]["ladder"]["pin_verified"] is True + assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False + assert plan["area_support"] == manifest["rowwise_dataset"]["area_support"] + assert plan["region_mix"] == manifest["rowwise_dataset"]["region_mix"] # The dry-run's realized support is exact: identical draws to the build. realized = { @@ -462,6 +566,162 @@ def test_driver_ladder_dry_run_matches_real_assignment( plan["realized_support"]["constituency"]["n_areas"] >= manifest["rowwise_dataset"]["assigned_constituencies"] ) + assert plan["source_lineage"]["explicit"] is None + assert manifest["rowwise_dataset"]["source_lineage"]["explicit"] is None + + +def test_driver_ladder_candidate_k_matches_independent_single_k_plan( + monkeypatch, toy_ladder, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + import json + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + candidate_dir = tmp_path / "candidates" + independent_dir = tmp_path / "independent" + base_argv = [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--seed", + "7", + ] + + monkeypatch.setattr( + sys, + "argv", + [ + *base_argv, + "--out", + str(candidate_dir), + "--n-clones", + "2", + "--candidate-clone-counts", + "3,1,3", + "--dry-run", + ], + ) + assert builder.main() == 0 + candidate_plan = json.loads( + (candidate_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() + ) + + monkeypatch.setattr( + sys, + "argv", + [ + *base_argv, + "--out", + str(independent_dir), + "--n-clones", + "3", + "--dry-run", + ], + ) + assert builder.main() == 0 + independent_plan = json.loads( + (independent_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() + ) + + assert candidate_plan["plan"]["n_clones"] == 2 + candidates = candidate_plan["candidates"] + assert candidates["clone_counts"] == [1, 3] + assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] + input_bytes = input_h5.stat().st_size + base_rows = {"person": 5, "benunit": 4, "household": 4} + for candidate in candidates["plans"]: + n_clones = candidate["n_clones"] + assert candidate["rows"] == { + name: rows * n_clones for name, rows in base_rows.items() + } + assert candidate["output_bytes_estimate"] == input_bytes * n_clones + assert set(candidate) == { + "n_clones", + "rows", + "output_bytes_estimate", + "realized_support", + "expected_support", + "area_support", + } + + candidate_k3 = candidates["plans"][1] + assert candidate_k3["realized_support"] == { + area_type: independent_plan["realized_support"][area_type] + for area_type in ("constituency", "la") + } + assert candidate_k3["expected_support"] == { + area_type: independent_plan["expected_support"][area_type] + for area_type in ("constituency", "la") + } + assert candidate_k3["area_support"] == independent_plan["area_support"] + + +def test_driver_ladder_spine_lineage_plan_manifest_parity( + monkeypatch, toy_ladder, tmp_path +): + pytest.importorskip("tables") + pytest.importorskip("h5py") + import json + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "spine.h5" + _write_seam_h5(input_h5, household=_spine_household_frame()) + plan_dir = tmp_path / "plan" + build_dir = tmp_path / "build" + base_argv = [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--n-clones", + "2", + "--seed", + "7", + ] + + monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(plan_dir), "--dry-run"]) + assert builder.main() == 0 + plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) + + monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) + assert builder.main() == 0 + manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) + + explicit = plan["source_lineage"]["explicit"] + assert explicit == manifest["rowwise_dataset"]["source_lineage"]["explicit"] + assert plan["area_support"]["source_basis"] == "source_household_id" + assert ( + manifest["rowwise_dataset"]["area_support"]["source_basis"] + == "source_household_id" + ) + assert explicit == { + "basis": "explicit_lineage_columns", + "columns_present": [ + "source_household_id", + "household_support_channel", + "household_support_clone_index", + "household_is_spi_synthetic", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", + ], + "distinct_source_households": 3, + "distinct_by_support_channel": {"frs": 1, "spi": 2}, + "flag_counts": { + "household_is_spi_synthetic": 2, + "household_is_capital_gains_clone": 1, + "household_is_cgt_band_donor": 1, + }, + } def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path): @@ -491,6 +751,166 @@ def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path builder.main() +def test_driver_ladder_sha256_refuses_crosswalk(monkeypatch, tmp_path) -> None: + pytest.importorskip("tables") + import sys + + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--crosswalk", + str(tmp_path / "crosswalk.csv"), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(ValueError, match="ladder-sha256.*ladder"): + builder.main() + + +def test_driver_ladder_sha256_refuses_generated_crosswalk_route( + monkeypatch, tmp_path +) -> None: + # Neither --ladder nor --crosswalk: the driver would download and build a + # crosswalk, and there is no ladder artifact the pin could verify. A + # silently ignored pin would report verification that never happened. + pytest.importorskip("tables") + import sys + + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(ValueError, match="ladder-sha256 requires --ladder"): + builder.main() + + +def test_area_support_stats_read_nonzero_rows_not_assigned() -> None: + # An area whose only rows are zero-weight synthetics shapes no estimate + # there: the stats and the thinness ranking must read mass-carrying rows, + # with the assigned count still visible per bottom area. + builder = _load_builder_module() + support = pd.DataFrame( + { + "area_code": ["A1", "A2"], + "assigned_households": [40, 3], + "nonzero_households": [0, 3], + "nonzero_source_households": [0, 3], + "weight_sum": [0.0, 30.0], + "max_weight": [0.0, 10.0], + "effective_sample_size": [0.0, 3.0], + } + ) + + stats = builder._area_support_stats(support) + + assert stats["rows_basis"] == "nonzero_households" + assert stats["min_rows"] == 0 + assert stats["bottom_by_rows"][0] == { + "area_code": "A1", + "rows": 0, + "assigned": 40, + } + + +def test_driver_ladder_pin_mismatch_refuses_before_parse( + monkeypatch, toy_ladder, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + + def unexpected_parse(_path): + raise AssertionError("ladder pin mismatch must refuse before NPZ parsing") + + monkeypatch.setattr(builder, "load_uk_oa_ladder", unexpected_parse) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(SystemExit, match=r"--ladder sha mismatch: measured"): + builder.main() + + +def test_driver_ladder_dry_run_refuses_legacy_preassigned_geography( + monkeypatch, toy_ladder, tmp_path +): + pytest.importorskip("tables") + pytest.importorskip("h5py") + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "preassigned.h5" + household = _household_frame().assign( + constituency_code_oa="stale-constituency" + ) + _write_seam_h5(input_h5, household=household) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--out", + str(tmp_path / "plan"), + "--dry-run", + ], + ) + + with pytest.raises( + ValueError, + match=r"stale \*_oa columns cannot ride through", + ): + builder.main() + + def test_ladder_clone_pins_per_copy_weights_and_fk_alignment(toy_ladder) -> None: ladder, _ = toy_ladder result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=2) diff --git a/packages/microcosm-build/tests/test_uk_local_rowwise.py b/packages/microcosm-build/tests/test_uk_local_rowwise.py index 2b4252efa7..69e58fbb8d 100644 --- a/packages/microcosm-build/tests/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/test_uk_local_rowwise.py @@ -28,7 +28,9 @@ rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, uk_household_weight_kind, + uk_ladder_area_support_summary, uk_national_frame, ) from microcosm.build.uk_runtime.weighted_integrity import ( @@ -407,6 +409,79 @@ def test_rowwise_area_support_summary_reports_all_target_areas() -> None: assert rows["E001"].effective_sample_size == pytest.approx(1.0) +def test_rowwise_area_support_wrapper_equals_frame_agnostic_core() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + weights = np.array([2.0, 0.0, 5.0]) + source_household_ids = [(2023, 1), (2023, 1), (2023, 2)] + + wrapped = rowwise_area_support_summary( + problem, + weights, + source_household_ids=source_household_ids, + ) + core = uk_area_support_summary( + problem.assigned_areas, + weights, + area_codes=problem.area_codes, + source_household_ids=source_household_ids, + ) + + pd.testing.assert_frame_equal(wrapped, core) + + +def test_ladder_area_support_includes_zeros_and_distinct_cloned_sources() -> None: + ladder = type( + "Ladder", + (), + { + "constituency_code": np.asarray(["C1", "C2", "C3"]), + "local_authority_code": np.asarray(["L1", "L2", "L3"]), + }, + )() + household = pd.DataFrame( + { + "household_id": [101, 102, 201, 202], + "source_household_id": [10, 10, 20, 20], + "household_weight": [2.0, 3.0, 4.0, 0.0], + "constituency_code": ["C1", "C1", "C2", "C2"], + "local_authority_code": ["L1", "L1", "L2", "L2"], + } + ) + + summaries = uk_ladder_area_support_summary(household, ladder) + + constituency = summaries["constituency"].set_index("area_code") + assert constituency.loc["C1", "assigned_households"] == 2 + assert constituency.loc["C1", "nonzero_households"] == 2 + assert constituency.loc["C1", "nonzero_source_households"] == 1 + assert constituency.loc["C2", "nonzero_source_households"] == 1 + assert constituency.loc["C3", "assigned_households"] == 0 + assert constituency.loc["C3", "effective_sample_size"] == 0.0 + assert summaries["la"]["area_code"].tolist() == ["L1", "L2", "L3"] + + +def test_ladder_area_support_refuses_missing_source_column() -> None: + ladder = type( + "Ladder", + (), + { + "constituency_code": np.asarray(["C1"]), + "local_authority_code": np.asarray(["L1"]), + }, + )() + household = pd.DataFrame( + { + "household_id": [1], + "household_weight": [1.0], + "constituency_code": ["C1"], + "local_authority_code": ["L1"], + } + ) + + with pytest.raises(ValueError, match="distinct-source honesty requires it"): + uk_ladder_area_support_summary(household, ladder) + + def test_matrix_builder_fails_closed_on_unreachable_nonzero_targets() -> None: # A target area with no assigned households cannot be hit; a nonzero # target there must refuse at build time, while a zero target is fine. diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index 2ece175f64..d65a6d487f 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -132,6 +132,42 @@ def _crosswalk_frame() -> pd.DataFrame: ) +@pytest.mark.parametrize("route_option", ["--crosswalk", "--ladder"]) +def test_input_pin_mismatch_refuses_before_side_effects( + monkeypatch, + tmp_path, + route_option, +) -> None: + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "input.h5" + route_artifact = tmp_path / "route-artifact" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5) + route_artifact.write_bytes(b"must not be read") + + def unexpected_h5_read(_path): + raise AssertionError("input pin mismatch must refuse before H5 parsing") + + monkeypatch.setattr(builder, "_h5_summary", unexpected_h5_read) + + with pytest.raises(SystemExit, match=r"--input-h5 sha mismatch: measured"): + builder.main( + [ + "--input-h5", + str(input_h5), + "--input-sha256", + "0" * 64, + "--out", + str(output_dir), + route_option, + str(route_artifact), + ] + ) + + assert not output_dir.exists() + + def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( monkeypatch, tmp_path, capsys ): @@ -143,7 +179,11 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( la_codes = tmp_path / "local_authorities.csv" output_dir = tmp_path / "out" _write_toy_h5(input_h5) + input_sha256 = builder._sha256(input_h5) _crosswalk_frame().to_csv(crosswalk_path, index=False) + output_dir.mkdir() + stale_area_support = output_dir / builder.AREA_SUPPORT_FILENAME + stale_area_support.write_text("stale") pd.DataFrame({"code": ["E14000001", "W07000041", "S14000001", "N05000001"]}).to_csv( constituency_codes, index=False ) @@ -157,6 +197,8 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( "build_uk_rowwise_dataset.py", "--input-h5", str(input_h5), + "--input-sha256", + input_sha256, "--out", str(output_dir), "--crosswalk", @@ -182,11 +224,13 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( assert output_h5.exists() assert manifest_path.exists() assert coverage_path.exists() + assert not stale_area_support.exists() manifest = json.loads(manifest_path.read_text()) assert manifest["build_kind"] == "uk_rowwise_local_geography_dataset" assert manifest["parameters"]["n_clones"] == 2 assert manifest["parameters"]["source_year"] == 2023 assert manifest["parameters"]["require_all_countries"] is False + assert manifest["inputs"]["dataset"]["pin_verified"] is True assert manifest["base_dataset"]["household_weight_sum"] == pytest.approx(30.0) assert manifest["rowwise_dataset"]["household_weight_sum"] == pytest.approx(30.0) assert manifest["rowwise_dataset"]["household_weight_delta"] == pytest.approx(0.0) @@ -195,6 +239,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 2 assert manifest["coverage"][0]["covered_areas"] == 4 assert manifest["outputs"]["crosswalk"] is None + assert manifest["outputs"]["area_support_summary"] is None with pd.HDFStore(output_h5, mode="r") as store: assert store["household"].shape[0] == 4 assert store["person"].shape[0] == 6 @@ -202,7 +247,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( rows = _spool_rows(output_dir) assert len(rows) == 1 first_row = rows[0] - assert first_row.pipeline == "uk-locals-rowwise" + assert first_row.pipeline == "uk-local-rowwise" assert first_row.rung == "f100" assert first_row.seed == 42 assert first_row.disposition == "iterating" @@ -364,6 +409,7 @@ def fake_clone(*_args, output_path: Path, **_kwargs): "ward_code": ["E05000001", "W05000001"], "constituency_code": ["E14000001", "W07000041"], "region_code": ["E12000007", "W99999999"], + "region": ["LONDON", "WALES"], "itl3_code": ["TLI", "TLL"], "itl2_code": ["TL", "TL"], "itl1_code": ["T", "T"], @@ -391,7 +437,17 @@ def fake_clone(*_args, output_path: Path, **_kwargs): }, )() - monkeypatch.setattr(builder, "load_uk_oa_ladder", lambda _path: object()) + ladder = type( + "Ladder", + (), + { + "constituency_code": pd.Series(["E14000001", "W07000041"]).to_numpy(), + "local_authority_code": pd.Series( + ["E06000063", "W06000001"] + ).to_numpy(), + }, + )() + monkeypatch.setattr(builder, "load_uk_oa_ladder", lambda _path: ladder) monkeypatch.setattr( builder, "clone_uk_dataset_with_ladder_geography", @@ -416,6 +472,10 @@ def fake_clone(*_args, output_path: Path, **_kwargs): assert builder.main() == 0 manifest_path = output_dir / builder.MANIFEST_FILENAME + manifest = json.loads(manifest_path.read_text()) + assert manifest["rowwise_dataset"]["area_support"]["source_basis"] == ( + "household_id" + ) rows = _spool_rows(output_dir) assert len(rows) == 1 assert rows[0].gate_verdicts == { @@ -470,6 +530,25 @@ def fail_clone(*_args, **_kwargs): assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") +def test_candidate_clone_counts_refused_on_real_build(tmp_path) -> None: + builder = _load_builder_module() + output_dir = tmp_path / "out" + + with pytest.raises(ValueError, match="candidate-K planning is a dry-run surface"): + builder.main( + [ + "--input-h5", + str(tmp_path / "not-read.h5"), + "--out", + str(output_dir), + "--candidate-clone-counts", + "1,2,4", + ] + ) + + assert not output_dir.exists() + + @pytest.mark.parametrize( "dataset_filename", [ @@ -477,6 +556,7 @@ def fail_clone(*_args, **_kwargs): "/tmp/escaped.h5", "rowwise_build_manifest.json", "geography_coverage_summary.csv", + "area_support_summary.csv", "uk_official_geography_crosswalk.csv.gz", ], ) @@ -516,7 +596,11 @@ def test_validate_output_paths_rejects_crosswalk_collision(tmp_path): @pytest.mark.parametrize( "sidecar_name", - ["rowwise_build_manifest.json", "geography_coverage_summary.csv"], + [ + "rowwise_build_manifest.json", + "geography_coverage_summary.csv", + "area_support_summary.csv", + ], ) def test_validate_output_paths_rejects_supplied_crosswalk_sidecar_collision( sidecar_name, diff --git a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py index b7965cadd4..f98c54f1ca 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py @@ -349,7 +349,7 @@ def test_candidate_build_writes_calibrated_h5_and_evidence(tmp_path) -> None: rows = _spool_rows(output_dir) assert len(rows) == 1 row = rows[0] - assert row.pipeline == "uk-locals-candidate" + assert row.pipeline == "uk-local-candidate" assert row.rung == "f100" assert row.seed == 7 assert row.disposition == "iterating" diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index 9d1b2a2fbf..b1cb6c77bc 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -12,6 +12,7 @@ from __future__ import annotations +import argparse import importlib.util import json import sys @@ -141,6 +142,28 @@ def _dry_run_argv( ] +def test_candidate_clone_counts_parser_deduplicates_and_sorts() -> None: + builder = _load_builder_module() + + assert builder._candidate_clone_counts_argument("10, 2,4,2,1") == ( + 1, + 2, + 4, + 10, + ) + + +@pytest.mark.parametrize( + "value", + ["", " ", ",", "0", "-1", "1,0", "1,-2", "1,junk", "1,,2"], +) +def test_candidate_clone_counts_parser_refuses_invalid_values(value) -> None: + builder = _load_builder_module() + + with pytest.raises(argparse.ArgumentTypeError, match="positive integers"): + builder._candidate_clone_counts_argument(value) + + def test_collision_free_expectation_matches_distribution_math() -> None: support = expected_uk_rowwise_area_support( _household_frame(), @@ -214,6 +237,7 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["build_kind"] == "uk_rowwise_local_geography_dry_run" assert plan["parameters"]["n_clones"] == 2 assert plan["input"]["dataset"]["sha256"] + assert plan["input"]["dataset"]["pin_verified"] is False assert plan["input"]["household_weight_kind"] == WeightKind.DESIGN.value assert plan["input"]["mass_log_records"] == 0 assert plan["plan"]["rows"] == { @@ -244,6 +268,54 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["source_lineage"]["pool_modulus"] is None assert plan["source_lineage"]["pool"] is None + assert plan["source_lineage"]["explicit"] is None + + +def test_crosswalk_dry_run_reports_candidate_k_rows_bytes_and_support( + monkeypatch, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2023.h5" + crosswalk_path = tmp_path / "crosswalk.csv.gz" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5) + _crosswalk_frame().to_csv(crosswalk_path, index=False) + + monkeypatch.setattr( + sys, + "argv", + _dry_run_argv( + input_h5, + output_dir, + crosswalk_path, + "--candidate-clone-counts", + "3,1,3", + ), + ) + assert builder.main() == 0 + plan = json.loads((output_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) + + assert plan["parameters"]["n_clones"] == 2 + assert plan["parameters"]["candidate_clone_counts"] == [1, 3] + candidates = plan["candidates"] + assert candidates["clone_counts"] == [1, 3] + assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] + input_bytes = input_h5.stat().st_size + base_rows = {"person": 3, "benunit": 2, "household": 2} + for candidate in candidates["plans"]: + n_clones = candidate["n_clones"] + assert candidate["rows"] == { + name: rows * n_clones for name, rows in base_rows.items() + } + assert candidate["output_bytes_estimate"] == input_bytes * n_clones + assert set(candidate["realized_support"]) == {"constituency", "la"} + assert set(candidate["collision_free_expected_support"]) == { + "constituency", + "la", + } + assert "area_support" not in candidate def test_driver_dry_run_reports_weight_chain_and_pool_lineage( @@ -316,6 +388,7 @@ def test_driver_dry_run_reports_weight_chain_and_pool_lineage( assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 # The staging input's immediate layer is reported untouched alongside. assert lineage["immediate"]["distinct_source_households"] == 4 + assert lineage["explicit"] is None def test_driver_dry_run_supports_fixed_format_stores(monkeypatch, tmp_path) -> None: @@ -466,6 +539,7 @@ def test_driver_full_build_records_weight_chain_and_lineage( assert lineage["pool"]["pool_copies_per_source"]["min"] == 2 assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 assert lineage["immediate"] is None + assert lineage["explicit"] is None assert manifest["base_dataset"]["distinct_source_households"] is None output_h5 = output_dir / "pool_rowwise.h5" diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index b9f6ba84ce..5db2c3aefd 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -32,6 +32,7 @@ from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) +from microcosm.build.uk_runtime.rowwise_dataset import UK_SPINE_LINEAGE_COLUMNS from microcosm.frame import Frame, MassChangeRecord, WeightKind @@ -415,6 +416,18 @@ def test_source_lineage_modulus_rejects_bad_inputs() -> None: apply_uk_source_lineage_modulus(fractional, modulus=100) +@pytest.mark.parametrize("column", UK_SPINE_LINEAGE_COLUMNS) +def test_source_lineage_modulus_refuses_spine_lineage(column: str) -> None: + household = household_frame() + household[column] = 1 + + with pytest.raises( + ValueError, + match="explicit spine lineage columns are the source of truth", + ): + apply_uk_source_lineage_modulus(household, modulus=100) + + def test_clone_entity_frame_refuses_int64_overflow() -> None: frame = pd.DataFrame({"benunit_id": [np.iinfo(np.int64).max - 5]}) with pytest.raises(ValueError, match="overflow"): diff --git a/supabase/migrations/20260829000000_logbook_uk_local_scope.sql b/supabase/migrations/20260829000000_logbook_uk_local_scope.sql new file mode 100644 index 0000000000..ed9aa23342 --- /dev/null +++ b/supabase/migrations/20260829000000_logbook_uk_local_scope.sql @@ -0,0 +1,13 @@ +-- Ratify uk/local for the local-areas line's first archived run (#761). +CREATE OR REPLACE FUNCTION logbook.scope_declared(p_scope text) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = pg_catalog +AS $function$ + SELECT p_scope IN ('us', 'uk/frs', 'uk/local'); +$function$; + +GRANT EXECUTE ON FUNCTION logbook.scope_declared(text) + TO logbook_writer, logbook_exporter, logbook_break_glass_admin; diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index 34b18da459..cc39600e09 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -86,7 +86,7 @@ _TARGET_RECORDS: int | None = None _L0_LAMBDA = 0.0 _BUDGET_ITERS = 10 -_UK_CANDIDATE_PIPELINE = "uk-locals-candidate" +_UK_CANDIDATE_PIPELINE = "uk-local-candidate" _REPOSITORY = Path(__file__).resolve().parents[1] _PAST_CAP_COUNT_KEYS = ( "n_targets", @@ -113,7 +113,7 @@ def __init__( def _new_candidate_build_id(*, seed: int, timestamp: datetime) -> str: instant = timestamp.astimezone(UTC) return ( - f"uk-locals-candidate-f100-s{seed}-" + f"uk-local-candidate-f100-s{seed}-" f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" ) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 0b6fb75412..6f3bb24827 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -1,10 +1,13 @@ """Build a Microcosm UK row-wise local-geography dataset. -This is the narrow build driver for the UK local replacement path. It starts -from an existing compact Microcosm UK single-year H5, builds or loads the -official-source geography crosswalk, clones the entity tables, assigns each -household a finest available geography row, and writes diagnostics that prove -coverage and weight preservation. +This is the narrow build driver for the UK local replacement path. Its +sanctioned input is the microcosm-built national spine — the assembly seam's +accepted, sha-pinned single-year H5 with its declared weight kind and mass +log — never the retired certified compact and never anything upstream of the +seam. The driver clones the entity tables, assigns each household a finest +available geography row (the ratified OA ladder, or the crosswalk sampler as +the coverage-check path), and writes diagnostics that prove coverage, weight +preservation, and per-area support. """ from __future__ import annotations @@ -51,18 +54,26 @@ clone_entity_frame, clone_uk_dataset_with_ladder_geography, clone_uk_dataset_with_rowwise_geography, + expected_uk_ladder_area_support, expected_uk_rowwise_area_support, geography_coverage_summary, id_multiplier_for_values, ladder_clone_index_column, + load_uk_local_area_crosswalk, load_uk_oa_ladder, read_uk_single_year_weight_metadata, uk_geography_ladder_gate, uk_household_weight_kind, + uk_ladder_area_support_summary, + uk_region_mix, uk_time_period, validate_geography_coverage, write_geography_crosswalk, ) +from microcosm.build.uk_runtime.rowwise_dataset import ( + UK_SPINE_LINEAGE_COLUMNS, + _refuse_preassigned_geography, +) from microcosm.frame import engine_tables CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" @@ -70,11 +81,31 @@ MANIFEST_FILENAME = "rowwise_build_manifest.json" COVERAGE_FILENAME = "geography_coverage_summary.csv" DRY_RUN_PLAN_FILENAME = "rowwise_dry_run_plan.json" +AREA_SUPPORT_FILENAME = "area_support_summary.csv" EXPECTED_SUPPORT_BOTTOM_AREAS = 15 -_UK_ROWWISE_PIPELINE = "uk-locals-rowwise" +_UK_ROWWISE_PIPELINE = "uk-local-rowwise" _REPOSITORY = Path(__file__).resolve().parents[1] +def _candidate_clone_counts_argument(value: str) -> tuple[int, ...]: + parts = value.split(",") + if not value.strip() or any(not part.strip() for part in parts): + raise argparse.ArgumentTypeError( + "candidate clone counts must be a non-empty comma list of positive integers" + ) + try: + clone_counts = [int(part.strip()) for part in parts] + except ValueError as error: + raise argparse.ArgumentTypeError( + "candidate clone counts must be a comma list of positive integers" + ) from error + if any(clone_count <= 0 for clone_count in clone_counts): + raise argparse.ArgumentTypeError( + "candidate clone counts must all be positive integers" + ) + return tuple(sorted(set(clone_counts))) + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( @@ -83,6 +114,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Compact Microcosm UK single-year H5 to clone.", ) + parser.add_argument( + "--input-sha256", + type=sha256_argument, + help="Optional SHA-256 pin for --input-h5; mismatches fail before H5 parsing.", + ) parser.add_argument( "--out", type=Path, @@ -107,6 +143,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "Mutually exclusive with --crosswalk and the coverage-code checks." ), ) + parser.add_argument( + "--ladder-sha256", + type=sha256_argument, + help="Optional SHA-256 pin for --ladder; mismatches fail before NPZ parsing.", + ) parser.add_argument( "--expected-constituency-vintage", default="2024_pcon", @@ -126,6 +167,14 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Optional CSV containing a `code` column for local-authority coverage checks.", ) parser.add_argument("--n-clones", type=int, default=2) + parser.add_argument( + "--candidate-clone-counts", + type=_candidate_clone_counts_argument, + help=( + "Dry-run only: comma-separated candidate clone counts to plan " + "independently at the build seed (deduplicated and sorted)." + ), + ) parser.add_argument("--seed", type=int, default=42) parser.add_argument( "--source-year", @@ -163,13 +212,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--source-lineage-modulus", type=int, help=( - "Derive pool_source_household_id as household_id mod this value " - "before cloning, recovering pool-grain lineage as a distinct " - "layer (the certified UK pool encodes its 10x clone tiers with a " - "10**8 id offset; the staging H5's immediate-layer " - "source_household_id is left untouched). Refused when the pool " - "column already exists or the modulus would be an identity " - "mapping." + "Pool inputs only: derive pool_source_household_id before cloning " + "from household_id = tier * 10**8 + base, leaving the immediate " + "source_household_id untouched. Spine inputs instead use sernum + " + "1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * " + "10**(d+2) and carry authoritative explicit lineage columns, so " + "the modulus is refused for them. Also refused when the pool " + "column exists or the mapping would be an identity." ), ) parser.add_argument( @@ -203,7 +252,7 @@ def _new_rowwise_build_id( ) -> str: instant = timestamp.astimezone(UTC) return ( - f"uk-locals-rowwise-{route}-f100-s{seed}-" + f"uk-local-rowwise-{route}-f100-s{seed}-" f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" ) @@ -293,6 +342,11 @@ def _record_rowwise_error( def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) + if args.candidate_clone_counts is not None and not args.dry_run: + raise ValueError( + "candidate-K planning is a dry-run surface; a real build has " + "exactly one K from --n-clones." + ) if args.dry_run: return _main_impl(args, attempt=None) @@ -348,10 +402,14 @@ def _main_impl( *, attempt: dict[str, object] | None, ) -> int: - args.out.mkdir(parents=True, exist_ok=True) - input_h5 = args.input_h5.resolve() input_artifact = _artifact_info(input_h5) + _verify_artifact_pin( + input_artifact, + pinned_sha256=args.input_sha256, + option="--input-h5", + ) + args.out.mkdir(parents=True, exist_ok=True) base_summary = _h5_summary(input_h5) source_year = _source_year(args.source_year, base_summary=base_summary) output_h5 = _dataset_output_path( @@ -361,6 +419,12 @@ def _main_impl( source_year=source_year, ) _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) + if args.ladder_sha256 is not None and args.ladder is None: + raise ValueError( + "--ladder-sha256 requires --ladder; the crosswalk routes have no " + "ladder artifact to pin, and silently ignoring a supplied pin " + "would report verification that never happened." + ) if args.ladder is not None: if args.crosswalk is not None: raise ValueError("--ladder and --crosswalk are mutually exclusive.") @@ -374,6 +438,7 @@ def _main_impl( (args.out / COVERAGE_FILENAME).resolve(), (args.out / DRY_RUN_PLAN_FILENAME).resolve(), (args.out / CROSSWALK_FILENAME).resolve(), + (args.out / AREA_SUPPORT_FILENAME).resolve(), } if args.ladder.resolve() in sidecars: raise ValueError( @@ -393,6 +458,8 @@ def _main_impl( source_year=source_year, attempt=attempt, ) + if not args.dry_run: + (args.out / AREA_SUPPORT_FILENAME).unlink(missing_ok=True) crosswalk_source = _load_or_build_crosswalk(args) crosswalk = crosswalk_source.frame crosswalk_path = crosswalk_source.path @@ -484,6 +551,7 @@ def _main_impl( _artifact_info(crosswalk_path) if crosswalk_source.generated else None ), "coverage_summary": coverage_artifact, + "area_support_summary": None, }, "base_dataset": base_summary, "rowwise_dataset": rowwise_summary, @@ -561,6 +629,7 @@ def _dataset_output_path( MANIFEST_FILENAME, COVERAGE_FILENAME, DRY_RUN_PLAN_FILENAME, + AREA_SUPPORT_FILENAME, } if path.name in reserved: raise ValueError( @@ -579,6 +648,7 @@ def _validate_output_paths( (args.out / MANIFEST_FILENAME).resolve(), (args.out / COVERAGE_FILENAME).resolve(), (args.out / DRY_RUN_PLAN_FILENAME).resolve(), + (args.out / AREA_SUPPORT_FILENAME).resolve(), } generated_crosswalk_path = (args.out / CROSSWALK_FILENAME).resolve() reserved_paths = { @@ -680,6 +750,11 @@ def _parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any] ladder_route = getattr(args, "ladder", None) is not None return { "n_clones": args.n_clones, + "candidate_clone_counts": ( + list(args.candidate_clone_counts) + if args.candidate_clone_counts is not None + else None + ), "seed": args.seed, "source_year": source_year, # Crosswalk-sampler knobs are meaningless on the ladder route and are @@ -790,7 +865,7 @@ def _dry_run_plan( for name in ("person", "benunit", "household") } input_bytes = input_h5.stat().st_size - return { + plan = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dry_run", "created_at": datetime.now(UTC).isoformat(), @@ -824,7 +899,9 @@ def _dry_run_plan( "sampleable areas included" ), **{ - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) for area_type in ("constituency", "la") }, }, @@ -835,7 +912,9 @@ def _dry_run_plan( "constituency count" ), **{ - area_type: _support_summary(collision_free, area_type) + area_type: _support_summary( + collision_free, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, }, @@ -845,6 +924,83 @@ def _dry_run_plan( ), "coverage": coverage.to_dict("records") if not coverage.empty else [], } + if args.candidate_clone_counts is not None: + plan["candidates"] = _crosswalk_candidate_plans( + args, + household=household, + crosswalk=crosswalk, + source_year=source_year, + id_multiplier=id_multiplier, + table_rows=table_rows, + input_bytes=input_bytes, + ) + return plan + + +def _crosswalk_candidate_plans( + args: argparse.Namespace, + *, + household: pd.DataFrame, + crosswalk: pd.DataFrame, + source_year: int, + id_multiplier: int, + table_rows: dict[str, int], + input_bytes: int, +) -> dict[str, Any]: + clone_counts = args.candidate_clone_counts + assert clone_counts is not None + plans = [] + for n_clones in clone_counts: + assignment = assign_household_geography( + household, + crosswalk, + n_clones=n_clones, + seed=args.seed, + id_multiplier=id_multiplier, + source_year=source_year, + require_all_countries=not args.allow_missing_country, + require_constituency=not args.allow_blank_constituency, + constrain_to_region=not args.allow_cross_region_assignment, + avoid_constituency_collisions=not args.allow_constituency_collisions, + ) + realized = _realized_area_support(assignment.household, crosswalk) + collision_free = expected_uk_rowwise_area_support( + household, + crosswalk, + n_clones=n_clones, + source_year=source_year, + require_all_countries=not args.allow_missing_country, + require_constituency=not args.allow_blank_constituency, + constrain_to_region=not args.allow_cross_region_assignment, + ) + plans.append( + { + "n_clones": n_clones, + "rows": {name: rows * n_clones for name, rows in table_rows.items()}, + "output_bytes_estimate": input_bytes * n_clones, + "realized_support": { + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) + for area_type in ("constituency", "la") + }, + "collision_free_expected_support": { + area_type: _support_summary( + collision_free, area_type, rows_basis="expected_rows" + ) + for area_type in ("constituency", "la") + }, + } + ) + return { + "basis": ( + "independent crosswalk assignments at the build seed for each " + "candidate K; realized and collision-free expected support only " + "because this route has no ladder roster for area-support stats" + ), + "clone_counts": list(clone_counts), + "plans": plans, + } def _run_ladder_route( @@ -861,6 +1017,12 @@ def _run_ladder_route( ladder_path = args.ladder.resolve() ladder_artifact = _artifact_info(ladder_path) + _verify_artifact_pin( + ladder_artifact, + pinned_sha256=args.ladder_sha256, + option="--ladder", + ) + _annotate_ladder_crosswalk_pin(ladder_artifact) ladder = load_uk_oa_ladder(ladder_path) if args.dry_run: @@ -928,6 +1090,17 @@ def _run_ladder_route( "passed": bool(result.gate.passed), "details": dict(result.gate.details), } + household = _result_household(result) + area_support, area_support_summaries = _ladder_area_support_diagnostics( + household, + ladder, + ) + rowwise_summary["area_support"] = area_support + rowwise_summary["region_mix"] = uk_region_mix(household).to_dict("records") + area_support_path = args.out / AREA_SUPPORT_FILENAME + _area_support_long_frame(area_support_summaries).to_csv( + area_support_path, index=False + ) manifest = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dataset", @@ -942,6 +1115,7 @@ def _run_ladder_route( "dataset": _artifact_info(output_h5), "crosswalk": None, "coverage_summary": None, + "area_support_summary": _artifact_info(area_support_path), }, "base_dataset": base_summary, "rowwise_dataset": rowwise_summary, @@ -1004,6 +1178,7 @@ def _ladder_dry_run_plan( person_ids=person_ids, benunit_ids=benunit_ids, ) + _refuse_preassigned_geography(household, label="household") id_multiplier = id_multiplier_for_values( household["household_id"], person_ids["person_id"], @@ -1030,39 +1205,31 @@ def _ladder_dry_run_plan( raise ValueError("household weights must carry positive total mass.") _kind, mass_log = read_uk_single_year_weight_metadata(input_h5) _assert_mass_log_current(mass_log, float(weight_values.sum())) - cloned = clone_entity_frame( + assigned = _ladder_planned_assignment( household, - id_columns=("household_id",), + ladder, n_clones=args.n_clones, + seed=args.seed, id_multiplier=id_multiplier, - clone_index_column="clone_index", - ).reset_index(drop=True) - cloned["household_weight"] = ( - pd.to_numeric(cloned["household_weight"], errors="raise").to_numpy(dtype=float) - / args.n_clones + expected_constituency_vintage=args.expected_constituency_vintage, ) - assigned = assign_uk_geography_ladder( - cloned, + realized = _ladder_realized_support(assigned, ladder) + expected = expected_uk_ladder_area_support( + household, ladder, - seed=args.seed, - expected_constituency_vintage=args.expected_constituency_vintage, + n_clones=args.n_clones, ) - gate = uk_geography_ladder_gate( + area_support, _ = _ladder_area_support_diagnostics( assigned, - assigned["household_weight"].to_numpy(dtype=float), + ladder, ) - if not gate.passed: - raise ValueError( - "UK geography ladder gate would fail this build: " - + "; ".join(gate.failures) - ) - realized = _ladder_realized_support(assigned, ladder) + region_mix = uk_region_mix(assigned).to_dict("records") table_rows = { name: base_summary["tables"][name][0] for name in ("person", "benunit", "household") } input_bytes = input_h5.stat().st_size - return { + plan = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dry_run", "created_at": datetime.now(UTC).isoformat(), @@ -1096,16 +1263,147 @@ def _ladder_dry_run_plan( "ladder areas included" ), **{ - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) + for area_type in ("constituency", "la") + }, + }, + "expected_support": { + "basis": ( + "analytic expectation: constituency household-count share " + "within region x the input's region mix x n_clones; OA " + "population shares within constituency for LA support" + ), + **{ + area_type: _support_summary( + expected, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, }, + "area_support": area_support, + "region_mix": region_mix, "source_lineage": _source_lineage_report( household, modulus=args.source_lineage_modulus, ), "coverage": [], } + if args.candidate_clone_counts is not None: + plan["candidates"] = _ladder_candidate_plans( + args, + household=household, + ladder=ladder, + id_multiplier=id_multiplier, + table_rows=table_rows, + input_bytes=input_bytes, + ) + return plan + + +def _ladder_planned_assignment( + household: pd.DataFrame, + ladder: Any, + *, + n_clones: int, + seed: int, + id_multiplier: int, + expected_constituency_vintage: str, +) -> pd.DataFrame: + """Execute the fenced ladder assignment used by every dry-run K.""" + + household_for_clone = household.copy() + if "source_household_id" not in household_for_clone.columns: + household_for_clone["source_household_id"] = household_for_clone["household_id"] + cloned = clone_entity_frame( + household_for_clone, + id_columns=("household_id",), + n_clones=n_clones, + id_multiplier=id_multiplier, + clone_index_column="clone_index", + ).reset_index(drop=True) + cloned["household_weight"] = ( + pd.to_numeric(cloned["household_weight"], errors="raise").to_numpy(dtype=float) + / n_clones + ) + assigned = assign_uk_geography_ladder( + cloned, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + ) + gate = uk_geography_ladder_gate( + assigned, + assigned["household_weight"].to_numpy(dtype=float), + ) + if not gate.passed: + raise ValueError( + "UK geography ladder gate would fail this build: " + + "; ".join(gate.failures) + ) + return assigned + + +def _ladder_candidate_plans( + args: argparse.Namespace, + *, + household: pd.DataFrame, + ladder: Any, + id_multiplier: int, + table_rows: dict[str, int], + input_bytes: int, +) -> dict[str, Any]: + clone_counts = args.candidate_clone_counts + assert clone_counts is not None + plans = [] + for n_clones in clone_counts: + assigned = _ladder_planned_assignment( + household, + ladder, + n_clones=n_clones, + seed=args.seed, + id_multiplier=id_multiplier, + expected_constituency_vintage=args.expected_constituency_vintage, + ) + realized = _ladder_realized_support(assigned, ladder) + expected = expected_uk_ladder_area_support( + household, + ladder, + n_clones=n_clones, + ) + area_support, _ = _ladder_area_support_diagnostics( + assigned, + ladder, + ) + plans.append( + { + "n_clones": n_clones, + "rows": {name: rows * n_clones for name, rows in table_rows.items()}, + "output_bytes_estimate": input_bytes * n_clones, + "realized_support": { + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) + for area_type in ("constituency", "la") + }, + "expected_support": { + area_type: _support_summary( + expected, area_type, rows_basis="expected_rows" + ) + for area_type in ("constituency", "la") + }, + "area_support": area_support, + } + ) + return { + "basis": ( + "independent fenced ladder assignments at the build seed for each " + "candidate K; summary-only plans" + ), + "clone_counts": list(clone_counts), + "plans": plans, + } def _ladder_realized_support( @@ -1225,12 +1523,14 @@ def _support_summary( support: pd.DataFrame, area_type: str, *, + rows_basis: str, bottom: int = EXPECTED_SUPPORT_BOTTOM_AREAS, ) -> dict[str, Any]: subset = support[support["area_type"] == area_type] if subset.empty: return { "n_areas": 0, + "rows_basis": rows_basis, "min_rows": 0.0, "median_rows": 0.0, "mean_rows": 0.0, @@ -1244,6 +1544,7 @@ def _support_summary( ).head(bottom) return { "n_areas": int(len(subset)), + "rows_basis": rows_basis, "min_rows": float(values.min()), "median_rows": float(values.median()), "mean_rows": float(values.mean()), @@ -1258,6 +1559,92 @@ def _support_summary( } +def _result_household(result: Any) -> pd.DataFrame: + if isinstance(result, UKLadderRowwiseDatasetResult): + return engine_tables(result.frame, weighted_entities=("household",))[ + "household" + ] + return result.household + + +def _ladder_area_support_diagnostics( + household: pd.DataFrame, + ladder: Any, +) -> tuple[dict[str, Any], dict[str, pd.DataFrame]]: + source_basis = ( + "source_household_id" + if "source_household_id" in household.columns + else "household_id" + ) + summaries = uk_ladder_area_support_summary( + household, + ladder, + source_column=source_basis, + ) + block = { + "source_basis": source_basis, + **{ + area_type: _area_support_stats(summaries[area_type]) + for area_type in ("constituency", "la") + }, + } + return block, summaries + + +def _area_support_long_frame(summaries: dict[str, pd.DataFrame]) -> pd.DataFrame: + """The full per-area table, built only where a consumer writes it.""" + + long_frames = [] + for area_type in ("constituency", "la"): + frame = summaries[area_type].copy() + frame.insert(0, "area_type", area_type) + long_frames.append(frame) + return pd.concat(long_frames, ignore_index=True) + + +def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: + # Support means rows that carry mass: a row assigned to an area at zero + # weight shapes no estimate there, so the headline stats and the thinness + # ranking read nonzero_households. The assigned count stays visible per + # bottom area and in the full per-area CSV. + rows = support["nonzero_households"] + ess = support["effective_sample_size"] + sources = support["nonzero_source_households"] + bottom_by_rows = support.sort_values( + ["nonzero_households", "area_code"], + kind="mergesort", + ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) + bottom_by_ess = support.sort_values( + ["effective_sample_size", "area_code"], + kind="mergesort", + ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) + return { + "n_areas": int(len(support)), + "rows_basis": "nonzero_households", + "min_rows": int(rows.min()) if len(rows) else 0, + "median_rows": float(rows.median()) if len(rows) else 0.0, + "min_ess": float(ess.min()) if len(ess) else 0.0, + "median_ess": float(ess.median()) if len(ess) else 0.0, + "min_distinct_sources": int(sources.min()) if len(sources) else 0, + "median_distinct_sources": (float(sources.median()) if len(sources) else 0.0), + "bottom_by_rows": [ + { + "area_code": str(row.area_code), + "rows": int(row.nonzero_households), + "assigned": int(row.assigned_households), + } + for row in bottom_by_rows.itertuples(index=False) + ], + "bottom_by_ess": [ + { + "area_code": str(row.area_code), + "ess": float(row.effective_sample_size), + } + for row in bottom_by_ess.itertuples(index=False) + ], + } + + def _pool_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: if POOL_SOURCE_LINEAGE_COLUMN not in household.columns: return None @@ -1278,13 +1665,45 @@ def _pool_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: return block +def _explicit_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: + spine_columns = [ + column for column in UK_SPINE_LINEAGE_COLUMNS if column in household.columns + ] + if not spine_columns: + return None + + columns_present = [ + column + for column in ("source_household_id", *UK_SPINE_LINEAGE_COLUMNS) + if column in household.columns + ] + block: dict[str, Any] = { + "basis": "explicit_lineage_columns", + "columns_present": columns_present, + "flag_counts": { + column: int(household[column].fillna(False).astype(bool).sum()) + for column in UK_SPINE_LINEAGE_COLUMNS + if column.startswith("household_is_") and column in household.columns + }, + } + if "source_household_id" in household.columns: + block["distinct_source_households"] = int( + household["source_household_id"].nunique() + ) + if "household_support_channel" in household.columns: + block["distinct_by_support_channel"] = { + str(channel): int(group["source_household_id"].nunique()) + for channel, group in household.groupby("household_support_channel") + } + return block + + def _source_lineage_report( household: pd.DataFrame, *, modulus: int | None, ) -> dict[str, Any]: - """Report both lineage layers honestly: the derived pool layer (when a - modulus was applied) and any immediate layer the input already carries.""" + """Report pool, immediate, and explicit spine lineage when supported.""" pool = _pool_lineage_block(household) immediate = None @@ -1298,6 +1717,7 @@ def _source_lineage_report( "pool_modulus": modulus, "pool": pool, "immediate": immediate, + "explicit": _explicit_lineage_block(household), } @@ -1368,6 +1788,7 @@ def _rowwise_summary( if base_summary.get("distinct_source_households") is not None else None ), + "explicit": _explicit_lineage_block(clone0), } return { "weights": { @@ -1439,6 +1860,36 @@ def _artifact_info(path: Path) -> dict[str, Any]: } +def _verify_artifact_pin( + artifact: dict[str, Any], + *, + pinned_sha256: str | None, + option: str, +) -> None: + measured_sha256 = str(artifact["sha256"]) + if pinned_sha256 is not None and measured_sha256 != pinned_sha256: + raise SystemExit( + f"error: {option} sha mismatch: " + f"measured {measured_sha256}, pinned {pinned_sha256}" + ) + artifact["pin_verified"] = pinned_sha256 is not None + + +def _annotate_ladder_crosswalk_pin(ladder_artifact: dict[str, Any]) -> None: + try: + crosswalk = load_uk_local_area_crosswalk() + expected_sha256 = str(crosswalk["ladder_artifact_sha256"]) + except Exception as error: + ladder_artifact["matches_local_area_crosswalk_pin"] = None + ladder_artifact["local_area_crosswalk_pin_error"] = ( + f"{type(error).__name__}: {error}" + ) + return + ladder_artifact["matches_local_area_crosswalk_pin"] = ( + str(ladder_artifact["sha256"]) == expected_sha256 + ) + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: diff --git a/tools/logbook.py b/tools/logbook.py index 4956799fde..78aaee9d40 100644 --- a/tools/logbook.py +++ b/tools/logbook.py @@ -60,11 +60,12 @@ _PIPELINE_SCOPE_PATTERN = re.compile( r"^(?P[a-z]{2})-(?P[a-z0-9_]+)(?:-[a-z0-9_-]+)?$" ) -#: Mirror of logbook.scope_declared() in the same migration: the ratified +#: Mirror of logbook.scope_declared() in +#: supabase/migrations/20260829000000_logbook_uk_local_scope.sql: the ratified #: scope vocabulary, closed-world. Opening a scope is a reviewed diff here, #: in the migration, and in logbook/README.md -- never a side effect of a #: well-formed pipeline name. -DECLARED_SCOPES = frozenset({"us", "uk/frs"}) +DECLARED_SCOPES = frozenset({"us", "uk/frs", "uk/local"}) def _parser() -> argparse.ArgumentParser: