diff --git a/changelog.d/953-uk-local-authority-input.added.md b/changelog.d/953-uk-local-authority-input.added.md new file mode 100644 index 000000000..1809aa7b3 --- /dev/null +++ b/changelog.d/953-uk-local-authority-input.added.md @@ -0,0 +1 @@ +Supply the engine's household `local_authority` input from the rowwise geography ladder (#953): every assigned April 2023 ONS local authority code resolves to its policyengine-uk `LocalAuthority` member name through the sha-pinned ONS names-and-codes resource (`local_authority_names.json`, `tools/generate_uk_local_authority_names.py`) and the mechanical key rule with one declared alias; the assignment refuses any code the roster does not carry, the ladder gate reports any row whose input disagrees with its code, and the clone verifies the assigned members against the installed engine, so council tax and the bespoke Council Tax Reduction schemes stop seeing the Maidstone default on Microcosm builds. The input is declared on the export allow-list rather than the coverage manifest, which measures the incumbent surface; the engine pin moves to policyengine-uk 2.100.0, the first release whose `LocalAuthority` enum carries the six April 2023 unitaries, so the roster-wide membership check is green at the pin. diff --git a/docs/shared-constants.md b/docs/shared-constants.md index 02442f68e..8d7670652 100644 --- a/docs/shared-constants.md +++ b/docs/shared-constants.md @@ -51,3 +51,13 @@ from microcosm.calibrate.geography_constants import ( Do not add another state FIPS/postal mapping to a country runtime or diagnostics module. Extend the shared definition and its consistency test when supported geographies change. + +The engine's `local_authority` member names are not a constants table: the 361 +April 2023 authority names are published ONS data, held in the sha-pinned +resource `microcosm/build/uk/local_authority_names.json` (regenerate with +`tools/generate_uk_local_authority_names.py`). Only the mechanical key rule and +its declared aliases are code, in +[`microcosm.build.uk_runtime.local_authority_input`](../packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py); +derive an engine key through `local_authority_engine_key` rather than +re-implementing the rule, and resolve codes through +`resolve_local_authority_engine_keys`, which fails closed (microcosm#953). diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 74f67dd47..7e131b0ee 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -39,7 +39,7 @@ us = [ # The UK extra adds the rules engine for local metric generation from a # Microcosm UK H5. Target tables remain explicit inputs, and the base package # still does not import policyengine-uk at import time. -uk = ["policyengine-uk>=2.98.0", "h5py>=3", "tables>=3", "openpyxl>=3.1"] +uk = ["policyengine-uk>=2.100.0", "h5py>=3", "tables>=3", "openpyxl>=3.1"] [project.scripts] microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index de9dd56ef..dd0ff27c1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -307,6 +307,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "local_authority_names.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "target_references.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index aba2d591b..05f8abbf5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -625,6 +625,7 @@ "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", + "household.local_authority", "household.lsoa_code", "household.mortgage_debt", "household.msoa_code", diff --git a/packages/microcosm-build/src/microcosm/build/uk/local_authority_names.json b/packages/microcosm-build/src/microcosm/build/uk/local_authority_names.json new file mode 100644 index 000000000..33f6c2b32 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/local_authority_names.json @@ -0,0 +1,1465 @@ +{ + "schema_version": 1, + "country": "uk", + "kind": "uk_local_authority_names", + "description": "Every UK local authority district on the April 2023 ONS frame, with its ONS display name and the policyengine-uk LocalAuthority member name the rowwise build writes as the household local_authority input. engine_key is the mechanical rule in microcosm.build.uk_runtime.local_authority_input (upper-case, non-alphanumeric runs to one underscore) with the module's declared aliases; regenerate with tools/generate_uk_local_authority_names.py (microcosm#953).", + "source": { + "name": "Local Authority Districts (April 2023) Names and Codes in the United Kingdom", + "publisher": "ONS Open Geography Portal", + "item_id": "e8b361ba9e98418ba8ff2f892d00c352", + "url": "https://open-geography-portalx-ons.hub.arcgis.com/api/download/v1/items/e8b361ba9e98418ba8ff2f892d00c352/csv?layers=0", + "sha256": "6c2d811f50756c459c6a1c7c692ae9262fb4d70df3f535958ef7cdbbbd76cc44", + "columns": [ + "LAD23CD", + "LAD23NM" + ], + "vintage": "2023_april_lad" + }, + "area_count": 361, + "areas": { + "E06000001": { + "name": "Hartlepool", + "engine_key": "HARTLEPOOL" + }, + "E06000002": { + "name": "Middlesbrough", + "engine_key": "MIDDLESBROUGH" + }, + "E06000003": { + "name": "Redcar and Cleveland", + "engine_key": "REDCAR_AND_CLEVELAND" + }, + "E06000004": { + "name": "Stockton-on-Tees", + "engine_key": "STOCKTON_ON_TEES" + }, + "E06000005": { + "name": "Darlington", + "engine_key": "DARLINGTON" + }, + "E06000006": { + "name": "Halton", + "engine_key": "HALTON" + }, + "E06000007": { + "name": "Warrington", + "engine_key": "WARRINGTON" + }, + "E06000008": { + "name": "Blackburn with Darwen", + "engine_key": "BLACKBURN_WITH_DARWEN" + }, + "E06000009": { + "name": "Blackpool", + "engine_key": "BLACKPOOL" + }, + "E06000010": { + "name": "Kingston upon Hull, City of", + "engine_key": "KINGSTON_UPON_HULL_CITY_OF" + }, + "E06000011": { + "name": "East Riding of Yorkshire", + "engine_key": "EAST_RIDING_OF_YORKSHIRE" + }, + "E06000012": { + "name": "North East Lincolnshire", + "engine_key": "NORTH_EAST_LINCOLNSHIRE" + }, + "E06000013": { + "name": "North Lincolnshire", + "engine_key": "NORTH_LINCOLNSHIRE" + }, + "E06000014": { + "name": "York", + "engine_key": "YORK" + }, + "E06000015": { + "name": "Derby", + "engine_key": "DERBY" + }, + "E06000016": { + "name": "Leicester", + "engine_key": "LEICESTER" + }, + "E06000017": { + "name": "Rutland", + "engine_key": "RUTLAND" + }, + "E06000018": { + "name": "Nottingham", + "engine_key": "NOTTINGHAM" + }, + "E06000019": { + "name": "Herefordshire, County of", + "engine_key": "HEREFORDSHIRE_COUNTY_OF" + }, + "E06000020": { + "name": "Telford and Wrekin", + "engine_key": "TELFORD_AND_WREKIN" + }, + "E06000021": { + "name": "Stoke-on-Trent", + "engine_key": "STOKE_ON_TRENT" + }, + "E06000022": { + "name": "Bath and North East Somerset", + "engine_key": "BATH_AND_NORTH_EAST_SOMERSET" + }, + "E06000023": { + "name": "Bristol, City of", + "engine_key": "BRISTOL_CITY_OF" + }, + "E06000024": { + "name": "North Somerset", + "engine_key": "NORTH_SOMERSET" + }, + "E06000025": { + "name": "South Gloucestershire", + "engine_key": "SOUTH_GLOUCESTERSHIRE" + }, + "E06000026": { + "name": "Plymouth", + "engine_key": "PLYMOUTH" + }, + "E06000027": { + "name": "Torbay", + "engine_key": "TORBAY" + }, + "E06000030": { + "name": "Swindon", + "engine_key": "SWINDON" + }, + "E06000031": { + "name": "Peterborough", + "engine_key": "PETERBOROUGH" + }, + "E06000032": { + "name": "Luton", + "engine_key": "LUTON" + }, + "E06000033": { + "name": "Southend-on-Sea", + "engine_key": "SOUTHEND_ON_SEA" + }, + "E06000034": { + "name": "Thurrock", + "engine_key": "THURROCK" + }, + "E06000035": { + "name": "Medway", + "engine_key": "MEDWAY" + }, + "E06000036": { + "name": "Bracknell Forest", + "engine_key": "BRACKNELL_FOREST" + }, + "E06000037": { + "name": "West Berkshire", + "engine_key": "WEST_BERKSHIRE" + }, + "E06000038": { + "name": "Reading", + "engine_key": "READING" + }, + "E06000039": { + "name": "Slough", + "engine_key": "SLOUGH" + }, + "E06000040": { + "name": "Windsor and Maidenhead", + "engine_key": "WINDSOR_AND_MAIDENHEAD" + }, + "E06000041": { + "name": "Wokingham", + "engine_key": "WOKINGHAM" + }, + "E06000042": { + "name": "Milton Keynes", + "engine_key": "MILTON_KEYNES" + }, + "E06000043": { + "name": "Brighton and Hove", + "engine_key": "BRIGHTON_AND_HOVE" + }, + "E06000044": { + "name": "Portsmouth", + "engine_key": "PORTSMOUTH" + }, + "E06000045": { + "name": "Southampton", + "engine_key": "SOUTHAMPTON" + }, + "E06000046": { + "name": "Isle of Wight", + "engine_key": "ISLE_OF_WIGHT" + }, + "E06000047": { + "name": "County Durham", + "engine_key": "COUNTY_DURHAM" + }, + "E06000049": { + "name": "Cheshire East", + "engine_key": "CHESHIRE_EAST" + }, + "E06000050": { + "name": "Cheshire West and Chester", + "engine_key": "CHESHIRE_WEST_AND_CHESTER" + }, + "E06000051": { + "name": "Shropshire", + "engine_key": "SHROPSHIRE" + }, + "E06000052": { + "name": "Cornwall", + "engine_key": "CORNWALL" + }, + "E06000053": { + "name": "Isles of Scilly", + "engine_key": "ISLES_OF_SCILLY" + }, + "E06000054": { + "name": "Wiltshire", + "engine_key": "WILTSHIRE" + }, + "E06000055": { + "name": "Bedford", + "engine_key": "BEDFORD" + }, + "E06000056": { + "name": "Central Bedfordshire", + "engine_key": "CENTRAL_BEDFORDSHIRE" + }, + "E06000057": { + "name": "Northumberland", + "engine_key": "NORTHUMBERLAND" + }, + "E06000058": { + "name": "Bournemouth, Christchurch and Poole", + "engine_key": "BOURNEMOUTH_CHRISTCHURCH_AND_POOLE" + }, + "E06000059": { + "name": "Dorset", + "engine_key": "DORSET" + }, + "E06000060": { + "name": "Buckinghamshire", + "engine_key": "BUCKINGHAMSHIRE" + }, + "E06000061": { + "name": "North Northamptonshire", + "engine_key": "NORTH_NORTHAMPTONSHIRE" + }, + "E06000062": { + "name": "West Northamptonshire", + "engine_key": "WEST_NORTHAMPTONSHIRE" + }, + "E06000063": { + "name": "Cumberland", + "engine_key": "CUMBERLAND" + }, + "E06000064": { + "name": "Westmorland and Furness", + "engine_key": "WESTMORLAND_AND_FURNESS" + }, + "E06000065": { + "name": "North Yorkshire", + "engine_key": "NORTH_YORKSHIRE" + }, + "E06000066": { + "name": "Somerset", + "engine_key": "SOMERSET" + }, + "E07000008": { + "name": "Cambridge", + "engine_key": "CAMBRIDGE" + }, + "E07000009": { + "name": "East Cambridgeshire", + "engine_key": "EAST_CAMBRIDGESHIRE" + }, + "E07000010": { + "name": "Fenland", + "engine_key": "FENLAND" + }, + "E07000011": { + "name": "Huntingdonshire", + "engine_key": "HUNTINGDONSHIRE" + }, + "E07000012": { + "name": "South Cambridgeshire", + "engine_key": "SOUTH_CAMBRIDGESHIRE" + }, + "E07000032": { + "name": "Amber Valley", + "engine_key": "AMBER_VALLEY" + }, + "E07000033": { + "name": "Bolsover", + "engine_key": "BOLSOVER" + }, + "E07000034": { + "name": "Chesterfield", + "engine_key": "CHESTERFIELD" + }, + "E07000035": { + "name": "Derbyshire Dales", + "engine_key": "DERBYSHIRE_DALES" + }, + "E07000036": { + "name": "Erewash", + "engine_key": "EREWASH" + }, + "E07000037": { + "name": "High Peak", + "engine_key": "HIGH_PEAK" + }, + "E07000038": { + "name": "North East Derbyshire", + "engine_key": "NORTH_EAST_DERBYSHIRE" + }, + "E07000039": { + "name": "South Derbyshire", + "engine_key": "SOUTH_DERBYSHIRE" + }, + "E07000040": { + "name": "East Devon", + "engine_key": "EAST_DEVON" + }, + "E07000041": { + "name": "Exeter", + "engine_key": "EXETER" + }, + "E07000042": { + "name": "Mid Devon", + "engine_key": "MID_DEVON" + }, + "E07000043": { + "name": "North Devon", + "engine_key": "NORTH_DEVON" + }, + "E07000044": { + "name": "South Hams", + "engine_key": "SOUTH_HAMS" + }, + "E07000045": { + "name": "Teignbridge", + "engine_key": "TEIGNBRIDGE" + }, + "E07000046": { + "name": "Torridge", + "engine_key": "TORRIDGE" + }, + "E07000047": { + "name": "West Devon", + "engine_key": "WEST_DEVON" + }, + "E07000061": { + "name": "Eastbourne", + "engine_key": "EASTBOURNE" + }, + "E07000062": { + "name": "Hastings", + "engine_key": "HASTINGS" + }, + "E07000063": { + "name": "Lewes", + "engine_key": "LEWES" + }, + "E07000064": { + "name": "Rother", + "engine_key": "ROTHER" + }, + "E07000065": { + "name": "Wealden", + "engine_key": "WEALDEN" + }, + "E07000066": { + "name": "Basildon", + "engine_key": "BASILDON" + }, + "E07000067": { + "name": "Braintree", + "engine_key": "BRAINTREE" + }, + "E07000068": { + "name": "Brentwood", + "engine_key": "BRENTWOOD" + }, + "E07000069": { + "name": "Castle Point", + "engine_key": "CASTLE_POINT" + }, + "E07000070": { + "name": "Chelmsford", + "engine_key": "CHELMSFORD" + }, + "E07000071": { + "name": "Colchester", + "engine_key": "COLCHESTER" + }, + "E07000072": { + "name": "Epping Forest", + "engine_key": "EPPING_FOREST" + }, + "E07000073": { + "name": "Harlow", + "engine_key": "HARLOW" + }, + "E07000074": { + "name": "Maldon", + "engine_key": "MALDON" + }, + "E07000075": { + "name": "Rochford", + "engine_key": "ROCHFORD" + }, + "E07000076": { + "name": "Tendring", + "engine_key": "TENDRING" + }, + "E07000077": { + "name": "Uttlesford", + "engine_key": "UTTLESFORD" + }, + "E07000078": { + "name": "Cheltenham", + "engine_key": "CHELTENHAM" + }, + "E07000079": { + "name": "Cotswold", + "engine_key": "COTSWOLD" + }, + "E07000080": { + "name": "Forest of Dean", + "engine_key": "FOREST_OF_DEAN" + }, + "E07000081": { + "name": "Gloucester", + "engine_key": "GLOUCESTER" + }, + "E07000082": { + "name": "Stroud", + "engine_key": "STROUD" + }, + "E07000083": { + "name": "Tewkesbury", + "engine_key": "TEWKESBURY" + }, + "E07000084": { + "name": "Basingstoke and Deane", + "engine_key": "BASINGSTOKE_AND_DEANE" + }, + "E07000085": { + "name": "East Hampshire", + "engine_key": "EAST_HAMPSHIRE" + }, + "E07000086": { + "name": "Eastleigh", + "engine_key": "EASTLEIGH" + }, + "E07000087": { + "name": "Fareham", + "engine_key": "FAREHAM" + }, + "E07000088": { + "name": "Gosport", + "engine_key": "GOSPORT" + }, + "E07000089": { + "name": "Hart", + "engine_key": "HART" + }, + "E07000090": { + "name": "Havant", + "engine_key": "HAVANT" + }, + "E07000091": { + "name": "New Forest", + "engine_key": "NEW_FOREST" + }, + "E07000092": { + "name": "Rushmoor", + "engine_key": "RUSHMOOR" + }, + "E07000093": { + "name": "Test Valley", + "engine_key": "TEST_VALLEY" + }, + "E07000094": { + "name": "Winchester", + "engine_key": "WINCHESTER" + }, + "E07000095": { + "name": "Broxbourne", + "engine_key": "BROXBOURNE" + }, + "E07000096": { + "name": "Dacorum", + "engine_key": "DACORUM" + }, + "E07000098": { + "name": "Hertsmere", + "engine_key": "HERTSMERE" + }, + "E07000099": { + "name": "North Hertfordshire", + "engine_key": "NORTH_HERTFORDSHIRE" + }, + "E07000102": { + "name": "Three Rivers", + "engine_key": "THREE_RIVERS" + }, + "E07000103": { + "name": "Watford", + "engine_key": "WATFORD" + }, + "E07000105": { + "name": "Ashford", + "engine_key": "ASHFORD" + }, + "E07000106": { + "name": "Canterbury", + "engine_key": "CANTERBURY" + }, + "E07000107": { + "name": "Dartford", + "engine_key": "DARTFORD" + }, + "E07000108": { + "name": "Dover", + "engine_key": "DOVER" + }, + "E07000109": { + "name": "Gravesham", + "engine_key": "GRAVESHAM" + }, + "E07000110": { + "name": "Maidstone", + "engine_key": "MAIDSTONE" + }, + "E07000111": { + "name": "Sevenoaks", + "engine_key": "SEVENOAKS" + }, + "E07000112": { + "name": "Folkestone and Hythe", + "engine_key": "FOLKESTONE_AND_HYTHE" + }, + "E07000113": { + "name": "Swale", + "engine_key": "SWALE" + }, + "E07000114": { + "name": "Thanet", + "engine_key": "THANET" + }, + "E07000115": { + "name": "Tonbridge and Malling", + "engine_key": "TONBRIDGE_AND_MALLING" + }, + "E07000116": { + "name": "Tunbridge Wells", + "engine_key": "TUNBRIDGE_WELLS" + }, + "E07000117": { + "name": "Burnley", + "engine_key": "BURNLEY" + }, + "E07000118": { + "name": "Chorley", + "engine_key": "CHORLEY" + }, + "E07000119": { + "name": "Fylde", + "engine_key": "FYLDE" + }, + "E07000120": { + "name": "Hyndburn", + "engine_key": "HYNDBURN" + }, + "E07000121": { + "name": "Lancaster", + "engine_key": "LANCASTER" + }, + "E07000122": { + "name": "Pendle", + "engine_key": "PENDLE" + }, + "E07000123": { + "name": "Preston", + "engine_key": "PRESTON" + }, + "E07000124": { + "name": "Ribble Valley", + "engine_key": "RIBBLE_VALLEY" + }, + "E07000125": { + "name": "Rossendale", + "engine_key": "ROSSENDALE" + }, + "E07000126": { + "name": "South Ribble", + "engine_key": "SOUTH_RIBBLE" + }, + "E07000127": { + "name": "West Lancashire", + "engine_key": "WEST_LANCASHIRE" + }, + "E07000128": { + "name": "Wyre", + "engine_key": "WYRE" + }, + "E07000129": { + "name": "Blaby", + "engine_key": "BLABY" + }, + "E07000130": { + "name": "Charnwood", + "engine_key": "CHARNWOOD" + }, + "E07000131": { + "name": "Harborough", + "engine_key": "HARBOROUGH" + }, + "E07000132": { + "name": "Hinckley and Bosworth", + "engine_key": "HINCKLEY_AND_BOSWORTH" + }, + "E07000133": { + "name": "Melton", + "engine_key": "MELTON" + }, + "E07000134": { + "name": "North West Leicestershire", + "engine_key": "NORTH_WEST_LEICESTERSHIRE" + }, + "E07000135": { + "name": "Oadby and Wigston", + "engine_key": "OADBY_AND_WIGSTON" + }, + "E07000136": { + "name": "Boston", + "engine_key": "BOSTON" + }, + "E07000137": { + "name": "East Lindsey", + "engine_key": "EAST_LINDSEY" + }, + "E07000138": { + "name": "Lincoln", + "engine_key": "LINCOLN" + }, + "E07000139": { + "name": "North Kesteven", + "engine_key": "NORTH_KESTEVEN" + }, + "E07000140": { + "name": "South Holland", + "engine_key": "SOUTH_HOLLAND" + }, + "E07000141": { + "name": "South Kesteven", + "engine_key": "SOUTH_KESTEVEN" + }, + "E07000142": { + "name": "West Lindsey", + "engine_key": "WEST_LINDSEY" + }, + "E07000143": { + "name": "Breckland", + "engine_key": "BRECKLAND" + }, + "E07000144": { + "name": "Broadland", + "engine_key": "BROADLAND" + }, + "E07000145": { + "name": "Great Yarmouth", + "engine_key": "GREAT_YARMOUTH" + }, + "E07000146": { + "name": "King's Lynn and West Norfolk", + "engine_key": "KINGS_LYNN_AND_WEST_NORFOLK" + }, + "E07000147": { + "name": "North Norfolk", + "engine_key": "NORTH_NORFOLK" + }, + "E07000148": { + "name": "Norwich", + "engine_key": "NORWICH" + }, + "E07000149": { + "name": "South Norfolk", + "engine_key": "SOUTH_NORFOLK" + }, + "E07000170": { + "name": "Ashfield", + "engine_key": "ASHFIELD" + }, + "E07000171": { + "name": "Bassetlaw", + "engine_key": "BASSETLAW" + }, + "E07000172": { + "name": "Broxtowe", + "engine_key": "BROXTOWE" + }, + "E07000173": { + "name": "Gedling", + "engine_key": "GEDLING" + }, + "E07000174": { + "name": "Mansfield", + "engine_key": "MANSFIELD" + }, + "E07000175": { + "name": "Newark and Sherwood", + "engine_key": "NEWARK_AND_SHERWOOD" + }, + "E07000176": { + "name": "Rushcliffe", + "engine_key": "RUSHCLIFFE" + }, + "E07000177": { + "name": "Cherwell", + "engine_key": "CHERWELL" + }, + "E07000178": { + "name": "Oxford", + "engine_key": "OXFORD" + }, + "E07000179": { + "name": "South Oxfordshire", + "engine_key": "SOUTH_OXFORDSHIRE" + }, + "E07000180": { + "name": "Vale of White Horse", + "engine_key": "VALE_OF_WHITE_HORSE" + }, + "E07000181": { + "name": "West Oxfordshire", + "engine_key": "WEST_OXFORDSHIRE" + }, + "E07000192": { + "name": "Cannock Chase", + "engine_key": "CANNOCK_CHASE" + }, + "E07000193": { + "name": "East Staffordshire", + "engine_key": "EAST_STAFFORDSHIRE" + }, + "E07000194": { + "name": "Lichfield", + "engine_key": "LICHFIELD" + }, + "E07000195": { + "name": "Newcastle-under-Lyme", + "engine_key": "NEWCASTLE_UNDER_LYME" + }, + "E07000196": { + "name": "South Staffordshire", + "engine_key": "SOUTH_STAFFORDSHIRE" + }, + "E07000197": { + "name": "Stafford", + "engine_key": "STAFFORD" + }, + "E07000198": { + "name": "Staffordshire Moorlands", + "engine_key": "STAFFORDSHIRE_MOORLANDS" + }, + "E07000199": { + "name": "Tamworth", + "engine_key": "TAMWORTH" + }, + "E07000200": { + "name": "Babergh", + "engine_key": "BABERGH" + }, + "E07000202": { + "name": "Ipswich", + "engine_key": "IPSWICH" + }, + "E07000203": { + "name": "Mid Suffolk", + "engine_key": "MID_SUFFOLK" + }, + "E07000207": { + "name": "Elmbridge", + "engine_key": "ELMBRIDGE" + }, + "E07000208": { + "name": "Epsom and Ewell", + "engine_key": "EPSOM_AND_EWELL" + }, + "E07000209": { + "name": "Guildford", + "engine_key": "GUILDFORD" + }, + "E07000210": { + "name": "Mole Valley", + "engine_key": "MOLE_VALLEY" + }, + "E07000211": { + "name": "Reigate and Banstead", + "engine_key": "REIGATE_AND_BANSTEAD" + }, + "E07000212": { + "name": "Runnymede", + "engine_key": "RUNNYMEDE" + }, + "E07000213": { + "name": "Spelthorne", + "engine_key": "SPELTHORNE" + }, + "E07000214": { + "name": "Surrey Heath", + "engine_key": "SURREY_HEATH" + }, + "E07000215": { + "name": "Tandridge", + "engine_key": "TANDRIDGE" + }, + "E07000216": { + "name": "Waverley", + "engine_key": "WAVERLEY" + }, + "E07000217": { + "name": "Woking", + "engine_key": "WOKING" + }, + "E07000218": { + "name": "North Warwickshire", + "engine_key": "NORTH_WARWICKSHIRE" + }, + "E07000219": { + "name": "Nuneaton and Bedworth", + "engine_key": "NUNEATON_AND_BEDWORTH" + }, + "E07000220": { + "name": "Rugby", + "engine_key": "RUGBY" + }, + "E07000221": { + "name": "Stratford-on-Avon", + "engine_key": "STRATFORD_ON_AVON" + }, + "E07000222": { + "name": "Warwick", + "engine_key": "WARWICK" + }, + "E07000223": { + "name": "Adur", + "engine_key": "ADUR" + }, + "E07000224": { + "name": "Arun", + "engine_key": "ARUN" + }, + "E07000225": { + "name": "Chichester", + "engine_key": "CHICHESTER" + }, + "E07000226": { + "name": "Crawley", + "engine_key": "CRAWLEY" + }, + "E07000227": { + "name": "Horsham", + "engine_key": "HORSHAM" + }, + "E07000228": { + "name": "Mid Sussex", + "engine_key": "MID_SUSSEX" + }, + "E07000229": { + "name": "Worthing", + "engine_key": "WORTHING" + }, + "E07000234": { + "name": "Bromsgrove", + "engine_key": "BROMSGROVE" + }, + "E07000235": { + "name": "Malvern Hills", + "engine_key": "MALVERN_HILLS" + }, + "E07000236": { + "name": "Redditch", + "engine_key": "REDDITCH" + }, + "E07000237": { + "name": "Worcester", + "engine_key": "WORCESTER" + }, + "E07000238": { + "name": "Wychavon", + "engine_key": "WYCHAVON" + }, + "E07000239": { + "name": "Wyre Forest", + "engine_key": "WYRE_FOREST" + }, + "E07000240": { + "name": "St Albans", + "engine_key": "ST_ALBANS" + }, + "E07000241": { + "name": "Welwyn Hatfield", + "engine_key": "WELWYN_HATFIELD" + }, + "E07000242": { + "name": "East Hertfordshire", + "engine_key": "EAST_HERTFORDSHIRE" + }, + "E07000243": { + "name": "Stevenage", + "engine_key": "STEVENAGE" + }, + "E07000244": { + "name": "East Suffolk", + "engine_key": "EAST_SUFFOLK" + }, + "E07000245": { + "name": "West Suffolk", + "engine_key": "WEST_SUFFOLK" + }, + "E08000001": { + "name": "Bolton", + "engine_key": "BOLTON" + }, + "E08000002": { + "name": "Bury", + "engine_key": "BURY" + }, + "E08000003": { + "name": "Manchester", + "engine_key": "MANCHESTER" + }, + "E08000004": { + "name": "Oldham", + "engine_key": "OLDHAM" + }, + "E08000005": { + "name": "Rochdale", + "engine_key": "ROCHDALE" + }, + "E08000006": { + "name": "Salford", + "engine_key": "SALFORD" + }, + "E08000007": { + "name": "Stockport", + "engine_key": "STOCKPORT" + }, + "E08000008": { + "name": "Tameside", + "engine_key": "TAMESIDE" + }, + "E08000009": { + "name": "Trafford", + "engine_key": "TRAFFORD" + }, + "E08000010": { + "name": "Wigan", + "engine_key": "WIGAN" + }, + "E08000011": { + "name": "Knowsley", + "engine_key": "KNOWSLEY" + }, + "E08000012": { + "name": "Liverpool", + "engine_key": "LIVERPOOL" + }, + "E08000013": { + "name": "St. Helens", + "engine_key": "ST_HELENS" + }, + "E08000014": { + "name": "Sefton", + "engine_key": "SEFTON" + }, + "E08000015": { + "name": "Wirral", + "engine_key": "WIRRAL" + }, + "E08000016": { + "name": "Barnsley", + "engine_key": "BARNSLEY" + }, + "E08000017": { + "name": "Doncaster", + "engine_key": "DONCASTER" + }, + "E08000018": { + "name": "Rotherham", + "engine_key": "ROTHERHAM" + }, + "E08000019": { + "name": "Sheffield", + "engine_key": "SHEFFIELD" + }, + "E08000021": { + "name": "Newcastle upon Tyne", + "engine_key": "NEWCASTLE_UPON_TYNE" + }, + "E08000022": { + "name": "North Tyneside", + "engine_key": "NORTH_TYNESIDE" + }, + "E08000023": { + "name": "South Tyneside", + "engine_key": "SOUTH_TYNESIDE" + }, + "E08000024": { + "name": "Sunderland", + "engine_key": "SUNDERLAND" + }, + "E08000025": { + "name": "Birmingham", + "engine_key": "BIRMINGHAM" + }, + "E08000026": { + "name": "Coventry", + "engine_key": "COVENTRY" + }, + "E08000027": { + "name": "Dudley", + "engine_key": "DUDLEY" + }, + "E08000028": { + "name": "Sandwell", + "engine_key": "SANDWELL" + }, + "E08000029": { + "name": "Solihull", + "engine_key": "SOLIHULL" + }, + "E08000030": { + "name": "Walsall", + "engine_key": "WALSALL" + }, + "E08000031": { + "name": "Wolverhampton", + "engine_key": "WOLVERHAMPTON" + }, + "E08000032": { + "name": "Bradford", + "engine_key": "BRADFORD" + }, + "E08000033": { + "name": "Calderdale", + "engine_key": "CALDERDALE" + }, + "E08000034": { + "name": "Kirklees", + "engine_key": "KIRKLEES" + }, + "E08000035": { + "name": "Leeds", + "engine_key": "LEEDS" + }, + "E08000036": { + "name": "Wakefield", + "engine_key": "WAKEFIELD" + }, + "E08000037": { + "name": "Gateshead", + "engine_key": "GATESHEAD" + }, + "E09000001": { + "name": "City of London", + "engine_key": "CITY_OF_LONDON" + }, + "E09000002": { + "name": "Barking and Dagenham", + "engine_key": "BARKING_AND_DAGENHAM" + }, + "E09000003": { + "name": "Barnet", + "engine_key": "BARNET" + }, + "E09000004": { + "name": "Bexley", + "engine_key": "BEXLEY" + }, + "E09000005": { + "name": "Brent", + "engine_key": "BRENT" + }, + "E09000006": { + "name": "Bromley", + "engine_key": "BROMLEY" + }, + "E09000007": { + "name": "Camden", + "engine_key": "CAMDEN" + }, + "E09000008": { + "name": "Croydon", + "engine_key": "CROYDON" + }, + "E09000009": { + "name": "Ealing", + "engine_key": "EALING" + }, + "E09000010": { + "name": "Enfield", + "engine_key": "ENFIELD" + }, + "E09000011": { + "name": "Greenwich", + "engine_key": "GREENWICH" + }, + "E09000012": { + "name": "Hackney", + "engine_key": "HACKNEY" + }, + "E09000013": { + "name": "Hammersmith and Fulham", + "engine_key": "HAMMERSMITH_AND_FULHAM" + }, + "E09000014": { + "name": "Haringey", + "engine_key": "HARINGEY" + }, + "E09000015": { + "name": "Harrow", + "engine_key": "HARROW" + }, + "E09000016": { + "name": "Havering", + "engine_key": "HAVERING" + }, + "E09000017": { + "name": "Hillingdon", + "engine_key": "HILLINGDON" + }, + "E09000018": { + "name": "Hounslow", + "engine_key": "HOUNSLOW" + }, + "E09000019": { + "name": "Islington", + "engine_key": "ISLINGTON" + }, + "E09000020": { + "name": "Kensington and Chelsea", + "engine_key": "KENSINGTON_AND_CHELSEA" + }, + "E09000021": { + "name": "Kingston upon Thames", + "engine_key": "KINGSTON_UPON_THAMES" + }, + "E09000022": { + "name": "Lambeth", + "engine_key": "LAMBETH" + }, + "E09000023": { + "name": "Lewisham", + "engine_key": "LEWISHAM" + }, + "E09000024": { + "name": "Merton", + "engine_key": "MERTON" + }, + "E09000025": { + "name": "Newham", + "engine_key": "NEWHAM" + }, + "E09000026": { + "name": "Redbridge", + "engine_key": "REDBRIDGE" + }, + "E09000027": { + "name": "Richmond upon Thames", + "engine_key": "RICHMOND_UPON_THAMES" + }, + "E09000028": { + "name": "Southwark", + "engine_key": "SOUTHWARK" + }, + "E09000029": { + "name": "Sutton", + "engine_key": "SUTTON" + }, + "E09000030": { + "name": "Tower Hamlets", + "engine_key": "TOWER_HAMLETS" + }, + "E09000031": { + "name": "Waltham Forest", + "engine_key": "WALTHAM_FOREST" + }, + "E09000032": { + "name": "Wandsworth", + "engine_key": "WANDSWORTH" + }, + "E09000033": { + "name": "Westminster", + "engine_key": "WESTMINSTER" + }, + "N09000001": { + "name": "Antrim and Newtownabbey", + "engine_key": "ANTRIM_AND_NEWTOWNABBEY" + }, + "N09000002": { + "name": "Armagh City, Banbridge and Craigavon", + "engine_key": "ARMAGH_CITY_BANBRIDGE_AND_CRAIGAVON" + }, + "N09000003": { + "name": "Belfast", + "engine_key": "BELFAST" + }, + "N09000004": { + "name": "Causeway Coast and Glens", + "engine_key": "CAUSEWAY_COAST_AND_GLENS" + }, + "N09000005": { + "name": "Derry City and Strabane", + "engine_key": "DERRY_CITY_AND_STRABANE" + }, + "N09000006": { + "name": "Fermanagh and Omagh", + "engine_key": "FERMANAGH_AND_OMAGH" + }, + "N09000007": { + "name": "Lisburn and Castlereagh", + "engine_key": "LISBURN_AND_CASTLEREAGH" + }, + "N09000008": { + "name": "Mid and East Antrim", + "engine_key": "MID_AND_EAST_ANTRIM" + }, + "N09000009": { + "name": "Mid Ulster", + "engine_key": "MID_ULSTER" + }, + "N09000010": { + "name": "Newry, Mourne and Down", + "engine_key": "NEWRY_MOURNE_AND_DOWN" + }, + "N09000011": { + "name": "Ards and North Down", + "engine_key": "ARDS_AND_NORTH_DOWN" + }, + "S12000005": { + "name": "Clackmannanshire", + "engine_key": "CLACKMANNANSHIRE" + }, + "S12000006": { + "name": "Dumfries and Galloway", + "engine_key": "DUMFRIES_AND_GALLOWAY" + }, + "S12000008": { + "name": "East Ayrshire", + "engine_key": "EAST_AYRSHIRE" + }, + "S12000010": { + "name": "East Lothian", + "engine_key": "EAST_LOTHIAN" + }, + "S12000011": { + "name": "East Renfrewshire", + "engine_key": "EAST_RENFREWSHIRE" + }, + "S12000013": { + "name": "Na h-Eileanan Siar", + "engine_key": "NA_H_EILEANAN_SIAR" + }, + "S12000014": { + "name": "Falkirk", + "engine_key": "FALKIRK" + }, + "S12000017": { + "name": "Highland", + "engine_key": "HIGHLAND" + }, + "S12000018": { + "name": "Inverclyde", + "engine_key": "INVERCLYDE" + }, + "S12000019": { + "name": "Midlothian", + "engine_key": "MIDLOTHIAN" + }, + "S12000020": { + "name": "Moray", + "engine_key": "MORAY" + }, + "S12000021": { + "name": "North Ayrshire", + "engine_key": "NORTH_AYRSHIRE" + }, + "S12000023": { + "name": "Orkney Islands", + "engine_key": "ORKNEY_ISLANDS" + }, + "S12000026": { + "name": "Scottish Borders", + "engine_key": "SCOTTISH_BORDERS" + }, + "S12000027": { + "name": "Shetland Islands", + "engine_key": "SHETLAND_ISLANDS" + }, + "S12000028": { + "name": "South Ayrshire", + "engine_key": "SOUTH_AYRSHIRE" + }, + "S12000029": { + "name": "South Lanarkshire", + "engine_key": "SOUTH_LANARKSHIRE" + }, + "S12000030": { + "name": "Stirling", + "engine_key": "STIRLING" + }, + "S12000033": { + "name": "Aberdeen City", + "engine_key": "ABERDEEN_CITY" + }, + "S12000034": { + "name": "Aberdeenshire", + "engine_key": "ABERDEENSHIRE" + }, + "S12000035": { + "name": "Argyll and Bute", + "engine_key": "ARGYLL_AND_BUTE" + }, + "S12000036": { + "name": "City of Edinburgh", + "engine_key": "CITY_OF_EDINBURGH" + }, + "S12000038": { + "name": "Renfrewshire", + "engine_key": "RENFREWSHIRE" + }, + "S12000039": { + "name": "West Dunbartonshire", + "engine_key": "WEST_DUNBARTONSHIRE" + }, + "S12000040": { + "name": "West Lothian", + "engine_key": "WEST_LOTHIAN" + }, + "S12000041": { + "name": "Angus", + "engine_key": "ANGUS" + }, + "S12000042": { + "name": "Dundee City", + "engine_key": "DUNDEE_CITY" + }, + "S12000045": { + "name": "East Dunbartonshire", + "engine_key": "EAST_DUNBARTONSHIRE" + }, + "S12000047": { + "name": "Fife", + "engine_key": "FIFE" + }, + "S12000048": { + "name": "Perth and Kinross", + "engine_key": "PERTH_AND_KINROSS" + }, + "S12000049": { + "name": "Glasgow City", + "engine_key": "GLASGOW_CITY" + }, + "S12000050": { + "name": "North Lanarkshire", + "engine_key": "NORTH_LANARKSHIRE" + }, + "W06000001": { + "name": "Isle of Anglesey", + "engine_key": "ISLE_OF_ANGLESEY" + }, + "W06000002": { + "name": "Gwynedd", + "engine_key": "GWYNEDD" + }, + "W06000003": { + "name": "Conwy", + "engine_key": "CONWY" + }, + "W06000004": { + "name": "Denbighshire", + "engine_key": "DENBIGHSHIRE" + }, + "W06000005": { + "name": "Flintshire", + "engine_key": "FLINTSHIRE" + }, + "W06000006": { + "name": "Wrexham", + "engine_key": "WREXHAM" + }, + "W06000008": { + "name": "Ceredigion", + "engine_key": "CEREDIGION" + }, + "W06000009": { + "name": "Pembrokeshire", + "engine_key": "PEMBROKESHIRE" + }, + "W06000010": { + "name": "Carmarthenshire", + "engine_key": "CARMARTHENSHIRE" + }, + "W06000011": { + "name": "Swansea", + "engine_key": "SWANSEA" + }, + "W06000012": { + "name": "Neath Port Talbot", + "engine_key": "NEATH_PORT_TALBOT" + }, + "W06000013": { + "name": "Bridgend", + "engine_key": "BRIDGEND" + }, + "W06000014": { + "name": "Vale of Glamorgan", + "engine_key": "VALE_OF_GLAMORGAN" + }, + "W06000015": { + "name": "Cardiff", + "engine_key": "CARDIFF" + }, + "W06000016": { + "name": "Rhondda Cynon Taf", + "engine_key": "RHONDDA_CYNON_TAF" + }, + "W06000018": { + "name": "Caerphilly", + "engine_key": "CAERPHILLY" + }, + "W06000019": { + "name": "Blaenau Gwent", + "engine_key": "BLAENAU_GWENT" + }, + "W06000020": { + "name": "Torfaen", + "engine_key": "TORFAEN" + }, + "W06000021": { + "name": "Monmouthshire", + "engine_key": "MONMOUTHSHIRE" + }, + "W06000022": { + "name": "Newport", + "engine_key": "NEWPORT" + }, + "W06000023": { + "name": "Powys", + "engine_key": "POWYS" + }, + "W06000024": { + "name": "Merthyr Tydfil", + "engine_key": "MERTHYR_TYDFIL" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/geography.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/geography.yaml index 807b7e8e9..8adebf59e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/geography.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/geography.yaml @@ -21,6 +21,7 @@ assignment: - lsoa_code - msoa_code - local_authority_code + - local_authority - ward_code - constituency_code - region_code diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index b21e0c973..0473ee95c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -176,7 +176,11 @@ EW_OA_LAD23_URL, EW_OA_POPULATION_URL, EW_OA_WARD_URL, + LAD23_COUNT, LAD23_ITL_URL, + LAD23_NAMES_ITEM_ID, + LAD23_NAMES_SHA256, + LAD23_NAMES_URL, NI_DZ2021_COUNT, NI_DZ_GEOJSON_ZIP_URL, NI_DZ_HOUSEHOLDS_CSV_URL, @@ -203,6 +207,7 @@ load_england_wales_oa_population, load_england_wales_oa_ward_lookup, load_ew_oa_lad23_lookup, + load_lad23_names_lookup, load_lad_itl_lookup, load_ni_dz_hierarchy, load_ni_dz_households, @@ -215,7 +220,9 @@ load_scotland_oa_lau_lookup, load_scotland_oa_population, load_scotland_oa_ward_lookup, + normalise_lad23_names, update_england_wales_lad_codes, + verify_lad23_names_bytes, write_geography_crosswalk, ) from microcosm.build.uk_runtime.hmrc_calibration import ( @@ -282,6 +289,22 @@ uk_ledger_households_total, uk_local_target_surface, ) +from microcosm.build.uk_runtime.local_authority_input import ( + LOCAL_AUTHORITY_ENGINE_KEY_ALIASES, + UK_LOCAL_AUTHORITY_CODE_COLUMN, + UK_LOCAL_AUTHORITY_INPUT_COLUMN, + UK_LOCAL_AUTHORITY_NAMES_KIND, + UK_LOCAL_AUTHORITY_NAMES_RESOURCE, + UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION, + UK_LOCAL_AUTHORITY_VINTAGE, + load_uk_local_authority_names_resource, + local_authority_consistency_failures, + local_authority_engine_key, + local_authority_engine_key_by_code, + local_authority_keys_missing_from_engine, + resolve_local_authority_engine_keys, + verify_local_authority_engine_domain, +) from microcosm.build.uk_runtime.local_doctrine import ( UK_LOCAL_CLONE_COUNT, UK_LOCAL_MAX_WEIGHT_RATIO, @@ -1009,4 +1032,25 @@ "resolve_uk_year_rule", "uk_weight_ratio_gate", "uk_zero_weight_strata_gate", + "LAD23_COUNT", + "LAD23_NAMES_ITEM_ID", + "LAD23_NAMES_SHA256", + "LAD23_NAMES_URL", + "load_lad23_names_lookup", + "normalise_lad23_names", + "LOCAL_AUTHORITY_ENGINE_KEY_ALIASES", + "UK_LOCAL_AUTHORITY_CODE_COLUMN", + "UK_LOCAL_AUTHORITY_INPUT_COLUMN", + "UK_LOCAL_AUTHORITY_NAMES_KIND", + "UK_LOCAL_AUTHORITY_NAMES_RESOURCE", + "UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION", + "UK_LOCAL_AUTHORITY_VINTAGE", + "load_uk_local_authority_names_resource", + "local_authority_consistency_failures", + "local_authority_engine_key", + "local_authority_engine_key_by_code", + "local_authority_keys_missing_from_engine", + "resolve_local_authority_engine_keys", + "verify_lad23_names_bytes", + "verify_local_authority_engine_domain", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index 58e26f583..fb64b2917 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -36,8 +36,11 @@ checks exist to forbid (the #205 lesson). Column names are policyengine-uk household *inputs* where one exists -(``region`` is the pre-assigned enum input; the ladder never overwrites it), -and plain data columns otherwise: policyengine-uk has no OA/LSOA/MSOA/ward/ +(``region`` is the pre-assigned enum input; the ladder never overwrites it; +``local_authority`` is the enum input the ladder itself writes, resolved from +the assigned April 2023 ONS code through the pinned names resource, see +:mod:`microcosm.build.uk_runtime.local_authority_input`, microcosm#953), and +plain data columns otherwise: policyengine-uk has no OA/LSOA/MSOA/ward/ constituency/ITL input variable, so those ride as ``*_code`` data columns. The exported artifact never carries a formula output — ``country`` recomputes from ``region`` in the engine, so it is never persisted (the #34 regression). @@ -87,6 +90,11 @@ import pandas as pd from microcosm.build.gates import GateResult +from microcosm.build.uk_runtime.local_authority_input import ( + UK_LOCAL_AUTHORITY_VINTAGE, + local_authority_consistency_failures, + resolve_local_authority_engine_keys, +) from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE from microcosm.calibrate.geography_constants import ( UK_LADDER_NATION_REGION_CODES, @@ -108,14 +116,16 @@ ) #: Household spine columns the ladder assignment writes, in write order. Every -#: name is either a policyengine-uk household input (none finer than ``region`` -#: exists, and ``region`` is pre-assigned, so it is not rewritten) or a plain -#: ``*_code`` data column carrying an ONS GSS code. +#: name is either a policyengine-uk household input (``local_authority``, the +#: enum member name resolved from ``local_authority_code``; ``region`` is +#: pre-assigned, so it is not rewritten) or a plain ``*_code`` data column +#: carrying an ONS GSS code. UK_GEOGRAPHY_LADDER_COLUMNS = ( "oa_code", "lsoa_code", "msoa_code", "local_authority_code", + "local_authority", "ward_code", "constituency_code", "region_code", @@ -431,6 +441,17 @@ def assign_uk_geography_ladder( "match the vintage household constituencies were assigned under " f"({expected_constituency_vintage!r})." ) + # The engine input resolves through the April 2023 names resource, so a + # ladder rebuilt on another local authority frame refuses by vintage here + # rather than surfacing later as off-roster codes (microcosm#953). + local_authority_vintage = ladder.layer_vintages["local_authority"] + if local_authority_vintage != UK_LOCAL_AUTHORITY_VINTAGE: + raise ValueError( + f"UK OA ladder local authority vintage {local_authority_vintage!r} " + f"is not {UK_LOCAL_AUTHORITY_VINTAGE!r}, the vintage of the names " + "resource that resolves local_authority; regenerate " + "local_authority_names.json for the new roster first." + ) region_codes = _validated_household_ladder_region_codes( household, @@ -468,6 +489,12 @@ def assign_uk_geography_ladder( assigned["local_authority_code"] = ladder.local_authority_code[ assigned_index ].astype(object) + # The engine's household enum input, fail closed: a code the April 2023 + # names resource does not carry raises here rather than falling to the + # engine default (microcosm#953). + assigned["local_authority"] = resolve_local_authority_engine_keys( + assigned["local_authority_code"] + ) assigned["ward_code"] = ladder.ward_code[assigned_index].astype(object) assigned["constituency_code"] = ladder.constituency_code[assigned_index].astype( object @@ -725,6 +752,11 @@ def uk_geography_ladder_gate( f"GSS codes; examples {sorted(set(values[bad]))[:5]}" ) + # The engine input must agree with the code it was resolved from on every + # row, and every code must sit on the April 2023 roster; reported, not + # raised, so the local battery's rerun of this gate yields a verdict. + failures.extend(local_authority_consistency_failures(household)) + itl3 = household["itl3_code"].astype(str).to_numpy() itl2 = household["itl2_code"].astype(str).to_numpy() itl1 = household["itl1_code"].astype(str).to_numpy() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_sources.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_sources.py index 016b8ca09..e1bc1910e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_sources.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_sources.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib import io import json import time @@ -103,8 +104,24 @@ "https://open-geography-portalx-ons.hub.arcgis.com/api/download/v1/items/" "02b4942973374f039ec1d2e7d35c16a9/csv?layers=0" ) +# "Local Authority Districts (April 2023) Names and Codes in the United +# Kingdom" — ONS Open Geography Portal item e8b361ba.... The names source for +# the engine's ``local_authority`` input (microcosm#953). One file on the +# April 2023 LAD frame covers all four nations: Scottish council-area and +# Northern Irish district codes are unchanged since ca_2019 / lgd_2014, so the +# ladder's three roster vintages resolve through a single lookup. +LAD23_NAMES_ITEM_ID = "e8b361ba9e98418ba8ff2f892d00c352" +LAD23_NAMES_URL = ( + "https://open-geography-portalx-ons.hub.arcgis.com/api/download/v1/items/" + f"{LAD23_NAMES_ITEM_ID}/csv?layers=0" +) +#: sha256 of the published CSV bytes (361 rows, LAD23CD/LAD23NM/LAD23NMW/ +#: ObjectId). The loader, the names generator and the committed resource all +#: assert this digest, so a re-published lookup is a reviewed change here. +LAD23_NAMES_SHA256 = "6c2d811f50756c459c6a1c7c692ae9262fb4d70df3f535958ef7cdbbbd76cc44" ENGLAND_WALES_OA2021_COUNT = 188_880 +LAD23_COUNT = 361 SCOTLAND_OA2022_COUNT = 46_363 NI_DZ2021_COUNT = 3_780 NI_PARLCON24_COUNT = 18 @@ -1343,6 +1360,65 @@ def _normalise_scotland_population( return population.reset_index(drop=True) +def load_lad23_names_lookup(url: str = LAD23_NAMES_URL) -> pd.DataFrame: + """Load the ONS April 2023 local authority names-and-codes lookup. + + Returns ``local_authority_code`` and ``local_authority_name`` for every UK + local authority district on the April 2023 frame (361 rows across England, + Wales, Scotland and Northern Ireland), after the downloaded bytes match + ``LAD23_NAMES_SHA256``. The Welsh-language name column (``LAD23NMW``) is + not carried: the engine's ``LocalAuthority`` member names derive from the + English display name (microcosm#953). + """ + + payload = verify_lad23_names_bytes(_read_url_bytes(url)) + return normalise_lad23_names(pd.read_csv(io.BytesIO(payload), dtype=str)) + + +def verify_lad23_names_bytes(payload: bytes) -> bytes: + """Return ``payload`` if its sha256 is the pinned ``LAD23_NAMES_SHA256``.""" + + observed = hashlib.sha256(payload).hexdigest() + if observed != LAD23_NAMES_SHA256: + raise ValueError( + "LAD23 names lookup digest mismatch: expected " + f"{LAD23_NAMES_SHA256}, got {observed}. A re-published lookup " + "must be re-pinned in geography_sources.LAD23_NAMES_SHA256 and the " + "names resource regenerated (microcosm#953)." + ) + return payload + + +def normalise_lad23_names(frame: pd.DataFrame) -> pd.DataFrame: + """Normalise an ONS LAD23 names-and-codes table to code and name columns. + + Refuses a table without ``LAD23CD``/``LAD23NM``, blank codes or names, + duplicate codes, and any roster size other than ``LAD23_COUNT``. + """ + + upper_to_column = { + str(column).strip().lstrip("\ufeff").upper(): column for column in frame + } + code_column = upper_to_column.get("LAD23CD") + name_column = upper_to_column.get("LAD23NM") + if code_column is None or name_column is None: + raise ValueError("LAD23 names lookup is missing LAD23CD or LAD23NM columns.") + lookup = pd.DataFrame( + { + "local_authority_code": frame[code_column], + "local_authority_name": frame[name_column], + } + ) + return _normalise_code_rows( + lookup, + label="LAD23 names lookup", + unique_column="local_authority_code", + expected_count=LAD23_COUNT, + unit_label="LAD23", + prefixes=("E", "W", "S", "N"), + ) + + def _normalise_ew_lad_lookup(frame: pd.DataFrame) -> pd.DataFrame: column_map = {} for column in frame.columns: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py new file mode 100644 index 000000000..dbe56dc70 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py @@ -0,0 +1,291 @@ +"""The engine's ``local_authority`` input, resolved from the ladder's GSS codes. + +policyengine-uk reads a household enum input ``local_authority`` +(``LocalAuthority``: one member per UK local authority district, default +``MAIDSTONE``). The rowwise geography ladder writes every household's April +2023 ONS code as ``local_authority_code``; this module resolves that code to +the enum member name the engine loader expects, through the committed ONS +names-and-codes resource ``local_authority_names.json`` (published data; +``geography_sources.LAD23_NAMES_SHA256`` pins the lookup bytes, and the +generator, the loader and this resource all assert it) and the mechanical +key rule below (code, with one declared alias). + +Fail closed, never default: a code the resource does not carry raises at the +write point; :func:`uk_geography_ladder_gate` reports any row whose +``local_authority`` disagrees with its code; and the rowwise clone verifies +the assigned member names against the installed engine +(:func:`verify_local_authority_engine_domain`), with the whole roster asserted +by ``test_uk_local_authority_input.py``. No battery ``enum_domain`` gate is +declared for this column: that family needs a ``rules_engine`` evidence +artifact the rowwise local battery never supplies, and +``uk_local_geography_ladder_post_calibration`` already reruns the ladder gate, +consistency check included, on the calibrated candidate (microcosm#953). +""" + +from __future__ import annotations + +import functools +import json +import re +from collections.abc import Iterable, Mapping +from importlib import resources as importlib_resources +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +import microcosm.build.uk_runtime.geography_sources as geography_sources + +UK_LOCAL_AUTHORITY_INPUT_COLUMN = "local_authority" +UK_LOCAL_AUTHORITY_CODE_COLUMN = "local_authority_code" +UK_LOCAL_AUTHORITY_NAMES_RESOURCE = "local_authority_names.json" +UK_LOCAL_AUTHORITY_NAMES_KIND = "uk_local_authority_names" +UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION = 1 +UK_LOCAL_AUTHORITY_VINTAGE = "2023_april_lad" + +#: ONS display names whose ``LocalAuthority`` member name is not the +#: mechanical rule's output. The engine drops the apostrophe where the rule +#: would keep a separator. +LOCAL_AUTHORITY_ENGINE_KEY_ALIASES: Mapping[str, str] = MappingProxyType( + {"King's Lynn and West Norfolk": "KINGS_LYNN_AND_WEST_NORFOLK"} +) + +_NON_ALPHANUMERIC = re.compile(r"[^A-Z0-9]+") +_GSS_CODE = re.compile(r"^[ENSW]\d{8}$") + + +def local_authority_engine_key(name: object) -> str: + """Return the ``LocalAuthority`` member name for an ONS display name. + + Upper-case, collapse every non-alphanumeric run to one underscore and strip + the ends (``Bristol, City of`` -> ``BRISTOL_CITY_OF``, ``Na h-Eileanan + Siar`` -> ``NA_H_EILEANAN_SIAR``). ``LOCAL_AUTHORITY_ENGINE_KEY_ALIASES`` + overrides the rule for the members the engine spells differently. + """ + + text = str(name).strip() + alias = LOCAL_AUTHORITY_ENGINE_KEY_ALIASES.get(text) + if alias is not None: + return alias + key = _NON_ALPHANUMERIC.sub("_", text.upper()).strip("_") + if not key: + raise ValueError(f"Local authority name {name!r} yields an empty engine key.") + return key + + +@functools.cache +def load_uk_local_authority_names_resource() -> Mapping[str, Any]: + """Load and validate the committed ONS April 2023 names resource.""" + + payload = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath(UK_LOCAL_AUTHORITY_NAMES_RESOURCE) + .read_text(encoding="utf-8") + ) + label = UK_LOCAL_AUTHORITY_NAMES_RESOURCE + if payload.get("schema_version") != UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION: + raise ValueError(f"{label}: unsupported schema_version.") + if payload.get("kind") != UK_LOCAL_AUTHORITY_NAMES_KIND: + raise ValueError(f"{label}: kind must be {UK_LOCAL_AUTHORITY_NAMES_KIND!r}.") + if payload.get("country") != "uk": + raise ValueError(f"{label}: country must be 'uk'.") + source = payload.get("source") + if not isinstance(source, Mapping): + raise ValueError(f"{label}: source block is missing.") + if source.get("sha256") != geography_sources.LAD23_NAMES_SHA256: + raise ValueError( + f"{label}: source sha256 {source.get('sha256')!r} is not the pinned " + f"LAD23 names digest {geography_sources.LAD23_NAMES_SHA256}." + ) + if source.get("url") != geography_sources.LAD23_NAMES_URL: + raise ValueError(f"{label}: source url is not LAD23_NAMES_URL.") + if source.get("vintage") != UK_LOCAL_AUTHORITY_VINTAGE: + raise ValueError( + f"{label}: source vintage must be {UK_LOCAL_AUTHORITY_VINTAGE!r}." + ) + areas = payload.get("areas") + if not isinstance(areas, Mapping) or not areas: + raise ValueError(f"{label}: areas must be a non-empty mapping.") + if payload.get("area_count") != len(areas): + raise ValueError(f"{label}: area_count disagrees with the areas mapping.") + keys: dict[str, str] = {} + for code, entry in areas.items(): + if not _GSS_CODE.match(str(code)): + raise ValueError(f"{label}: {code!r} is not a GSS local authority code.") + name = str(entry.get("name", "")).strip() if isinstance(entry, Mapping) else "" + key = ( + str(entry.get("engine_key", "")).strip() + if isinstance(entry, Mapping) + else "" + ) + if not name or not key: + raise ValueError(f"{label}: {code} must carry a name and an engine_key.") + if key != local_authority_engine_key(name): + raise ValueError( + f"{label}: {code} engine_key {key!r} is not the key rule's output " + f"for {name!r}." + ) + if key in keys: + raise ValueError( + f"{label}: engine_key {key!r} is shared by {keys[key]} and {code}." + ) + keys[key] = str(code) + return MappingProxyType(payload) + + +@functools.cache +def local_authority_engine_key_by_code() -> Mapping[str, str]: + """The resource's derived view: GSS code -> ``LocalAuthority`` member name.""" + + areas = load_uk_local_authority_names_resource()["areas"] + return MappingProxyType( + {str(code): str(entry["engine_key"]) for code, entry in sorted(areas.items())} + ) + + +def resolve_local_authority_engine_keys(codes: Iterable[object]) -> np.ndarray: + """Map GSS local authority codes to engine member names, failing closed. + + Every code must sit on the April 2023 roster the resource carries; an + unknown or blank code raises naming it. The engine input is never + defaulted (microcosm#953). + """ + + values = pd.Series( + np.asarray(pd.Series(codes, dtype=object).to_numpy(), dtype=object) + ) + text = values.fillna("").astype(str).str.strip() + mapped = text.map(local_authority_engine_key_by_code()) + unknown = mapped.isna().to_numpy() + if unknown.any(): + bad = sorted(set(text.to_numpy()[unknown].tolist())) + raise ValueError( + "local_authority_code value(s) are not on the April 2023 local " + f"authority roster ({len(bad)} distinct): {bad[:10]}; the engine " + "input local_authority is never defaulted (microcosm#953)." + ) + return mapped.to_numpy(dtype=object) + + +def local_authority_consistency_failures( + household: pd.DataFrame, + *, + code_column: str = UK_LOCAL_AUTHORITY_CODE_COLUMN, + input_column: str = UK_LOCAL_AUTHORITY_INPUT_COLUMN, +) -> list[str]: + """Report rows whose engine input disagrees with their code (gate use). + + Returns failure messages and never raises, so the ladder gate can carry + the verdict through the local battery. + """ + + missing = [c for c in (code_column, input_column) if c not in household.columns] + if missing: + return [f"household table is missing local authority column(s): {missing}"] + total = len(household) + codes = ( + household[code_column].fillna("").astype(str).str.strip().to_numpy(dtype=object) + ) + keys = ( + household[input_column] + .fillna("") + .astype(str) + .str.strip() + .to_numpy(dtype=object) + ) + table = local_authority_engine_key_by_code() + failures: list[str] = [] + blank = np.array([key == "" for key in keys], dtype=bool) + if blank.any(): + failures.append(f"{input_column}: {int(blank.sum())}/{total} row(s) are blank") + expected = [table.get(code) for code in codes] + off_roster = np.array([value is None for value in expected], dtype=bool) + if off_roster.any(): + examples = sorted(set(codes[off_roster].tolist()))[:5] + failures.append( + f"{code_column}: {int(off_roster.sum())}/{total} row(s) are not on " + f"the April 2023 local authority roster; examples {examples}" + ) + mismatched = np.array( + [ + not off and not is_blank and want != key + for off, is_blank, want, key in zip( + off_roster, blank, expected, keys, strict=True + ) + ], + dtype=bool, + ) + if mismatched.any(): + examples = sorted( + { + f"{code}->{key}" + for code, key in zip(codes[mismatched], keys[mismatched], strict=True) + } + )[:5] + failures.append( + f"{input_column}: {int(mismatched.sum())}/{total} row(s) disagree with " + f"{code_column}; examples {examples}" + ) + return failures + + +def _engine_local_authority_members() -> tuple[str, frozenset[str]]: + from importlib import metadata + + from policyengine_uk.variables.household.demographic.locations import ( + LocalAuthority, + ) + + try: + version = metadata.version("policyengine-uk") + except metadata.PackageNotFoundError: # pragma: no cover - source checkout + version = "unknown" + return version, frozenset(LocalAuthority.__members__) + + +def local_authority_keys_missing_from_engine( + keys: Iterable[object] | None = None, +) -> tuple[str, ...]: + """Member names (the whole roster's by default) absent from the installed engine.""" + + _version, members = _engine_local_authority_members() + wanted = ( + set(local_authority_engine_key_by_code().values()) + if keys is None + else {str(key) for key in keys} + ) + return tuple(sorted(wanted - members)) + + +def verify_local_authority_engine_domain(keys: Iterable[object]) -> None: + """Refuse any member name that the installed engine's enum does not carry.""" + + wanted = [str(key) for key in keys] + missing = local_authority_keys_missing_from_engine(wanted) + if missing: + version, _members = _engine_local_authority_members() + raise ValueError( + f"{len(missing)} local_authority member name(s) are not in the " + f"installed policyengine-uk {version} LocalAuthority enum: " + f"{list(missing[:10])}; the engine pin must carry every roster " + "authority before the input can ship (microcosm#953)." + ) + + +__all__ = [ + "LOCAL_AUTHORITY_ENGINE_KEY_ALIASES", + "UK_LOCAL_AUTHORITY_CODE_COLUMN", + "UK_LOCAL_AUTHORITY_INPUT_COLUMN", + "UK_LOCAL_AUTHORITY_NAMES_KIND", + "UK_LOCAL_AUTHORITY_NAMES_RESOURCE", + "UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION", + "UK_LOCAL_AUTHORITY_VINTAGE", + "load_uk_local_authority_names_resource", + "local_authority_consistency_failures", + "local_authority_engine_key", + "local_authority_engine_key_by_code", + "local_authority_keys_missing_from_engine", + "resolve_local_authority_engine_keys", + "verify_local_authority_engine_domain", +] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index c49a78a1e..dcf1c8530 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -7,6 +7,7 @@ from __future__ import annotations +import importlib.util from dataclasses import dataclass, replace from pathlib import Path from typing import Any @@ -21,6 +22,9 @@ assign_uk_geography_ladder, uk_geography_ladder_gate, ) +from microcosm.build.uk_runtime.local_authority_input import ( + verify_local_authority_engine_domain, +) from microcosm.build.uk_runtime.national_frame import ( _mass_log_from_stored, _read_weight_metadata, @@ -444,6 +448,13 @@ def clone_uk_dataset_tables_with_ladder_geography( "UK geography ladder gate failed on the cloned assignment: " + "; ".join(gate.failures) ) + # Fail closed against the installed engine: every assigned local_authority + # member name must exist in this pin's LocalAuthority enum. Only the + # assigned keys are checked here, so a toy ladder stays evaluable and an + # engine-free unit lane is not asked to import the engine; the whole + # roster is asserted by test_uk_local_authority_input.py (microcosm#953). + if importlib.util.find_spec("policyengine_uk") is not None: + verify_local_authority_engine_domain(assigned["local_authority"].unique()) validate_uk_ladder_rowwise_dataset_tables(cloned_person, cloned_benunit, assigned) # The frame construction re-runs linkage validation and binds the typed diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index ad80791e2..377af985d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -185,6 +185,12 @@ def __post_init__(self) -> None: "household.household_is_cgt_band_donor", "household.household_is_spi_synthetic", "household.la_code_oa", + # #953: the engine's household local_authority enum input, written by the + # rowwise geography ladder from local_authority_code. The incumbent never + # carried it, so the coverage manifest cannot list it; this allow-list + # binds at the national release-cut export gate, and the rowwise lane's + # own guard is the ladder gate's code/member consistency check. + "household.local_authority", "household.lsoa_code", "household.mortgage_debt", "household.msoa_code", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 5e336b231..6b0024b2a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "eb328a5cf6cc0bc567c209fce03294f4f3b0b5e59adf47dfa7ff4173545e03be" + "ee85373fdc0a53cd299bf2c3e93e9c5f09ef3bb67919db66865a5b24651321f4" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index ef6146120..bacd4e539 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -971,6 +971,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "uk_population_targets.json", "uk_firms_targets.json", "local_area_crosswalk.json", + "local_authority_names.json", "target_references.json", "target_reference_membership.json", "local_target_references.json", @@ -1078,6 +1079,7 @@ def test_uk_package_loads(self) -> None: "uk_population_targets.json", "uk_firms_targets.json", "local_area_crosswalk.json", + "local_authority_names.json", "target_references.json", "target_reference_membership.json", "local_target_references.json", diff --git a/packages/microcosm-build/tests/test_uk_geography_ladder.py b/packages/microcosm-build/tests/test_uk_geography_ladder.py index 7fdbd3e8f..cfd033146 100644 --- a/packages/microcosm-build/tests/test_uk_geography_ladder.py +++ b/packages/microcosm-build/tests/test_uk_geography_ladder.py @@ -244,6 +244,7 @@ def test_assignment_writes_the_full_ladder_from_the_oa_anchor(tmp_path) -> None: assert wales["itl3_code"] == "TLL11" assert wales["itl2_code"] == "TLL1" assert wales["itl1_code"] == "TLL" + assert wales["local_authority"] == "ISLE_OF_ANGLESEY" # London households draw a London OA only. assert set(assigned.loc[[100, 200, 300], "oa_code"]).issubset( {"E00000001", "E00000002", "E00000003"} @@ -472,9 +473,15 @@ def _gated_household() -> tuple[pd.DataFrame, np.ndarray]: "local_authority_code": [ "E09000001", "E07000010", - "E08000020", + "E08000003", "W06000001", ], + "local_authority": [ + "CITY_OF_LONDON", + "FENLAND", + "MANCHESTER", + "ISLE_OF_ANGLESEY", + ], "ward_code": ["E05000001", "E05000010", "E05000020", "W05000001"], "constituency_code": [ "E14000001", @@ -522,6 +529,71 @@ def test_gate_fails_on_itl_prefix_inconsistency() -> None: assert any("disagree with the ITL3 prefix" in f for f in result.failures) +def test_gate_fails_when_local_authority_disagrees_with_its_code() -> None: + household, weights = _gated_household() + household.loc[1, "local_authority"] = "MAIDSTONE" + + result = uk_geography_ladder_gate(household, weights) + + assert not result.passed + assert any( + "local_authority: 1/4 row(s) disagree with local_authority_code" in f + for f in result.failures + ) + + +def test_gate_fails_on_local_authority_code_outside_the_roster() -> None: + household, weights = _gated_household() + # Gateshead's pre-2013 code: a valid GSS pattern, absent from the April + # 2023 roster, so it must be reported rather than pass on shape alone. + household.loc[1, "local_authority_code"] = "E08000020" + + result = uk_geography_ladder_gate(household, weights) + + assert not result.passed + assert any( + "not on the April 2023 local authority roster" in f for f in result.failures + ) + + +def test_gate_fails_when_local_authority_is_blank() -> None: + household, weights = _gated_household() + household.loc[2, "local_authority"] = "" + + result = uk_geography_ladder_gate(household, weights) + + assert not result.passed + assert any("local_authority: 1/4 row(s) are blank" in f for f in result.failures) + + +def test_assignment_refuses_ladder_codes_outside_the_lad23_roster(tmp_path) -> None: + ladder = load_uk_oa_ladder( + _write_ladder( + tmp_path / "ladder.npz", + local_authority_code=["E09000001", "E09000001", "E09000002", "W09999999"], + ) + ) + + with pytest.raises( + ValueError, match="not on the April 2023 local authority roster" + ): + assign_uk_geography_ladder(_household(), ladder, seed=0) + + +def test_assignment_refuses_local_authority_layer_vintage_mismatch(tmp_path) -> None: + metadata = _ladder_metadata() + metadata["layers"]["local_authority"] = { + "vintage": "2025_april_lad", + "source": "a rebuilt ladder", + } + ladder = load_uk_oa_ladder( + _write_ladder(tmp_path / "ladder.npz", metadata_json=json.dumps(metadata)) + ) + + with pytest.raises(ValueError, match="local authority vintage '2025_april_lad'"): + assign_uk_geography_ladder(_household(), ladder, seed=0) + + def test_gate_fails_when_columns_are_missing() -> None: household, weights = _gated_household() household = household.drop(columns=["ward_code"]) diff --git a/packages/microcosm-build/tests/test_uk_geography_sources.py b/packages/microcosm-build/tests/test_uk_geography_sources.py index cf247fa2f..4198efafd 100644 --- a/packages/microcosm-build/tests/test_uk_geography_sources.py +++ b/packages/microcosm-build/tests/test_uk_geography_sources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import io import urllib.error import zipfile @@ -15,6 +16,7 @@ build_official_uk_geography_crosswalk, build_scotland_crosswalk, geography_coverage_summary, + load_lad23_names_lookup, load_ni_dz_parlcon24_lookup, load_scotland_oa_constituencies, load_scotland_oa_lau_lookup, @@ -265,6 +267,77 @@ def test_build_england_wales_crosswalk_rejects_source_mismatch() -> None: ) +def _lad23_names_csv(rows: str) -> bytes: + return ("\ufeffLAD23CD,LAD23NM,LAD23NMW,ObjectId\n" + rows).encode("utf-8") + + +def _serve_lad23_names(monkeypatch, payload: bytes) -> None: + """Serve ``payload`` as the lookup and pin its digest for the test.""" + + monkeypatch.setattr(geography_sources, "_read_url_bytes", lambda url: payload) + monkeypatch.setattr( + geography_sources, "LAD23_NAMES_SHA256", hashlib.sha256(payload).hexdigest() + ) + + +def test_load_lad23_names_lookup_refuses_a_digest_mismatch(monkeypatch) -> None: + payload = _lad23_names_csv("E06000001,Hartlepool,,1\n") + monkeypatch.setattr(geography_sources, "_read_url_bytes", lambda url: payload) + + with pytest.raises(ValueError, match="LAD23 names lookup digest mismatch"): + load_lad23_names_lookup("memory://lad23-names.csv") + + +def test_load_lad23_names_lookup_normalises_columns(monkeypatch) -> None: + monkeypatch.setattr(geography_sources, "LAD23_COUNT", 3) + _serve_lad23_names( + monkeypatch, + _lad23_names_csv( + "E06000001,Hartlepool,,1\n" + "W06000001,Isle of Anglesey,Ynys Môn,288\n" + "S12000013,Na h-Eileanan Siar,,261\n" + "K04000001,England and Wales,,999\n" + ), + ) + + lookup = load_lad23_names_lookup("memory://lad23-names.csv") + + # The BOM-prefixed header resolves, the Welsh name column is dropped, and + # the K04 aggregate row falls outside the four nation prefixes. + assert lookup.to_dict("records") == [ + {"local_authority_code": "E06000001", "local_authority_name": "Hartlepool"}, + { + "local_authority_code": "W06000001", + "local_authority_name": "Isle of Anglesey", + }, + { + "local_authority_code": "S12000013", + "local_authority_name": "Na h-Eileanan Siar", + }, + ] + + +def test_load_lad23_names_lookup_refuses_blank_names_and_wrong_counts( + monkeypatch, +) -> None: + monkeypatch.setattr(geography_sources, "LAD23_COUNT", 2) + _serve_lad23_names( + monkeypatch, _lad23_names_csv("E06000001,Hartlepool,,1\nE06000002,,,2\n") + ) + with pytest.raises(ValueError, match="blank local_authority_name"): + load_lad23_names_lookup("memory://lad23-names.csv") + + _serve_lad23_names(monkeypatch, _lad23_names_csv("E06000001,Hartlepool,,1\n")) + with pytest.raises(ValueError, match="LAD23"): + load_lad23_names_lookup("memory://lad23-names.csv") + + _serve_lad23_names( + monkeypatch, "\ufeffLAD23CD,ObjectId\nE06000001,1\n".encode("utf-8") + ) + with pytest.raises(ValueError, match="missing LAD23CD or LAD23NM"): + load_lad23_names_lookup("memory://lad23-names.csv") + + def test_load_scotland_oa_lau_lookup_maps_lau_to_council_area(monkeypatch) -> None: monkeypatch.setattr(geography_sources, "SCOTLAND_OA2022_COUNT", 2) monkeypatch.setattr( diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 6f7934c74..e3cf6f065 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -42,7 +42,9 @@ def layer(vintage: str) -> dict[str, object]: "constituency": layer("2024_pcon"), "lsoa": layer("synthetic"), "msoa": layer("synthetic"), - "local_authority": layer("synthetic"), + # Real April 2023 codes, so the engine input resolves through the + # names resource (the ladder refuses any other vintage here). + "local_authority": layer("2023_april_lad"), "ward": layer("synthetic"), "itl": layer("2021_itl"), "region": layer("synthetic"), @@ -185,6 +187,7 @@ def test_ladder_clone_assigns_gates_and_conserves(toy_ladder, tmp_path) -> None: "lsoa_code", "msoa_code", "local_authority_code", + "local_authority", "ward_code", "constituency_code", "region_code", @@ -274,6 +277,95 @@ def test_ladder_clone_refuses_preassigned_geography(toy_ladder) -> None: clone_uk_dataset_with_ladder_geography(preassigned, ladder, n_clones=1) +def test_ladder_clone_refuses_preassigned_local_authority(toy_ladder) -> None: + ladder, _ = toy_ladder + preassigned = _seam_frame( + household=_household_frame().assign(local_authority="MAIDSTONE"), + ) + + with pytest.raises( + ValueError, + match="pre-assigned geography cannot be silently overwritten", + ): + clone_uk_dataset_with_ladder_geography(preassigned, ladder, n_clones=1) + + +def test_ladder_clone_writes_the_engine_local_authority_from_its_code( + toy_ladder, +) -> None: + from microcosm.build.uk_runtime import local_authority_engine_key_by_code + + ladder, _ = toy_ladder + result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=2) + + household = result.frame.table("household") + table = local_authority_engine_key_by_code() + assert household["local_authority"].tolist() == [ + table[code] for code in household["local_authority_code"] + ] + assert set(household["local_authority"]) <= { + "CITY_OF_LONDON", + "BARKING_AND_DAGENHAM", + "ISLE_OF_ANGLESEY", + "ABERDEEN_CITY", + "ANTRIM_AND_NEWTOWNABBEY", + } + + +@pytest.mark.requires_uk +def test_cloned_local_authority_survives_the_engine_loader( + toy_ladder, tmp_path +) -> None: + """The engine decodes the written member names, never its MAIDSTONE default. + + Through the written H5, not only the in-memory tables: the artifact is + read back with pandas and loaded through the multi-year dataset path, + which is the loader's ``set_input`` surface without the single-year + economic-assumption uprating (that path reads council tax and rent + columns a toy frame does not carry). + """ + + pytest.importorskip("tables") + pytest.importorskip("h5py") + from policyengine_uk import Microsimulation + from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset + + ladder, _ = toy_ladder + result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=1) + written = result.frame.table("household")["local_authority"].tolist() + # Household order is the seam order (London, Wales, Scotland, Northern + # Ireland); the London row draws one of the two London OAs. + assert written[0] in {"CITY_OF_LONDON", "BARKING_AND_DAGENHAM"} + assert written[1:] == [ + "ISLE_OF_ANGLESEY", + "ABERDEEN_CITY", + "ANTRIM_AND_NEWTOWNABBEY", + ] + + path = write_uk_rowwise_dataset(result, tmp_path / "rowwise.h5") + with pd.HDFStore(path, mode="r") as store: + tables = { + entity: store[entity] for entity in ("person", "benunit", "household") + } + assert tables["household"]["local_authority"].tolist() == written + + single_year = UKSingleYearDataset( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + fiscal_year=2023, + ) + simulation = Microsimulation(dataset=UKMultiYearDataset(datasets=[single_year])) + + decoded = [ + str(value) + for value in simulation.calculate("local_authority", 2023, decode_enums=True) + ] + + assert decoded == written + assert "MAIDSTONE" not in decoded + + def test_ladder_clone_refuses_uncovered_region(tmp_path) -> None: frame = _ladder_frame() england_only = frame[frame["region_code"].str.startswith("E")] @@ -357,6 +449,7 @@ def test_write_round_trip_preserves_ladder_columns(toy_ladder, tmp_path) -> None household = store["household"] assert "ward_code" in household.columns assert "itl1_code" in household.columns + assert "local_authority" in household.columns def _load_builder_module(): diff --git a/packages/microcosm-build/tests/test_uk_ladder_sni_sources.py b/packages/microcosm-build/tests/test_uk_ladder_sni_sources.py index ed6d56dce..b7619bc56 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_sni_sources.py +++ b/packages/microcosm-build/tests/test_uk_ladder_sni_sources.py @@ -28,6 +28,7 @@ load_ni_dz_ward_lookup, load_scotland_oa_households, load_scotland_oa_ward_lookup, + local_authority_engine_key_by_code, ) @@ -290,6 +291,16 @@ def test_ward_pattern_accepts_split_ward_part_codes() -> None: _gss_code_array(np.array(["E05R14284"]), label="constituency_code") +#: One April 2023 roster authority per nation for the synthetic gate table, so +#: the ladder gate's local_authority consistency check resolves every row. +_ROSTER_CODE_BY_NATION = { + "E": "E06000001", # Hartlepool + "W": "W06000001", # Isle of Anglesey + "S": "S12000033", # Aberdeen City + "N": "N09000001", # Antrim and Newtownabbey +} + + def _uk_gate_household() -> pd.DataFrame: rows = [ ("E00000001", "LONDON", "E12000007", "E05014284", "E14000001", "TLI31"), @@ -311,9 +322,10 @@ def _uk_gate_household() -> pd.DataFrame: "oa_code": oa, "lsoa_code": oa, "msoa_code": oa, - "local_authority_code": "E06000001" - if oa.startswith("E") - else oa[:1] + "09000001", + "local_authority_code": _ROSTER_CODE_BY_NATION[oa[:1]], + "local_authority": local_authority_engine_key_by_code()[ + _ROSTER_CODE_BY_NATION[oa[:1]] + ], "ward_code": ward, "constituency_code": constituency, "region_code": region_code, diff --git a/packages/microcosm-build/tests/test_uk_local_authority_input.py b/packages/microcosm-build/tests/test_uk_local_authority_input.py new file mode 100644 index 000000000..4d0a94e57 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_local_authority_input.py @@ -0,0 +1,337 @@ +"""Guarantees for the engine's ``local_authority`` input (microcosm#953). + +The rowwise ladder resolves every household's April 2023 ONS local authority +code to a policyengine-uk ``LocalAuthority`` member name through the committed +ONS names resource and the mechanical key rule. These tests pin the resource, +its coherence with the local-area crosswalk roster, the rule, the fail-closed +resolver, and membership in the installed engine's enum. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from importlib import resources as importlib_resources + +import numpy as np +import pandas as pd +import pytest + +import microcosm.build.uk_runtime.geography_sources as geography_sources +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import ( + LAD23_NAMES_ITEM_ID, + LAD23_NAMES_SHA256, + LAD23_NAMES_URL, + LOCAL_AUTHORITY_ENGINE_KEY_ALIASES, + UK_GEOGRAPHY_LADDER_COLUMNS, + UK_LOCAL_AUTHORITY_NAMES_KIND, + UK_LOCAL_AUTHORITY_NAMES_RESOURCE, + UK_LOCAL_AUTHORITY_VINTAGE, + load_uk_local_authority_names_resource, + local_authority_consistency_failures, + local_authority_engine_key, + local_authority_engine_key_by_code, + local_authority_keys_missing_from_engine, + resolve_local_authority_engine_keys, + verify_local_authority_engine_domain, +) +from tools.generate_uk_local_authority_names import ( + DEFAULT_SOURCE_CSV, + build_local_authority_names, +) + +#: The reviewed digest of the ONS lookup, held here as a literal so a re-pin +#: of ``LAD23_NAMES_SHA256`` is visible in this test's diff too. +SOURCE_SHA256 = "6c2d811f50756c459c6a1c7c692ae9262fb4d70df3f535958ef7cdbbbd76cc44" + +#: The April 2023 unitaries the engine enum did not carry at 2.98.0; the +#: roster-wide membership test below names them until the pin moves past the +#: policyengine-uk release that adds them. +APRIL_2023_UNITARIES = ( + "CUMBERLAND", + "NORTH_NORTHAMPTONSHIRE", + "NORTH_YORKSHIRE", + "SOMERSET", + "WEST_NORTHAMPTONSHIRE", + "WESTMORLAND_AND_FURNESS", +) + + +def _resource() -> dict: + return json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath(UK_LOCAL_AUTHORITY_NAMES_RESOURCE) + .read_text(encoding="utf-8") + ) + + +def _crosswalk_area_ids() -> list[str]: + crosswalk = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath("local_area_crosswalk.json") + .read_text(encoding="utf-8") + ) + return crosswalk["levels"]["local_authority"]["area_ids"] + + +def test_uk_local_authority_names_is_registered_in_the_country_package() -> None: + spec = load_country_spec("uk") + + assert UK_LOCAL_AUTHORITY_NAMES_RESOURCE in spec.resources + assert UK_LOCAL_AUTHORITY_NAMES_RESOURCE in spec.resource_hashes + + +def test_uk_local_authority_names_pins_source_and_roster() -> None: + resource = _resource() + + assert resource["schema_version"] == 1 + assert resource["country"] == "uk" + assert resource["kind"] == UK_LOCAL_AUTHORITY_NAMES_KIND + assert resource["source"] == { + "name": ( + "Local Authority Districts (April 2023) Names and Codes in the " + "United Kingdom" + ), + "publisher": "ONS Open Geography Portal", + "item_id": LAD23_NAMES_ITEM_ID, + "url": LAD23_NAMES_URL, + "sha256": SOURCE_SHA256, + "columns": ["LAD23CD", "LAD23NM"], + "vintage": UK_LOCAL_AUTHORITY_VINTAGE, + } + assert LAD23_NAMES_ITEM_ID in LAD23_NAMES_URL + assert LAD23_NAMES_SHA256 == SOURCE_SHA256 + assert resource["area_count"] == 361 == len(resource["areas"]) + by_nation: dict[str, int] = {} + for code in resource["areas"]: + by_nation[code[0]] = by_nation.get(code[0], 0) + 1 + assert by_nation == {"E": 296, "W": 22, "S": 32, "N": 11} + assert list(resource["areas"]) == sorted(resource["areas"]) + # The validating loader accepts the committed resource as-is. + assert load_uk_local_authority_names_resource()["area_count"] == 361 + + +def test_uk_local_authority_names_covers_the_crosswalk_roster_exactly() -> None: + area_ids = _crosswalk_area_ids() + + assert len(area_ids) == 361 + assert set(_resource()["areas"]) == set(area_ids) + assert set(local_authority_engine_key_by_code()) == set(area_ids) + + +def test_engine_keys_follow_the_mechanical_rule_except_the_declared_alias() -> None: + areas = _resource()["areas"] + + for code, entry in areas.items(): + assert entry["engine_key"] == local_authority_engine_key(entry["name"]), code + + def bare_rule(name: str) -> str: + return re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_") + + bare_rule_differs = { + code + for code, entry in areas.items() + if entry["engine_key"] != bare_rule(entry["name"]) + } + assert bare_rule_differs == {"E07000146"} + assert bare_rule("King's Lynn and West Norfolk") == "KING_S_LYNN_AND_WEST_NORFOLK" + assert areas["E07000146"] == { + "name": "King's Lynn and West Norfolk", + "engine_key": "KINGS_LYNN_AND_WEST_NORFOLK", + } + assert dict(LOCAL_AUTHORITY_ENGINE_KEY_ALIASES) == { + "King's Lynn and West Norfolk": "KINGS_LYNN_AND_WEST_NORFOLK" + } + keys = [entry["engine_key"] for entry in areas.values()] + assert len(set(keys)) == len(keys) == 361 + + +def test_engine_key_rule_examples() -> None: + assert local_authority_engine_key("Bristol, City of") == "BRISTOL_CITY_OF" + assert local_authority_engine_key("Na h-Eileanan Siar") == "NA_H_EILEANAN_SIAR" + assert local_authority_engine_key(" Rhondda Cynon Taf ") == "RHONDDA_CYNON_TAF" + assert local_authority_engine_key("Herefordshire, County of") == ( + "HEREFORDSHIRE_COUNTY_OF" + ) + assert local_authority_engine_key("King's Lynn and West Norfolk") == ( + "KINGS_LYNN_AND_WEST_NORFOLK" + ) + with pytest.raises(ValueError, match="empty engine key"): + local_authority_engine_key(" , ") + + +def test_resolve_maps_known_codes_and_refuses_unknown() -> None: + resolved = resolve_local_authority_engine_keys( + pd.Series( + ["E09000001", "E07000146", "S12000013", "N09000011"], index=[7, 3, 9, 1] + ) + ) + + assert resolved.dtype == object + assert resolved.tolist() == [ + "CITY_OF_LONDON", + "KINGS_LYNN_AND_WEST_NORFOLK", + "NA_H_EILEANAN_SIAR", + "ARDS_AND_NORTH_DOWN", + ] + assert resolve_local_authority_engine_keys(np.asarray(["W06000001"])).tolist() == [ + "ISLE_OF_ANGLESEY" + ] + assert resolve_local_authority_engine_keys(["E06000063"]).tolist() == ["CUMBERLAND"] + + with pytest.raises(ValueError, match=r"not on the April 2023.*E09999999"): + resolve_local_authority_engine_keys(["E09000001", "E09999999", ""]) + with pytest.raises(ValueError, match="2 distinct"): + resolve_local_authority_engine_keys(["E09999999", None, "E09999999"]) + + +def test_consistency_failures_report_blank_unknown_and_mismatched_rows() -> None: + household = pd.DataFrame( + { + "local_authority_code": [ + "E09000001", + "E09000001", + "E08000020", + "W06000001", + "S12000033", + ], + "local_authority": [ + "CITY_OF_LONDON", + "MAIDSTONE", + "GATESHEAD", + "", + "ABERDEEN_CITY", + ], + } + ) + + failures = local_authority_consistency_failures(household) + + assert failures == [ + "local_authority: 1/5 row(s) are blank", + "local_authority_code: 1/5 row(s) are not on the April 2023 local " + "authority roster; examples ['E08000020']", + "local_authority: 1/5 row(s) disagree with local_authority_code; " + "examples ['E09000001->MAIDSTONE']", + ] + assert local_authority_consistency_failures(household.iloc[[0, 4]]) == [] + assert local_authority_consistency_failures( + household.drop(columns=["local_authority"]) + ) == ["household table is missing local authority column(s): ['local_authority']"] + + +def test_local_authority_is_a_ladder_column_after_its_code() -> None: + columns = list(UK_GEOGRAPHY_LADDER_COLUMNS) + + assert columns.index("local_authority") == columns.index("local_authority_code") + 1 + + +@pytest.mark.requires_uk +def test_every_roster_engine_key_is_a_local_authority_member_at_the_pinned_engine() -> ( + None +): + """Fail closed on the whole roster against the installed engine. + + Red while the engine pin predates the policyengine-uk release that adds + the six April 2023 unitaries (``APRIL_2023_UNITARIES``); the build refuses + those households' rows for the same reason, so a partial map never ships. + """ + + missing = local_authority_keys_missing_from_engine() + + assert missing == (), ( + f"{len(missing)} roster authorities are not LocalAuthority members at " + f"the pinned engine: {missing}" + ) + + +@pytest.mark.requires_uk +def test_verify_engine_domain_accepts_members_and_names_missing_keys() -> None: + verify_local_authority_engine_domain(["CITY_OF_LONDON", "MAIDSTONE"]) + assert local_authority_keys_missing_from_engine(["CITY_OF_LONDON"]) == () + + with pytest.raises(ValueError, match=r"not in the installed.*NOT_AN_AUTHORITY"): + verify_local_authority_engine_domain( + np.asarray(["CITY_OF_LONDON", "NOT_AN_AUTHORITY"], dtype=object) + ) + assert local_authority_keys_missing_from_engine(["NOT_AN_AUTHORITY"]) == ( + "NOT_AN_AUTHORITY", + ) + + +@pytest.mark.requires_uk +def test_engine_domain_gap_is_exactly_the_april_2023_unitaries_or_closed() -> None: + """Any other miss is a roster or rule defect, not the known enum lag.""" + + missing = local_authority_keys_missing_from_engine() + + assert set(missing) <= set(APRIL_2023_UNITARIES) + + +def test_resource_loader_refuses_a_drifted_source_digest(monkeypatch) -> None: + load_uk_local_authority_names_resource.cache_clear() + monkeypatch.setattr(geography_sources, "LAD23_NAMES_SHA256", "0" * 64) + try: + with pytest.raises(ValueError, match="not the pinned LAD23 names digest"): + load_uk_local_authority_names_resource() + finally: + load_uk_local_authority_names_resource.cache_clear() + monkeypatch.undo() + assert load_uk_local_authority_names_resource()["area_count"] == 361 + + +def test_uk_local_authority_names_matches_generator_output() -> None: + if not DEFAULT_SOURCE_CSV.exists(): + pytest.skip("ONS LAD23 names lookup is not mounted under build/uk") + + assert _resource() == build_local_authority_names(source_csv=DEFAULT_SOURCE_CSV) + + +def test_generator_refuses_wrong_row_count_and_duplicate_keys( + tmp_path, monkeypatch +) -> None: + source = tmp_path / "lad23.csv" + source.write_text( + "LAD23CD,LAD23NM,LAD23NMW,ObjectId\n" + "E06000001,Hartlepool,,1\n" + "E06000002,Middlesbrough,,2\n", + encoding="utf-8", + ) + # A lookup whose bytes differ from the pinned digest is refused before + # any row is parsed, naming both digests. + with pytest.raises(ValueError, match=f"expected {SOURCE_SHA256}, got "): + build_local_authority_names(source_csv=source) + + monkeypatch.setattr( + geography_sources, + "LAD23_NAMES_SHA256", + hashlib.sha256(source.read_bytes()).hexdigest(), + ) + monkeypatch.setattr(geography_sources, "LAD23_COUNT", 3) + with pytest.raises(ValueError, match="LAD23"): + build_local_authority_names(source_csv=source) + + monkeypatch.setattr(geography_sources, "LAD23_COUNT", 2) + built = build_local_authority_names(source_csv=source) + assert built["area_count"] == 2 + assert built["areas"]["E06000002"] == { + "name": "Middlesbrough", + "engine_key": "MIDDLESBROUGH", + } + assert built["source"]["url"] == LAD23_NAMES_URL + + source.write_text( + "LAD23CD,LAD23NM,LAD23NMW,ObjectId\n" + "E06000001,Somerset,,1\n" + "E06000066,Somerset!,,2\n", + encoding="utf-8", + ) + monkeypatch.setattr( + geography_sources, + "LAD23_NAMES_SHA256", + hashlib.sha256(source.read_bytes()).hexdigest(), + ) + with pytest.raises(ValueError, match="one engine key"): + build_local_authority_names(source_csv=source) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index d65a6d487..aad7b7d25 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -406,6 +406,7 @@ def fake_clone(*_args, output_path: Path, **_kwargs): "lsoa_code": ["E0101", "W0101"], "msoa_code": ["E0201", "W0201"], "local_authority_code": ["E06000063", "W06000001"], + "local_authority": ["CUMBERLAND", "ISLE_OF_ANGLESEY"], "ward_code": ["E05000001", "W05000001"], "constituency_code": ["E14000001", "W07000041"], "region_code": ["E12000007", "W99999999"], diff --git a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py index 1bc74fffd..4d2790925 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py @@ -142,7 +142,9 @@ def layer(vintage: str) -> dict[str, object]: "constituency": layer("2024_pcon"), "lsoa": layer("synthetic"), "msoa": layer("synthetic"), - "local_authority": layer("synthetic"), + # Real April 2023 London codes, so the engine input resolves through + # the names resource (the ladder refuses any other vintage). + "local_authority": layer("2023_april_lad"), "ward": layer("synthetic"), "itl": layer("2021_itl"), "region": layer("synthetic"), diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 6b60d493d..b0e2b78e3 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2160,6 +2160,7 @@ def test_stacked_operator_target_requires_preparation_before_activation_authorit ("error", None, "failed"), ], ) +@pytest.mark.requires_us def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_terminal_state( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -2315,6 +2316,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term } +@pytest.mark.requires_us def test_stacked_pool_fixed_h5_reaches_release_cd_vintage_preflight( pool_tool: ModuleType, release_tool: ModuleType, @@ -2650,6 +2652,7 @@ def test_constants_adapter_failure_receipt_preserves_requested_resolution_state( assert "spec_sha256" not in json.dumps(error_receipt["run_config"]) +@pytest.mark.requires_us def test_constants_adapter_post_resolution_failure_receipt_retains_binding( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -2695,6 +2698,7 @@ def test_constants_adapter_post_resolution_failure_receipt_retains_binding( assert error_receipt["run_config"] == resolved_config +@pytest.mark.requires_us def test_constants_adapter_fixture_checkpoints_are_byte_identical_and_only_receipt_changes( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -2791,6 +2795,7 @@ def run_fixture(root: Path, *, config_authority: str) -> dict[str, object]: assert constants_receipt == adapter_receipt +@pytest.mark.requires_us def test_stacked_entrypoint_rejects_noncanonical_post_puf_transfer_receipt( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -3260,6 +3265,7 @@ def fail_publication(*_args, **_kwargs) -> None: assert stale_h5.read_bytes() == b"prior-build-artifact" +@pytest.mark.requires_us def test_logbook_gate_receipts_are_immutable_across_later_attempts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -4164,6 +4170,7 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( ) +@pytest.mark.requires_us def test_stacked_entrypoint_resumes_each_checkpoint_boundary( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -4274,6 +4281,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( assert any(row.prev_row_digest == third_row.row_digest for row in final_rows) +@pytest.mark.requires_us def test_stacked_resume_error_uses_realized_stack_identity( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 0ba6e98d4..a579dca15 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -416,13 +416,13 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "211abff22b4eedf9cf69f4b43a6f77ca8966d61a386c1804c3fdb093b0e27aa0" + "62bc37def863b68d5595dcd5ec8b6483d1fad5f0874b80073f5752e0a1a5ebff" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "ee6b1eb9451866654bfeb0d2090a8412e08b8c2caa8bfb16fb9aa3c5b5002d84" + "4f2be1f12ebdbebd57b00dee355e27de0cac280be2b09e177cc82cc6afaf552e" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "55cbe8b9f9dab5817f02c2f690965dfae6406c16facecd9f7ab3b08ba06c272e" + "775ba91346d29ce958cd82c0e459140f77d0ba00f695254f93ca02d8d7e52981" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -779,10 +779,10 @@ def compatibility_claim_declarer_error(declared_by: object) -> str | None: }, "release_cut": { "gates_manifest_sha256": ( - "c65d20de918abd8dd2c31c89a49ed51f0ca75a7c5f31ef00e61089406c4db401" + "9b9623caf02ee37d48cef4ff60171877c94962b0526ee516c89fab253790e8f2" ), "policy_sha256": ( - "a7250c519e79e22d366316cd4943f5f4bd2cc86a0e76b919ee2ff2eacb2335f3" + "77eace424e352172496f8fd99dbf68f858514198f87f6c6a6b33353d7eff55ad" ), }, } diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index e61245fd3..0f25398c2 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -116,13 +116,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "211abff22b4eedf9cf69f4b43a6f77ca8966d61a386c1804c3fdb093b0e27aa0" + "62bc37def863b68d5595dcd5ec8b6483d1fad5f0874b80073f5752e0a1a5ebff" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "ee6b1eb9451866654bfeb0d2090a8412e08b8c2caa8bfb16fb9aa3c5b5002d84" + "4f2be1f12ebdbebd57b00dee355e27de0cac280be2b09e177cc82cc6afaf552e" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "55cbe8b9f9dab5817f02c2f690965dfae6406c16facecd9f7ab3b08ba06c272e" + "775ba91346d29ce958cd82c0e459140f77d0ba00f695254f93ca02d8d7e52981" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "6f0243bcda09dad26945376230c44ec3cf55d4e417c3a25e29bae8c59bc1a69d" diff --git a/tools/generate_uk_local_authority_names.py b/tools/generate_uk_local_authority_names.py new file mode 100644 index 000000000..92c78543b --- /dev/null +++ b/tools/generate_uk_local_authority_names.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +"""Generate the UK local authority names resource from the ONS LAD23 lookup. + +Writes ``local_authority_names.json``: every April 2023 local authority +district code with its ONS display name and the policyengine-uk +``LocalAuthority`` member name the rowwise build writes as the household +``local_authority`` input (microcosm#953). The source bytes must match +``geography_sources.LAD23_NAMES_SHA256``; the resource records that digest and +the loader asserts it, so a re-published lookup is a reviewed re-pin, never a +silent regeneration. + +Run from the repository root:: + + uv run python tools/generate_uk_local_authority_names.py \\ + --source-csv build/uk/lad23_names_and_codes.csv + +``--check`` compares the generated payload with the committed resource and +exits non-zero on drift; without ``--source-csv`` the lookup is downloaded. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import sys +import urllib.request +from pathlib import Path +from typing import Any + +import pandas as pd + +import microcosm.build.uk_runtime.geography_sources as geography_sources +from microcosm.build.uk_runtime.geography_sources import ( + LAD23_NAMES_ITEM_ID, + LAD23_NAMES_URL, + normalise_lad23_names, +) +from microcosm.build.uk_runtime.local_authority_input import ( + UK_LOCAL_AUTHORITY_NAMES_KIND, + UK_LOCAL_AUTHORITY_NAMES_RESOURCE, + UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION, + UK_LOCAL_AUTHORITY_VINTAGE, + local_authority_engine_key, +) + +DEFAULT_SOURCE_CSV = Path("build/uk/lad23_names_and_codes.csv") +DEFAULT_OUTPUT = ( + Path("packages/microcosm-build/src/microcosm/build/uk") + / UK_LOCAL_AUTHORITY_NAMES_RESOURCE +) +SOURCE_NAME = ( + "Local Authority Districts (April 2023) Names and Codes in the United Kingdom" +) +SOURCE_PUBLISHER = "ONS Open Geography Portal" + + +def _fetch_bytes(url: str) -> bytes: + request = urllib.request.Request( + url, headers={"User-Agent": "microcosm-build (UK local authority names)"} + ) + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 + payload = response.read() + if not payload: + raise RuntimeError(f"Empty download from {url}") + return payload + + +def build_local_authority_names( + *, + source_csv: Path | None = None, + url: str = LAD23_NAMES_URL, +) -> dict[str, Any]: + """Build the resource payload from a local CSV or the published lookup.""" + + payload = ( + Path(source_csv).read_bytes() if source_csv is not None else _fetch_bytes(url) + ) + if not payload: + raise ValueError("LAD23 names lookup is empty.") + # The pinned digest, not whatever was downloaded: a re-published lookup + # is re-pinned in geography_sources first, then regenerated here. + geography_sources.verify_lad23_names_bytes(payload) + lookup = normalise_lad23_names(pd.read_csv(io.BytesIO(payload), dtype=str)) + areas: dict[str, dict[str, str]] = {} + for row in lookup.itertuples(index=False): + areas[str(row.local_authority_code)] = { + "name": str(row.local_authority_name), + "engine_key": local_authority_engine_key(row.local_authority_name), + } + keys = [entry["engine_key"] for entry in areas.values()] + duplicates = sorted({key for key in keys if keys.count(key) > 1}) + if duplicates: + raise ValueError( + f"LAD23 names map two authorities to one engine key: {duplicates}." + ) + return { + "schema_version": UK_LOCAL_AUTHORITY_NAMES_SCHEMA_VERSION, + "country": "uk", + "kind": UK_LOCAL_AUTHORITY_NAMES_KIND, + "description": ( + "Every UK local authority district on the April 2023 ONS frame, " + "with its ONS display name and the policyengine-uk LocalAuthority " + "member name the rowwise build writes as the household " + "local_authority input. engine_key is the mechanical rule in " + "microcosm.build.uk_runtime.local_authority_input (upper-case, " + "non-alphanumeric runs to one underscore) with the module's " + "declared aliases; regenerate with " + "tools/generate_uk_local_authority_names.py (microcosm#953)." + ), + "source": { + "name": SOURCE_NAME, + "publisher": SOURCE_PUBLISHER, + "item_id": LAD23_NAMES_ITEM_ID, + "url": url, + "sha256": hashlib.sha256(payload).hexdigest(), + "columns": ["LAD23CD", "LAD23NM"], + "vintage": UK_LOCAL_AUTHORITY_VINTAGE, + }, + "area_count": len(areas), + "areas": dict(sorted(areas.items())), + } + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + source_csv = args.source_csv + if source_csv is None and DEFAULT_SOURCE_CSV.exists(): + source_csv = DEFAULT_SOURCE_CSV + resource = build_local_authority_names(source_csv=source_csv, url=args.url) + rendered = json.dumps(resource, indent=2, ensure_ascii=False) + "\n" + if args.check: + committed = args.output.read_text(encoding="utf-8") + if committed != rendered: + print(f"{args.output} is stale; regenerate it.", file=sys.stderr) + return 1 + print(f"{args.output} is current ({resource['area_count']} authorities).") + return 0 + args.output.write_text(rendered, encoding="utf-8") + print( + f"wrote {args.output} ({resource['area_count']} authorities, sha256 {resource['source']['sha256'][:12]})" + ) + return 0 + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--source-csv", + type=Path, + default=None, + help=f"local copy of the ONS lookup (default: {DEFAULT_SOURCE_CSV} when present, else download)", + ) + parser.add_argument("--url", default=LAD23_NAMES_URL) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--check", + action="store_true", + help="verify the committed resource instead of writing", + ) + return parser + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index 086a7f5e4..9949aefec 100644 --- a/uv.lock +++ b/uv.lock @@ -782,7 +782,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.26" }, { name = "openpyxl", marker = "extra == 'uk'", specifier = ">=3.1" }, { name = "pandas", specifier = ">=2" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.98.0" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = ">=2.100.0" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = ">=2.2.1,<3" }, { name = "pyarrow", specifier = ">=15" }, { name = "pyyaml", specifier = ">=6" }, @@ -1496,7 +1496,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.98.0" +version = "2.100.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1504,9 +1504,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/9c/bbd8c875fae959a4bab3061b1d72ca84d9b94cc9515bed3f631345b6b20e/policyengine_uk-2.98.0.tar.gz", hash = "sha256:fdbe9c197b2186e94a7122748ec1b34ce84451b46b4bd1558020e69c6d700005", size = 1270118, upload-time = "2026-09-16T13:42:03.677Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/6b/a84c78706f2f0c4be25a65034e624a1b1dc4e6e99e21291078b039bba30e/policyengine_uk-2.100.0.tar.gz", hash = "sha256:a7666023cc406eca0d2399dbe159d7712e61ff4bccedf841143ef3e85643e644", size = 1283060, upload-time = "2026-09-22T16:03:21.998Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/27/6a6a88c82537a0227101f11536fff362c2373091415d46bbe8f0f411bde7/policyengine_uk-2.98.0-py3-none-any.whl", hash = "sha256:ceeff3ff907992108ccd037f61e8d08306074331fee71c507ce653d1645012b4", size = 2084619, upload-time = "2026-09-16T13:42:01.587Z" }, + { url = "https://files.pythonhosted.org/packages/7c/54/f24bcee4879cda9920e1a837eca087502b73fa43e531940f86295baa1d67/policyengine_uk-2.100.0-py3-none-any.whl", hash = "sha256:ec1be1e181d26a57446292aa188b3cc1ad96b3951bf4ff0436580bfc9cd40b1a", size = 2106162, upload-time = "2026-09-22T16:03:20.37Z" }, ] [[package]]