diff --git a/changelog.d/760-uk-local-binding-enforcement.added.md b/changelog.d/760-uk-local-binding-enforcement.added.md new file mode 100644 index 0000000000..8784008a75 --- /dev/null +++ b/changelog.d/760-uk-local-binding-enforcement.added.md @@ -0,0 +1 @@ +UK rowwise local binding now enforces the 7 reviewed census binding fences at execution time: the doctrine solve derives bound families structurally from the target surface, refuses declared-vs-derived mismatches, checks those families against the committed schema-2 `local_binding_adjudications.json` register with correct-or-renew approval windows, and echoes the adjudications in both the dry-run plan and candidate manifest; the first shipped adjudication accepts census disclosure-control noise for household counts and names constituency as the winning grain for the rowwise arm. Both expiring adjudication registers — the local binding register and the national calibration measure exclusions — now warn when an in-force entry is within one week of its expiry date, so renewal is signalled before the correct-or-renew failure lands. diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index d09b020b64..16a694fd6a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -227,6 +227,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "local_binding_adjudications.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "uk_local_target_census.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/local_binding_adjudications.json b/packages/microcosm-build/src/microcosm/build/uk/local_binding_adjudications.json new file mode 100644 index 0000000000..1478e6c0ce --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/local_binding_adjudications.json @@ -0,0 +1,13 @@ +{ + "schema_version": 2, + "description": "Reviewed adjudications of the UK local binding fences (uk_local_target_census.json binding_fences). Binding a target family whose fence has no in-force entry here fails in the rowwise doctrine solve before any solve runs. Keys are fence ids; the reason states the ruling (accept the stated basis, or replace it with the named alternative) and cites the evidence.", + "exclusions": { + "census_disclosure_control_noise": { + "reason": "Accept the stated basis: constituency household counts bind to the published disclosure-controlled census values as-is — ONS/NRS cell-key perturbation and NISRA flexible-table-builder noise is documented, the counts are target values, never exact controls; the measured cross-grain deltas at review (constituency sums vs national totals: E&W +105, Scotland -554, NI +3) are accepted; constituency is the winning grain for the rowwise arm, whose targets are the sha-pinned OA ladder's occupied-household counts summed at constituency grain (ladder_targets.constituency_household_targets, provenance recorded per build) with no cross-grain reconciliation. Evidence: uk_local_target_census.json#/binding_fences/census_disclosure_control_noise.", + "approved_by": "juaristi22", + "adjudication": "microcosm#760", + "approved_on": "2026-08-27", + "expires_on": "2026-11-27" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_local_target_census.json b/packages/microcosm-build/src/microcosm/build/uk/uk_local_target_census.json index b1d2c26b6a..1fe4783157 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_local_target_census.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_local_target_census.json @@ -542,6 +542,6 @@ "bound_in_code": "The household-side metric is computed by local_targets.compute_household_metrics today.", "documented_unpinned": "The official product exists and was verified at the recorded URL on verified_on, but no exact table, vintage, or hash is pinned; binding work pins it per family.", "pinned_in_ladder": "The product is downloaded and sha-pinned per build by the UK OA ladder artifact tool, which records every source hash in the artifact metadata; target values derive from the artifact's own sums.", - "review_required_before_binding": "The fence is a reviewed adjudication requirement recorded for binding work; execution-time enforcement in the local solve configuration lands with the solve-doctrine increment of microcosm#495." + "review_required_before_binding": "The fence is a reviewed adjudication requirement enforced by the rowwise doctrine solve (local_rowwise.require_adjudicated_uk_local_binding): binding a family whose fences lack an in-force entry in local_binding_adjudications.json is refused before any solve runs." } } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 6a09ddf1d8..dfadc1dcc0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -264,10 +264,12 @@ align_area_targets, ) from microcosm.build.uk_runtime.local_rowwise import ( + UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, UKRowwiseDoctrineSolve, UKRowwiseLocalMatrix, build_uk_rowwise_local_matrix, past_cap_census, + require_adjudicated_uk_local_binding, rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, @@ -496,6 +498,7 @@ "UK_GATE_REGISTRY", "UK_FRAME_METADATA_KEY", "UK_LOCAL_MAX_WEIGHT_RATIO", + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE", "UK_LOCAL_SOLVE_DOCTRINE", "UK_LOCAL_TARGET_LOSS_CAP", "UK_CGT_GAINS_AMOUNT_COLUMN", @@ -506,6 +509,7 @@ "load_uk_stage_checkpoint", "load_uk_stage_predecessor", "materialize_uk_cgt_calibration_frame", + "require_adjudicated_uk_local_binding", "rowwise_calibration_mass_reason", "uk_cgt_annual_exempt_amount", "UK_CGT_REQUIRED_COLUMNS", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index 88dd55c76c..c25e285243 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -20,22 +20,31 @@ from __future__ import annotations +import warnings from collections.abc import Mapping, Sequence from dataclasses import dataclass +from datetime import date from typing import Any import numpy as np import pandas as pd from scipy import sparse as sp +from microcosm.build.uk_runtime import local_target_census from microcosm.build.uk_runtime.local_doctrine import ( UK_LOCAL_SOLVE_DOCTRINE, ) +from microcosm.build.uk_runtime.local_targets import AREA_TYPES from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.weighted_integrity import ( + coerce_reviewed_exclusions, + exclusion_evaluation_date, + load_uk_reviewed_exclusion_register, +) from microcosm.calibrate.solve import ( CONSERVE_MASS, FREE_MASS, @@ -47,15 +56,19 @@ from microcosm.frame import Frame, WeightKind __all__ = [ + "UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE", "UKRowwiseDoctrineSolve", "UKRowwiseLocalMatrix", "build_uk_rowwise_local_matrix", "past_cap_census", + "require_adjudicated_uk_local_binding", "rowwise_calibration_mass_reason", "rowwise_area_support_summary", "solve_uk_rowwise_weights_under_doctrine", ] +UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE = "local_binding_adjudications.json" + @dataclass(frozen=True) class UKRowwiseLocalMatrix: @@ -106,6 +119,7 @@ class UKRowwiseDoctrineSolve: final_loss: float n_nonzero: int past_cap_census: Mapping[str, Any] + binding_adjudications: Mapping[str, Any] def past_cap_census( @@ -403,6 +417,234 @@ def _require_uniform_target_surface(problem: UKRowwiseLocalMatrix) -> None: ) +def require_adjudicated_uk_local_binding( + bound_families: Sequence[str], + target_frame: pd.DataFrame, + *, + census: Mapping[str, Any] | None = None, + register: Mapping[str, Any] | None = None, + now: Any = None, +) -> dict[str, Any]: + """Require in-force review records before binding fenced UK local families.""" + + census_payload = ( + local_target_census.load_uk_local_target_census() + if census is None + else census + ) + family_rows = _uk_local_census_family_rows(census_payload) + declared, parsed = _normalise_uk_local_bound_families( + bound_families, + family_rows=family_rows, + ) + derived = _derive_uk_local_bound_families_from_target_frame( + target_frame, + family_rows=family_rows, + ) + if sorted(declared) != derived: + missing = sorted(set(derived) - set(declared)) + extra = sorted(set(declared) - set(derived)) + raise ValueError( + "UK local binding declarations: declared bound families " + f"{sorted(declared)} disagree with the families derived from " + f"the target surface {derived}; missing {missing}, extra " + f"{extra}; the declaration must name exactly what the matrix " + "binds." + ) + + records = ( + load_uk_reviewed_exclusion_register( + None, + resource=UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + ) + if register is None + else coerce_reviewed_exclusions( + register, + label=UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + ) + ) + evaluated_on = exclusion_evaluation_date(now) + stood_on: dict[str, dict[str, dict[str, str]]] = {} + expiring_soon: dict[str, str] = {} + for declared_family in declared: + census_family, _area_type = parsed[declared_family] + stood_on[declared_family] = {} + for fence_id in family_rows[census_family].get("adjudications", ()): + fence = str(fence_id) + record = records.get(fence) + if record is None: + raise ValueError( + f"{UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE}: " + f"bound family {declared_family!r} requires an in-force " + f"adjudication of fence {fence!r} and the committed " + "register records none. Record a reviewed adjudication " + "(reason stating the ruling and evidence, approver, " + "window) before binding — the fence is a reviewed " + "requirement, never a runtime flag." + ) + if record.expired(evaluated_on) or record.premature(evaluated_on): + raise ValueError( + f"{UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE}: " + f"bound family {declared_family!r} requires adjudication " + f"of fence {fence!r}, but the entry is outside its " + f"reviewed window ({record.approved_on}.." + f"{record.expires_on}, evaluated " + f"{evaluated_on.isoformat()}): correct the underlying " + "gap or renew the adjudication with a new approval and " + "expiry." + ) + if (date.fromisoformat(record.expires_on) - evaluated_on).days <= 7: + expiring_soon[fence] = record.expires_on + stood_on[declared_family][fence] = record.policy_payload() + + for fence, expires_on in sorted(expiring_soon.items()): + warnings.warn( + f"{UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE}: adjudication " + f"of fence {fence!r} expires {expires_on} — within one week of " + f"the evaluation date {evaluated_on.isoformat()}; renew the " + "adjudication or the fence closes the rowwise solve on expiry.", + UserWarning, + stacklevel=2, + ) + + fence_families: dict[str, set[str]] = {} + for family, row in family_rows.items(): + for fence_id in row.get("adjudications", ()): + fence_families.setdefault(str(fence_id), set()).add(family) + bound_census_families = {parsed[name][0] for name in declared} + dormant = sorted( + fence + for fence in records + if not (fence_families.get(fence, set()) & bound_census_families) + ) + return { + "register_resource": UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + "evaluated_on": evaluated_on.isoformat(), + "bound_families": list(declared), + "stood_on": stood_on, + "dormant": dormant, + } + + +def _uk_local_census_family_rows( + census: Mapping[str, Any], +) -> dict[str, Mapping[str, Any]]: + rows = census.get("families") + if not isinstance(rows, Sequence) or isinstance(rows, (str, bytes)): + raise ValueError("UK local target census must carry a families list.") + families: dict[str, Mapping[str, Any]] = {} + for row in rows: + if not isinstance(row, Mapping): + raise ValueError("UK local target census family rows must be objects.") + family = str(row.get("family", "")) + if not family: + raise ValueError("UK local target census family row has no family id.") + if family in families: + raise ValueError( + f"UK local target census declares duplicate family {family!r}." + ) + families[family] = row + return families + + +def _normalise_uk_local_bound_families( + bound_families: Sequence[str], + *, + family_rows: Mapping[str, Mapping[str, Any]], +) -> tuple[tuple[str, ...], dict[str, tuple[str, str]]]: + if isinstance(bound_families, str): + raise ValueError( + "UK local binding declarations: bound_families must be a " + "sequence of family/area_type strings, not one string." + ) + declared = tuple(str(name) for name in bound_families) + if not declared: + raise ValueError( + "UK local binding declarations: bound_families must name at " + "least one family/area_type pair." + ) + blanks = [name for name in declared if not name.strip()] + if blanks: + raise ValueError( + "UK local binding declarations: bound_families must not contain " + "blank family names." + ) + padded = [name for name in declared if name != name.strip()] + if padded: + raise ValueError( + "UK local binding declarations: bound_families must not carry " + f"surrounding whitespace: {padded}." + ) + duplicates = sorted({name for name in declared if declared.count(name) > 1}) + if duplicates: + raise ValueError( + "UK local binding declarations: duplicate bound family(ies) " + f"{duplicates}." + ) + + parsed: dict[str, tuple[str, str]] = {} + for name in declared: + if "/" not in name: + raise ValueError( + "UK local binding declarations: bound family " + f"{name!r} must have form '/'." + ) + family, area_type = name.rsplit("/", 1) + if not family or not area_type: + raise ValueError( + "UK local binding declarations: bound family " + f"{name!r} must have form '/'." + ) + if area_type not in AREA_TYPES: + raise ValueError( + "UK local binding declarations: bound family " + f"{name!r} names unsupported area_type {area_type!r}; " + f"expected one of {list(AREA_TYPES)}." + ) + if family not in family_rows: + raise ValueError( + "UK local binding declarations: bound family " + f"{name!r} names unknown census family {family!r}; " + f"expected one of {sorted(family_rows)}." + ) + parsed[name] = (family, area_type) + return declared, parsed + + +def _derive_uk_local_bound_families_from_target_frame( + target_frame: pd.DataFrame, + *, + family_rows: Mapping[str, Mapping[str, Any]], +) -> list[str]: + required = {"area_type", "metric"} + if not required <= set(target_frame.columns): + missing = sorted(required - set(target_frame.columns)) + raise ValueError( + "UK local binding declarations: target_frame column(s) " + f"{missing} are required to derive bound families." + ) + derived: set[str] = set() + cells = target_frame[["area_type", "metric"]].drop_duplicates() + for row in cells.itertuples(index=False): + area_type = str(row.area_type) + if area_type not in AREA_TYPES: + raise ValueError( + "UK local binding declarations: target surface names " + f"unsupported area_type {area_type!r}; expected one of " + f"{list(AREA_TYPES)}." + ) + metric = str(row.metric) + family = local_target_census.family_for_metric(metric) + if family not in family_rows: + raise ValueError( + "UK local binding declarations: target surface metric " + f"{metric!r} classified into unknown census family " + f"{family!r}." + ) + derived.add(f"{family}/{area_type}") + return sorted(derived) + + def rowwise_calibration_mass_reason(bound_families: Sequence[str]) -> str: """The mass-record reason a rowwise doctrine calibration declares. @@ -497,6 +739,10 @@ def solve_uk_rowwise_weights_under_doctrine( doctrine = UK_LOCAL_SOLVE_DOCTRINE _require_uniform_target_surface(problem) + binding_adjudications = require_adjudicated_uk_local_binding( + bound_families, + problem.target_frame, + ) mass_reason = rowwise_calibration_mass_reason(bound_families) household = frame.table("household") @@ -682,6 +928,7 @@ def solve_uk_rowwise_weights_under_doctrine( final_loss=float(result.final_loss), n_nonzero=int(result.n_nonzero), past_cap_census=census, + binding_adjudications=binding_adjudications, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_target_census.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_target_census.py index 257189e5bd..3916594800 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_target_census.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_target_census.py @@ -24,13 +24,14 @@ ``documented_unpinned`` and move to ``pinned_in_ladder`` when a build artifact sha-pins them per build, mirroring the HMRC/SPI source-contract discipline. -* **Fences by reference, enforcement declared.** The banded HMRC facts stay +* **Fences by reference, enforced at binding.** The banded HMRC facts stay fenced exactly as the national replay adjudicated them (``FULL_FRS_TI_BAND_FENCE_ID``, ``HMRC_SPI_TARGET_RECORD_COUNT`` are imported, never copied), and every census fence declares its enforcement - status: these are reviewed adjudication requirements recorded for binding - work — execution-time enforcement in the local solve lands with the solve - doctrine increment of microcosm#495. + status: the rowwise doctrine solve + (``local_rowwise.require_adjudicated_uk_local_binding``) refuses binding a + family whose fences lack an in-force entry in + ``local_binding_adjudications.json``. """ from __future__ import annotations @@ -49,6 +50,7 @@ "CENSUS_KIND", "CENSUS_RESOURCE", "CENSUS_SCHEMA_VERSION", + "family_for_metric", "METRIC_STATUS_BOUND_IN_CODE", "SOURCE_STATUS_DOCUMENTED_UNPINNED", "SOURCE_STATUS_PINNED_IN_LADDER", @@ -549,10 +551,11 @@ "sums." ), FENCE_ENFORCEMENT_REVIEW: ( - "The fence is a reviewed adjudication requirement recorded for " - "binding work; execution-time enforcement in the local solve " - "configuration lands with the solve-doctrine increment of " - "microcosm#495." + "The fence is a reviewed adjudication requirement enforced by the " + "rowwise doctrine solve " + "(local_rowwise.require_adjudicated_uk_local_binding): binding a " + "family whose fences lack an in-force entry in " + "local_binding_adjudications.json is refused before any solve runs." ), } @@ -708,6 +711,12 @@ def _family_for_metric(name: str) -> str: ) +def family_for_metric(name: str) -> str: + """Public fail-closed UK local metric-to-family classifier.""" + + return _family_for_metric(name) + + def _require_unique_ids( rows: list[dict[str, Any]], *, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index c2210865ee..feb72756ac 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import warnings from datetime import date from importlib import resources as importlib_resources from pathlib import Path @@ -269,6 +270,16 @@ def apply_uk_calibration_measure_exclusions( f"gap ({entry['tracking']}) or renew the adjudication with a new " "approval and expiry." ) + if (expires - evaluated_on).days <= 7: + warnings.warn( + f"UK calibration measure exclusion {entry['name']!r} expires " + f"{entry['expires_on']} — within one week of the evaluation " + f"date {evaluated_on.isoformat()}; correct the underlying gap " + f"({entry['tracking']}) or renew the adjudication before the " + "run fails on expiry.", + UserWarning, + stacklevel=2, + ) declared = {entry["name"]: entry for entry in exclusions} matched = {spec.name for spec in registry.specs if spec.name in declared} stale = sorted(set(declared) - matched) diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index e278f482f3..4c4d9ab1dd 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -320,6 +320,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "release_input_coverage_manifest.json", "registry_parity_fixture_2025.json", "was_wealth_support_bounds.json", + "local_binding_adjudications.json", "uk_local_target_census.json", "uk_national_targets.json", "target_references.json", @@ -397,6 +398,7 @@ def test_uk_package_loads(self) -> None: "release_input_coverage_manifest.json", "registry_parity_fixture_2025.json", "was_wealth_support_bounds.json", + "local_binding_adjudications.json", "uk_local_target_census.json", "uk_national_targets.json", "target_references.json", diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 0adfa3083c..1bd391ecce 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "276fddeb17857f64fa1bbaae6551e52580c4ef352f3aba4da0eabd14ef13a668", + "96efa147c0d80107af1e46ec691030aa001a82334d52459a4522a64853fa267e", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_uk_local_rowwise.py b/packages/microcosm-build/tests/test_uk_local_rowwise.py index 86790b6bd4..2b4252efa7 100644 --- a/packages/microcosm-build/tests/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/test_uk_local_rowwise.py @@ -12,20 +12,28 @@ from __future__ import annotations +from datetime import date + import numpy as np import pandas as pd import pytest from microcosm.build.uk_runtime import ( + UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, UK_LOCAL_MAX_WEIGHT_RATIO, UK_LOCAL_TARGET_LOSS_CAP, build_uk_rowwise_local_matrix, + load_uk_local_target_census, + require_adjudicated_uk_local_binding, rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, uk_household_weight_kind, uk_national_frame, ) +from microcosm.build.uk_runtime.weighted_integrity import ( + load_uk_reviewed_exclusion_register, +) from microcosm.frame import WeightKind @@ -53,8 +61,8 @@ def _clone_frame(weights=(1.0, 1.0, 1.0)): def _metrics() -> pd.DataFrame: return pd.DataFrame( { - "population": [2.0, 1.0, 3.0], - "uc_households": [1.0, 0.0, 1.0], + "households": [2.0, 1.0, 3.0], + "tenure/social_rent": [1.0, 0.0, 1.0], }, index=[101, 102, 103], ) @@ -68,30 +76,62 @@ def _targets() -> pd.DataFrame: return pd.DataFrame( { "code": ["E001", "S001"], - "population": [4.0, 2.0], + "households": [4.0, 2.0], + "tenure/social_rent": [1.0, 1.0], + } + ) + + +def _uc_problem(): + metrics = pd.DataFrame( + {"uc_households": [1.0, 0.0, 1.0]}, + index=[101, 102, 103], + ) + targets = pd.DataFrame( + { + "code": ["E001", "S001"], "uc_households": [1.0, 1.0], } ) + return build_uk_rowwise_local_matrix(metrics, _assigned(), targets) + + +def _reviewed_register_entry( + *, + approved_on: str = "2026-01-01", + expires_on: str = "2027-01-01", +) -> dict[str, str]: + return { + "reason": ( + "Accept the stated basis for test evidence. Evidence: " + "uk_local_target_census.json#/binding_fences/" + "census_disclosure_control_noise." + ), + "approved_by": "tester", + "adjudication": "microcosm#760-test", + "approved_on": approved_on, + "expires_on": expires_on, + } def test_matrix_builder_places_support_only_in_assigned_area() -> None: problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) assert problem.matrix.shape == (4, 3) assert problem.area_codes == ("E001", "S001") - assert problem.metric_names == ("population", "uc_households") + assert problem.metric_names == ("households", "tenure/social_rent") assert problem.household_ids == (101, 102, 103) dense = problem.matrix.toarray() frame = problem.target_frame # E001 rows carry only households 101/102; S001 rows only household 103. e_pop = int( - frame[(frame["area_code"] == "E001") & (frame["metric"] == "population")][ + frame[(frame["area_code"] == "E001") & (frame["metric"] == "households")][ "target_index" ].iloc[0] ) assert dense[e_pop].tolist() == [2.0, 1.0, 0.0] s_pop = int( - frame[(frame["area_code"] == "S001") & (frame["metric"] == "population")][ + frame[(frame["area_code"] == "S001") & (frame["metric"] == "households")][ "target_index" ].iloc[0] ) @@ -112,7 +152,7 @@ def test_matrix_builder_validates_alignment_and_finiteness() -> None: build_uk_rowwise_local_matrix(_metrics(), misaligned, _targets()) bad = _metrics() - bad.loc[101, "population"] = np.nan + bad.loc[101, "households"] = np.nan with pytest.raises(ValueError, match="finite"): build_uk_rowwise_local_matrix(bad, _assigned(), _targets()) @@ -124,7 +164,10 @@ def test_rowwise_doctrine_solve_uses_base_weights_directly() -> None: result = solve_uk_rowwise_weights_under_doctrine( frame, problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=60, learning_rate=0.2, seed=1, @@ -158,6 +201,144 @@ def test_rowwise_doctrine_solve_uses_base_weights_directly() -> None: result.frame.weights_for("household").values, result.weights, ) + receipt = result.binding_adjudications + assert ( + receipt["register_resource"] + == UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE + ) + assert receipt["dormant"] == [] + stood_on = receipt["stood_on"]["census_households/constituency"] + seed = stood_on["census_disclosure_control_noise"] + assert seed["approved_by"] == "juaristi22" + assert seed["approved_on"] == "2026-08-27" + assert seed["expires_on"] == "2026-11-27" + assert receipt["stood_on"]["tenure/constituency"] == {} + + +def test_rowwise_binding_refuses_unadjudicated_committed_fence() -> None: + problem = _uc_problem() + with pytest.raises(ValueError, match="uc_unit_vs_household_grain"): + require_adjudicated_uk_local_binding( + ["uc_households/constituency"], + problem.target_frame, + ) + + +def test_rowwise_binding_refuses_declared_derived_mismatch() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + with pytest.raises(ValueError, match="missing.*tenure/constituency"): + require_adjudicated_uk_local_binding( + ["census_households/constituency"], + problem.target_frame, + ) + with pytest.raises(ValueError, match="extra.*private_rent/constituency"): + require_adjudicated_uk_local_binding( + [ + "census_households/constituency", + "tenure/constituency", + "private_rent/constituency", + ], + problem.target_frame, + ) + + +def test_rowwise_binding_refuses_unknown_family_and_bad_area_type() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + with pytest.raises(ValueError, match="unknown census family"): + require_adjudicated_uk_local_binding( + ["not_a_family/constituency"], + problem.target_frame, + ) + with pytest.raises(ValueError, match="unsupported area_type"): + require_adjudicated_uk_local_binding( + ["census_households/ward"], + problem.target_frame, + ) + + +def test_rowwise_binding_refuses_expired_and_premature_adjudications() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + expired_register = { + "census_disclosure_control_noise": _reviewed_register_entry( + approved_on="2026-01-01", + expires_on="2026-02-01", + ) + } + with pytest.raises(ValueError, match="correct the underlying gap or renew"): + require_adjudicated_uk_local_binding( + [ + "census_households/constituency", + "tenure/constituency", + ], + problem.target_frame, + register=expired_register, + now=date(2026, 3, 1), + ) + + premature_register = { + "census_disclosure_control_noise": _reviewed_register_entry( + approved_on="2026-04-01", + expires_on="2027-04-01", + ) + } + with pytest.raises(ValueError, match="correct the underlying gap or renew"): + require_adjudicated_uk_local_binding( + [ + "census_households/constituency", + "tenure/constituency", + ], + problem.target_frame, + register=premature_register, + now=date(2026, 3, 1), + ) + + +def test_rowwise_binding_warns_within_week_of_expiry() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + register = { + "census_disclosure_control_noise": _reviewed_register_entry( + approved_on="2026-01-01", + expires_on="2026-03-05", + ) + } + with pytest.warns(UserWarning, match="within one week"): + receipt = require_adjudicated_uk_local_binding( + ["census_households/constituency", "tenure/constituency"], + problem.target_frame, + register=register, + now=date(2026, 3, 1), + ) + assert ( + "census_disclosure_control_noise" + in receipt["stood_on"]["census_households/constituency"] + ) + + # Far from expiry the same in-force entry passes silently. + import warnings as _warnings + + with _warnings.catch_warnings(): + _warnings.simplefilter("error") + require_adjudicated_uk_local_binding( + ["census_households/constituency", "tenure/constituency"], + problem.target_frame, + register=register, + now=date(2026, 1, 15), + ) + + +def test_committed_local_binding_register_references_committed_census() -> None: + census = load_uk_local_target_census() + fence_ids = {row["fence_id"] for row in census["binding_fences"]} + register = load_uk_reviewed_exclusion_register( + None, + resource=UK_LOCAL_BINDING_ADJUDICATION_REGISTER_RESOURCE, + ) + assert set(register) <= fence_ids + for fence_id, record in register.items(): + assert f"uk_local_target_census.json#/binding_fences/{fence_id}" in ( + record.reason + ) + assert len(record.reason) > 100 def test_rowwise_doctrine_solve_exposes_no_knobs() -> None: @@ -178,7 +359,10 @@ def test_rowwise_doctrine_solve_exposes_no_knobs() -> None: solve_uk_rowwise_weights_under_doctrine( _clone_frame(), problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], target_loss_weights=[1.0] * 4, ) @@ -198,7 +382,10 @@ def test_rowwise_doctrine_solve_refuses_duplicate_surface() -> None: solve_uk_rowwise_weights_under_doctrine( _clone_frame(), doctored, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=1, ) @@ -226,27 +413,27 @@ def test_matrix_builder_fails_closed_on_unreachable_nonzero_targets() -> None: targets = pd.DataFrame( { "code": ["E001", "S001", "W001"], - "population": [4.0, 2.0, 5.0], - "uc_households": [1.0, 1.0, 0.0], + "households": [4.0, 2.0, 5.0], + "tenure/social_rent": [1.0, 1.0, 0.0], } ) - with pytest.raises(ValueError, match="W001/population"): + with pytest.raises(ValueError, match="W001/households"): build_uk_rowwise_local_matrix(_metrics(), _assigned(), targets) zero_ok = targets.copy() - zero_ok.loc[zero_ok["code"] == "W001", "population"] = 0.0 + zero_ok.loc[zero_ok["code"] == "W001", "households"] = 0.0 problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), zero_ok) assert problem.n_areas == 3 def test_matrix_builder_refuses_duplicate_and_metadata_metric_labels() -> None: duplicated = _metrics() - duplicated.columns = ["population", "population"] + duplicated.columns = ["households", "households"] with pytest.raises(ValueError, match="duplicate column label"): build_uk_rowwise_local_matrix(duplicated, _assigned(), _targets()) - metadata = _metrics().rename(columns={"uc_households": "area_index"}) - targets = _targets().rename(columns={"uc_households": "area_index"}) + metadata = _metrics().rename(columns={"tenure/social_rent": "area_index"}) + targets = _targets().rename(columns={"tenure/social_rent": "area_index"}) with pytest.raises(ValueError, match="metadata"): build_uk_rowwise_local_matrix(metadata, _assigned(), targets) @@ -257,7 +444,10 @@ def test_rowwise_solve_refuses_dead_rows_and_misaligned_frames() -> None: solve_uk_rowwise_weights_under_doctrine( _clone_frame([1.0, 0.0, 1.0]), problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=1, ) @@ -290,7 +480,10 @@ def test_rowwise_solve_refuses_dead_rows_and_misaligned_frames() -> None: solve_uk_rowwise_weights_under_doctrine( reordered, reordered_problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=1, ) @@ -312,7 +505,10 @@ def test_rowwise_solve_refuses_dead_rows_and_misaligned_frames() -> None: solve_uk_rowwise_weights_under_doctrine( misaligned, problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=1, ) @@ -365,7 +561,10 @@ def spy(frame, targets, **kwargs): solve_uk_rowwise_weights_under_doctrine( _clone_frame(), problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], seed=3, ) @@ -405,6 +604,9 @@ def reordering(frame, targets, **kwargs): solve_uk_rowwise_weights_under_doctrine( _clone_frame(), problem, - bound_families=["census_households/constituency"], + bound_families=[ + "census_households/constituency", + "tenure/constituency", + ], epochs=1, ) diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py index 4e3800071a..eac8d83b39 100644 --- a/packages/microcosm-build/tests/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -243,6 +243,32 @@ def test_exclusion_applier_returns_pruned_registry_and_receipt(): ) +def test_exclusion_applier_warns_within_week_of_expiry(): + registry = TargetRegistry( + [ + TargetSpec(name="drop", entity="person", measure="drop", value=1.0, source="test"), + ], + country="uk", + ) + + from datetime import date + + window = _entry(name="drop", reason="reviewed") + # expires_on is 2026-11-25: five days out warns, mid-window is silent. + with pytest.warns(UserWarning, match="within one week"): + apply_uk_calibration_measure_exclusions( + registry, (window,), now=date(2026, 11, 20) + ) + + import warnings as _warnings + + with _warnings.catch_warnings(): + _warnings.simplefilter("error") + apply_uk_calibration_measure_exclusions( + registry, (window,), now=date(2026, 9, 1) + ) + + #: The adjudicated register census: one entry per class of the #757 #: ``uk_target_fit`` dispositions (issue comment 5427936411) plus the #: standing salary-sacrifice adjudication. The counts are the record of diff --git a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py index 5204c3e629..b7965cadd4 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py @@ -275,6 +275,19 @@ def test_candidate_build_writes_calibrated_h5_and_evidence(tmp_path) -> None: manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) assert manifest["candidate_scope"] == "adjudicated_partial" assert manifest["bound_target_families"] == ["census_households/constituency"] + adjudications = manifest["binding_adjudications"] + assert adjudications["register_resource"] == "local_binding_adjudications.json" + assert adjudications["bound_families"] == [ + "census_households/constituency" + ] + assert adjudications["evaluated_on"] + seed = adjudications["stood_on"]["census_households/constituency"][ + "census_disclosure_control_noise" + ] + assert seed["approved_by"] == "juaristi22" + assert seed["approved_on"] == "2026-08-27" + assert seed["expires_on"] == "2026-11-27" + assert adjudications["dormant"] == [] assert manifest["ladder_target_provenance"] == ladder_target_provenance(ladder) assert manifest["gate"]["passed"] is True assert manifest["gate"]["phase"] == "post_calibration" @@ -406,6 +419,17 @@ def forbidden(*_args, **_kwargs): plan = json.loads(captured.out) assert plan["dry_run"] is True assert plan["bound_target_families"] == ["census_households/constituency"] + adjudications = plan["binding_adjudications"] + assert adjudications["register_resource"] == "local_binding_adjudications.json" + assert adjudications["bound_families"] == [ + "census_households/constituency" + ] + assert adjudications["evaluated_on"] + assert ( + "census_disclosure_control_noise" + in adjudications["stood_on"]["census_households/constituency"] + ) + assert adjudications["dormant"] == [] assert plan["ladder_target_provenance"] == ladder_target_provenance(ladder) assert plan["shapes"]["person"][0] == 8 assert plan["shapes"]["benunit"][0] == 8 @@ -476,6 +500,55 @@ def failing_gate(*_args, **_kwargs): assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") +def test_candidate_binding_adjudication_failure_records_failed_row( + monkeypatch, + tmp_path, +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + ladder_path = tmp_path / "ladder.npz" + output_dir = tmp_path / "candidate" + _write_staging_h5(input_h5) + _write_ladder(ladder_path) + + import microcosm.build.uk_runtime.local_rowwise as local_rowwise + + monkeypatch.setattr( + local_rowwise, + "load_uk_reviewed_exclusion_register", + lambda *_args, **_kwargs: {}, + ) + + with pytest.raises(ValueError, match="census_disclosure_control_noise"): + builder.main( + [ + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--out", + str(output_dir), + "--n-clones", + "2", + "--seed", + "7", + "--epochs", + "2", + ] + ) + + rows = _spool_rows(output_dir) + assert len(rows) == 1 + row = rows[0] + assert row.disposition == "failed" + assert "targets_bound" in row.phases_reached + assert "solved" not in row.phases_reached + assert row.gate_verdicts["pipeline_error"]["verdict"] == "error" + assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") + + def test_candidate_setup_failure_records_failed_row(monkeypatch, tmp_path) -> None: """A pre-solve setup failure (ladder load) still spools a failed row. diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index bd1dacaf69..34b18da459 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -65,6 +65,7 @@ ladder_target_provenance, load_uk_national_frame, load_uk_oa_ladder, + require_adjudicated_uk_local_binding, rowwise_area_support_summary, solve_uk_rowwise_weights_under_doctrine, uk_geography_ladder_gate, @@ -374,6 +375,10 @@ def _run_candidate( append_phase(state, "targets_bound") if args.dry_run: + binding_adjudications = require_adjudicated_uk_local_binding( + BOUND_TARGET_FAMILIES, + problem.target_frame, + ) _assert_artifacts_unchanged( input_h5=input_h5, input_artifact=input_artifact, @@ -388,6 +393,7 @@ def _run_candidate( input_artifact=input_artifact, ladder_artifact=ladder_artifact, target_provenance=target_provenance, + binding_adjudications=binding_adjudications, ) print(_json_text(plan), end="") return 0 @@ -620,6 +626,7 @@ def _dry_run_plan( input_artifact: Mapping[str, Any], ladder_artifact: Mapping[str, Any], target_provenance: Mapping[str, Any], + binding_adjudications: Mapping[str, Any], ) -> dict[str, Any]: return { "schema_version": 1, @@ -627,6 +634,7 @@ def _dry_run_plan( "dry_run": True, "candidate_scope": "adjudicated_partial", "bound_target_families": list(BOUND_TARGET_FAMILIES), + "binding_adjudications": dict(binding_adjudications), "ladder_target_provenance": dict(target_provenance), "inputs": { "dataset": dict(input_artifact), @@ -746,6 +754,7 @@ def _manifest( "created_at": datetime.now(UTC).isoformat(), "git_commit": _git_commit(), "bound_target_families": list(BOUND_TARGET_FAMILIES), + "binding_adjudications": dict(solve.binding_adjudications), "ladder_target_provenance": dict(target_provenance), "parameters": _parameters(args, source_year=source_year), "inputs": {