diff --git a/changelog.d/623-uk-calibration-review-fixes.fixed.md b/changelog.d/623-uk-calibration-review-fixes.fixed.md new file mode 100644 index 0000000000..bc8d55abec --- /dev/null +++ b/changelog.d/623-uk-calibration-review-fixes.fixed.md @@ -0,0 +1,7 @@ +Count two-child-limit child targets as counts rather than household +indicators, attribute band edges to their own groupby dimension, make the +measure-provider fence answer only for routes it can take, put calibration +attempts on the ratified `uk/frs` Logbook chain with failure rows and unique +attempt ids, seal the verified Ledger identity into the run receipts, refuse +operator-supplied measure exclusions on release candidates, and let the rule-1 +scorer materialize and authenticate what it scores. diff --git a/changelog.d/623-uk-national-run-readiness.added.md b/changelog.d/623-uk-national-run-readiness.added.md new file mode 100644 index 0000000000..0ccf2d0383 --- /dev/null +++ b/changelog.d/623-uk-national-run-readiness.added.md @@ -0,0 +1,3 @@ +Add UK national Ledger-calibration run-readiness tooling, shared diagnostics, +synthetic incumbent scoring, and the held-run staging-candidate posture for +the first calibrated candidate. diff --git a/changelog.d/623-uk-target-surface-gate-note.changed.md b/changelog.d/623-uk-target-surface-gate-note.changed.md new file mode 100644 index 0000000000..0460eb56db --- /dev/null +++ b/changelog.d/623-uk-target-surface-gate-note.changed.md @@ -0,0 +1,4 @@ +State what the UK `uk_target_surface` gate proves — the solve's realized +target surface equals the declared register at name-and-period grain — rather +than claiming an incumbent comparison the frozen parity instrument cannot +source. Gate manifest and spec-fingerprint pins recomputed. diff --git a/docs/uk-national-calibration-runbook-623.md b/docs/uk-national-calibration-runbook-623.md new file mode 100644 index 0000000000..d7907f1c7c --- /dev/null +++ b/docs/uk-national-calibration-runbook-623.md @@ -0,0 +1,94 @@ +# UK national calibration runbook (#623) + +This runbook documents the held licensed run for the first Ledger-calibrated UK +national candidate. It is not an instruction to run it in PR CI. + +## Unblock conditions + +Run only after one of these is true: + +- WS-E spine is complete through E8 (#684) and E10 (#686). +- Maria explicitly names a different base spine for this run. + +The input posture is a non-certified staging candidate at pre-clone grain, +matching the incumbent published national surface. The seam records that +posture itself and marks its artifact `shippable: false`: release certification +is the release-cut producer's job, not calibration's. + +## Command shape + +Calibration runs through `tools/calibrate_uk_national_dataset.py`. That driver +is the only one that builds the measure resolver and applies the committed +measure-exclusion register, and 187 of the activated references bind model +outputs that no frame carries — so it is the only path on which this target +surface materializes. The June builder +(`tools/build_uk_national_dataset.py`) constructs the calibration stage without +either and aborts on the first unmaterializable reference; it also rebuilds SPI +income onto its input, which a spine artifact already carries. + +```bash +uv run --no-sync python tools/calibrate_uk_national_dataset.py \ + --input-h5 data/ukds/acceptance/623-first-calibrated-candidate/input-spine.h5 \ + --input-sha256 \ + --ledger-facts \ + --ledger-facts-sha256 \ + --ledger-manifest-sha256 \ + --staging-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ + --diagnostics-json data/ukds/acceptance/623-first-calibrated-candidate/calibration_diagnostics.json \ + --build-record-json data/ukds/acceptance/623-first-calibrated-candidate/build_record.json \ + --terminal-gate-json data/ukds/acceptance/623-first-calibrated-candidate/terminal_gates.json \ + --release-id dev-623-first-calibrated-candidate +``` + +The diagnostics digest is measured, not declared: the seam writes the +diagnostics file, hashes its actual bytes, and only then constructs and signs +the terminal gate evidence. There is no `--calibration-diagnostics-sha256` to +supply, and no way for the receipt to claim an identity the file does not have. + +Solve parameters are per-run overrides, not defaults. The campaign settings are +`--epochs 1500 --target-weight-rule family_equal`; each override is validated +through the doctrine dataclass and echoed as an explicit deviation in the +manifest, diagnostics and build record. `--release-candidate` refuses every +override flag, and refuses `--measure-exclusions`, so a release candidate is +always solved under declared doctrine against the committed target surface. + +Signing the terminal gate report needs +`MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY` in the environment. + +## Scoring + +Rule 1 is decided by a separate tool run against the staged artifact, so run +identity never depends on the incumbent's bytes: + +```bash +uv run --no-sync python tools/score_uk_national_candidate.py \ + --candidate-h5 data/ukds/acceptance/623-first-calibrated-candidate/populace_uk_2023.h5 \ + --candidate-sha256 \ + --incumbent-h5 \ + --incumbent-sha256 \ + --registry-json data/ukds/acceptance/623-first-calibrated-candidate/frozen-register.json \ + --output-json data/ukds/acceptance/623-first-calibrated-candidate/score_vs_enhanced_frs.json +``` + +Both artifacts are verified against the supplied digests before they are read, +and both sides are scored on the same frozen register. + +## Evidence directory + +`data/ukds/acceptance/623-first-calibrated-candidate/` should contain: + +- `build_record.json` +- `terminal_gates.json` +- `calibration_diagnostics.json` +- `score_vs_enhanced_frs.json` +- `logbook-spool/` (one row for the attempt, whatever its disposition) + +Acceptance follows #578: the candidate must not regress incumbent battery +observables, and the score block decides rule 1. A rule-1 loss is evidence for +#686/#736, not a threshold-edit instruction. + +The calibration battery is scoped to the calibration-relevant gates; the +spine-construction and imputation gates are out of scope here and are listed in +the report as scope exclusions with their rationale. A publishable +certification combines this with the spine build's own battery, which is +release-cut work (#757). diff --git a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py index b903cb5a21..4d83c2f549 100644 --- a/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/frame_serializer_registry.py @@ -77,7 +77,7 @@ class HdfWriteExclusion: FrameSerializerSpec( serializer_id="uk_single_year_h5", writer=HdfWriteSite( - "packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py", + "packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py", "_write_uk_single_year_tables", ), backend="pandas.HDFStore table", @@ -177,7 +177,7 @@ class HdfWriteExclusion: HdfWriteExclusion( exclusion_id="uk_weight_root_attrs", writer=HdfWriteSite( - "packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py", + "packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py", "_write_weight_metadata", ), reason="Adds weight-kind and mass-log root attributes only.", diff --git a/packages/microcosm-build/src/microcosm/build/target_materialization.py b/packages/microcosm-build/src/microcosm/build/target_materialization.py index e5209134f9..79fdfb228c 100644 --- a/packages/microcosm-build/src/microcosm/build/target_materialization.py +++ b/packages/microcosm-build/src/microcosm/build/target_materialization.py @@ -2,9 +2,12 @@ from __future__ import annotations -from collections.abc import Callable, Mapping +import math +import re +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any +from types import MappingProxyType +from typing import Any, Protocol import numpy as np @@ -12,6 +15,46 @@ Provider = Callable[[Any, Mapping[str, Any], int | str], np.ndarray] +# Banded measures declare their slice in two places: the contract binding +# names the variable to band on (``groupby_variable``), and each compiled +# spec carries its own band's lower edge in Ledger filter metadata. Two +# encodings are in use — a numeric lower bound (HMRC SPI income bands) and a +# published range label (DWP award bands, in monthly units, hence +# ``band_period_factor``) — and both reduce to one lower edge, because no +# reference anywhere declares an upper bound. A band's upper edge is +# therefore its sibling's lower edge within the same contract target, and the +# top band runs to infinity. +# Entity-count indicators: a value of one per record of the owning entity. +_COUNT_VALUE_VARIABLES = frozenset( + {"household_count", "person_count", "benunit_count"} +) + +_BAND_LOWER_BOUND_SUFFIX = "_lower_bound" +_LEDGER_FILTER_PREFIX = "ledger_filter_" +_RANGE_LABEL = re.compile( + r"([\d,]+(?:\.\d+)?)\s*(?:to|–|—|-)\s*(?:£\s*)?([\d,]+(?:\.\d+)?)", + re.IGNORECASE, +) +_OPEN_LABEL = re.compile( + r"([\d,]+(?:\.\d+)?)\s*(?:or more|or over|and over|and above|\+)", + re.IGNORECASE, +) +_MISSING_COLUMN = re.compile(r"^'(?:([a-z_0-9]+)\.)?([A-Za-z_0-9]+)'$") + + +class MeasureProvider(Protocol): + """Provider of adapter-level variables needed for target materialization. + + ``knows`` answers whether the provider can compute a variable for a given + entity. ``compute`` returns the values and a short receipted route. The + returned array is injected into fresh probe adapters only; it is never + written back to the source frame. + """ + + def knows(self, entity: str, variable: str) -> bool: ... + + def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: ... + @dataclass(frozen=True) class MaterializationSkip: @@ -22,6 +65,22 @@ class MaterializationSkip: reason: str +@dataclass(frozen=True) +class MeasureResolution: + """Adapter-level variables resolved for target materialization.""" + + measure_inputs: Mapping[tuple[str, str], np.ndarray] + receipt: Mapping[str, Any] + + +class MeasureResolutionError(RuntimeError): + """Raised when simulated measure resolution cannot bind the registry.""" + + def __init__(self, message: str, *, receipt: Mapping[str, Any]): + super().__init__(message) + self.receipt = dict(receipt) + + @dataclass(frozen=True) class TargetMaterializationResult: """Prepared target columns and skipped-measure diagnostics.""" @@ -40,6 +99,329 @@ def report(self) -> dict[str, object]: } +def assert_calibration_input_finite(frame: Any) -> None: + """Refuse NaN-bearing float inputs before calibration materialization.""" + + failures: list[str] = [] + for entity in frame.entities: + table = frame.table(entity) + for column in table.columns: + series = table[column] + if getattr(series.dtype, "kind", None) != "f": + continue + count = int(series.isna().sum()) + if count: + failures.append(f"{entity}.{column}: {count}") + if failures: + raise ValueError( + "Calibration input contains NaN float values: " + "; ".join(failures) + ) + + +def resolve_target_measures( + adapter_factory: Callable[[], Any], + registry: TargetRegistry, + provider: MeasureProvider, + *, + period: int | str, + max_rounds: int = 8, + contract_targets: Mapping[str, Mapping[str, Any]] | None = None, +) -> MeasureResolution: + """Resolve provider-computed inputs until target materialization binds. + + ``adapter_factory`` must return a fresh adapter over the same frame each + round. Previously resolved values are injected into those probe adapters' + table copies only, then the shared target-binding interpreter is retried. + + The source frame is never mutated — the guarantee rests on the adapter + copying each entity table at construction (``UKFrameTargetAdapter`` + does), not on the later ``restore``. Any adapter passed here must copy; + ``test_uk_national_calibration.py`` holds that property directly, on the + source frame rather than on the restored output. + """ + + contract = _measure_resolution_contract( + registry, provider, contract_targets=contract_targets + ) + measure_inputs: dict[tuple[str, str], np.ndarray] = {} + receipt: dict[str, Any] = { + "rounds": [], + "attached": {}, + "provider": _measure_provider_receipt(provider), + } + all_skips: list[dict[str, Any]] = [] + + for round_index in range(max_rounds): + probe = adapter_factory() + _inject_measure_inputs(probe, measure_inputs) + result = materialize_target_bindings( + probe, + registry, + contract, + period=period, + ) + skipped = tuple(result.skipped) + round_receipt = { + "round": round_index, + "provided_before": len(measure_inputs), + "skipped": [skip.__dict__ for skip in skipped], + } + receipt["rounds"].append(round_receipt) + all_skips.extend( + {**skip.__dict__, "round": round_index} for skip in skipped + ) + if not skipped: + return MeasureResolution( + measure_inputs=MappingProxyType(dict(measure_inputs)), + receipt=MappingProxyType(receipt), + ) + + provided_before = set(measure_inputs) + provided_this_round: set[tuple[str, str]] = set() + progressed = False + for skip in skipped: + reason = str(skip.reason) + if "counterfactual_delta" in reason or "counterfactual delta" in reason: + _raise_measure_resolution( + "counterfactual target measure cannot be resolved", + receipt, + all_skips, + ) + match = _MISSING_COLUMN.match(reason) + if match is None: + _raise_measure_resolution( + f"unparseable target materialization skip reason: {reason}", + receipt, + all_skips, + ) + entity, variable = match.group(1), match.group(2) + if entity is None: + entity = _provider_entity_for(provider, variable) + if entity is None: + _raise_measure_resolution( + f"target measure {variable!r} did not identify an entity", + receipt, + all_skips, + ) + key = (entity, variable) + if key in provided_this_round: + continue + if key in provided_before: + _raise_measure_resolution( + f"{entity}.{variable} remained unmaterializable after injection", + receipt, + all_skips, + ) + if not provider.knows(entity, variable): + _raise_measure_resolution( + f"provider does not know {entity}.{variable}", + receipt, + all_skips, + ) + try: + values, route = provider.compute(entity, variable) + except Exception as exc: # noqa: BLE001 - receipt preserves cause + _raise_measure_resolution( + f"provider failed computing {entity}.{variable}: {exc}", + receipt, + all_skips, + ) + measure_inputs[key] = np.asarray(values) + provided_this_round.add(key) + receipt["attached"][f"{entity}.{variable}"] = route + progressed = True + if not progressed: + _raise_measure_resolution( + f"measure materialization made no progress on round {round_index}", + receipt, + all_skips, + ) + + _raise_measure_resolution( + f"measure materialization did not converge after {max_rounds} rounds", + receipt, + all_skips, + ) + + +def _measure_resolution_contract( + registry: TargetRegistry, + provider: MeasureProvider, + *, + contract_targets: Mapping[str, Mapping[str, Any]] | None, +) -> Mapping[str, Mapping[str, Any]]: + if contract_targets is not None: + return contract_targets + for owner in (registry, provider): + value = getattr(owner, "contract_targets", None) + if value is not None: + return value + raise TypeError( + "resolve_target_measures requires contract_targets, or a registry/provider " + "with a contract_targets attribute." + ) + + +def _measure_provider_receipt(provider: MeasureProvider) -> Mapping[str, Any]: + receipt = getattr(provider, "receipt", None) + if callable(receipt): + return dict(receipt()) + return {"class": type(provider).__name__} + + +def _provider_entity_for(provider: MeasureProvider, variable: str) -> str | None: + entity_for = getattr(provider, "entity_for", None) + if callable(entity_for): + value = entity_for(variable) + return None if value is None else str(value) + return None + + +def _inject_measure_inputs( + adapter: Any, measure_inputs: Mapping[tuple[str, str], np.ndarray] +) -> None: + for (entity, variable), values in measure_inputs.items(): + adapter.tables[entity][variable] = values + + +def _raise_measure_resolution( + message: str, + receipt: dict[str, Any], + skips: list[dict[str, Any]], +) -> None: + receipt["skips"] = list(skips) + raise MeasureResolutionError(message, receipt=receipt) + + +def _band_lower_edge( + spec: Any, binding: Mapping[str, Any] +) -> float | None: + """This spec's band lower edge in model units, or None if unbanded. + + Reads the compiled spec's Ledger filter metadata: a ``*_lower_bound`` + entry when the publisher gives numeric edges, otherwise a range label + such as ``"£500.01 to £600.00"`` or ``"5 or more"``. ``band_period_factor`` + converts published units to the model's (monthly award bands to annual). + Non-band filters — ``ledger_filter_family_type = "Single, no children"`` + — carry no numeric range and are ignored here; the binding's own + ``filters`` list applies those. + """ + + metadata = getattr(spec, "metadata", None) or {} + factor = float(binding.get("band_period_factor", 1) or 1) + numeric: list[tuple[str, float]] = [] + labelled: list[tuple[str, float]] = [] + for key, value in sorted(metadata.items()): + if not key.startswith(_LEDGER_FILTER_PREFIX): + continue + if key.endswith(_BAND_LOWER_BOUND_SUFFIX): + numeric.append((key, float(str(value).replace(",", "")))) + continue + match = _RANGE_LABEL.search(str(value)) or _OPEN_LABEL.search(str(value)) + if match is not None: + labelled.append((key, float(match.group(1).replace(",", "")))) + for candidates in (numeric, labelled): + if not candidates: + continue + return _select_band_edge(spec, binding, candidates) * factor + return None + + +def _select_band_edge( + spec: Any, + binding: Mapping[str, Any], + candidates: Sequence[tuple[str, float]], +) -> float: + """Pick the band edge belonging to this binding's groupby variable. + + A spec can carry several Ledger filters, and more than one of them can + look like a band — an age filter alongside an income band, say. Slicing + on the wrong variable's edges produces plausible-looking but wrong + subpopulation totals, so the edge is bound to the declared banding + dimension and an unresolvable tie refuses rather than picking the + alphabetically-first key. + """ + + if len(candidates) == 1: + return candidates[0][1] + declared = binding.get("band_filter_dimension") + wanted = { + f"{_LEDGER_FILTER_PREFIX}{name}" + for name in (declared, binding.get("groupby_variable")) + if name + } + matched = [ + value + for key, value in candidates + if any(key == name or key.startswith(f"{name}_") for name in wanted) + ] + if len(matched) == 1: + return matched[0] + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} carries " + f"{len(candidates)} band-like Ledger filters " + f"({', '.join(key for key, _ in candidates)}) and " + f"{'none' if not matched else 'several'} of them belong to " + f"groupby_variable {binding.get('groupby_variable')!r}; declare " + "band_filter_dimension on the binding to name the banding dimension." + ) + + +def _band_edges_by_group( + registry: TargetRegistry, + contract_targets: Mapping[str, Mapping[str, Any]], +) -> dict[str, list[float]]: + """Sorted band lower edges per contract target that declares bands. + + Grouped by contract target rather than by dimension: two measures can + share a dimension while publishing different band sets, and taking the + union of their edges would slice each measure on the other's boundaries. + """ + + edges: dict[str, set[float]] = {} + for spec in registry.specs: + contract_target_id = str( + (getattr(spec, "metadata", None) or {}).get("contract_target_id", "") + ) + target = contract_targets.get(contract_target_id) + if target is None: + continue + binding = target.get("bindings", {}).get("policyengine", {}) + if not binding.get("groupby_variable"): + continue + lower = _band_lower_edge(spec, binding) + if lower is not None: + edges.setdefault(contract_target_id, set()).add(lower) + return {key: sorted(values) for key, values in edges.items()} + + +def _band_bounds( + spec: Any, + binding: Mapping[str, Any], + band_edges: Sequence[float], +) -> tuple[float, float]: + """The half-open ``[lower, upper)`` this spec's band covers. + + Raises rather than returning an unsliced column: a banded measure whose + band cannot be read would otherwise report the whole population as if it + were one band, which is the failure this function exists to end. + """ + + lower = _band_lower_edge(spec, binding) + if lower is None: + raise ValueError( + f"banded measure {getattr(spec, 'measure', '?')!r} declares " + f"groupby_variable {binding['groupby_variable']!r} but its spec " + "carries no readable band edge" + ) + upper = math.inf + for edge in band_edges: + if edge > lower: + upper = edge + break + return lower, upper + + def materialize_target_bindings( adapter: Any, registry: TargetRegistry, @@ -51,6 +433,7 @@ def materialize_target_bindings( """Prepare measure columns declared by compiled Ledger target specs.""" provider_registry = {**default_provider_registry(), **(providers or {})} + band_edges = _band_edges_by_group(registry, contract_targets) skipped: list[MaterializationSkip] = [] for spec in registry.specs: if hasattr(adapter, "has_column") and adapter.has_column( @@ -77,7 +460,16 @@ def materialize_target_bindings( raise ValueError(f"unsupported binding kind {kind!r}") values = provider(adapter, binding, period) else: - values = _prepared_column_values(adapter, spec.entity, binding) + band = None + if binding.get("groupby_variable"): + band = _band_bounds( + spec, + binding, + band_edges.get(str(contract_target_id), ()), + ) + values = _prepared_column_values( + adapter, spec.entity, binding, band=band + ) adapter.set_column(spec.entity, spec.measure, values) except (KeyError, TypeError, ValueError) as error: skipped.append( @@ -139,7 +531,15 @@ def baseline_flag_crosstab( flag = _column(adapter, entity, binding["affected_flag_variable"]).astype(bool) value_variable = str(binding.get("value_variable") or "") count_of = str(binding.get("count_of") or "") - if value_variable and value_variable not in {"household_count", "person_count"}: + reduction = binding.get("value_reduction") + if reduction: + # A count declared over a member-level variable — the number of + # children in the household, not whether it has any. Reading such a + # boolean at the crosstab's own grain would collapse it to an + # indicator and publish that against a count target, so the reduction + # is declared and performed member-wise instead. + values = _entity_reduction(adapter, reduction) + elif value_variable and value_variable not in _COUNT_VALUE_VARIABLES: values = _column(adapter, entity, value_variable) elif count_of and count_of not in {"household", "person", "household_count"}: values = _column(adapter, entity, count_of) @@ -169,6 +569,8 @@ def _prepared_column_values( adapter: Any, entity: str, binding: Mapping[str, Any], + *, + band: tuple[float, float] | None = None, ) -> np.ndarray: entity = str(binding.get("from_entity") or entity) if "value_expression" in binding: @@ -180,9 +582,32 @@ def _prepared_column_values( mask &= _predicate_mask(adapter, entity, predicate) for predicate in binding.get("household_conditions", ()): mask &= _predicate_mask(adapter, entity, predicate) + if band is not None: + lower, upper = band + banded = _column(adapter, entity, binding["groupby_variable"]).astype(float) + mask &= (banded >= lower) & (banded < upper) return np.where(mask, values, 0.0) +def _entity_reduction(adapter: Any, reduction: Mapping[str, Any]) -> np.ndarray: + """Reduce a member-level variable to the target entity's grain. + + Declared as ``value_reduction`` on a binding whose value is a count over + members. The adapter owns the linkage, the same way it owns + ``household_condition``; an adapter that cannot reduce refuses rather than + falling back to a same-grain read, which would silently substitute an + indicator for a count. + """ + + if not hasattr(adapter, "entity_reduction"): + raise ValueError( + "binding declares value_reduction " + f"{dict(reduction)!r} but the adapter cannot reduce member " + "variables; refusing to substitute a same-grain read." + ) + return np.asarray(adapter.entity_reduction(reduction), dtype=float) + + def _predicate_mask( adapter: Any, default_entity: str, diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json new file mode 100644 index 0000000000..087d6637a8 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "exclusions": [ + { + "name": "hmrc.salary_sacrifice.it_relief_basic_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.it_relief_higher_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.it_relief_additional_rate", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employee", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + }, + { + "name": "hmrc.salary_sacrifice.nics_relief_employer", + "reason": "The frame adapter reads only a precomputed slash-named delta column; no artifact ships a live counterfactual route.", + "tracking": "microcosm#623" + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 7883de48a5..aeacb95f06 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -67,6 +67,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "calibration_measure_exclusions.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "hmrc_cgt_size_bands.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index e9a1101bbf..51d48e864f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -304,7 +304,7 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": {}, - "notes": "Candidate and incumbent calibration target surfaces must agree exactly; the gate takes no tunable parameters." + "notes": "The solve's realized target surface must equal the declared register exactly, at name-and-period grain; the gate takes no tunable parameters. The reference side is the declared register, not an incumbent-derived surface \u2014 the frozen parity instrument carries input-column shares only \u2014 so this proves that every declared target was bound at its declared period, not agreement with the incumbent." }, { "id": "uk_target_fit", diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json index f36048e626..4cff07befa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_national_targets.json @@ -3619,10 +3619,9 @@ "metric_name": "dwp/uc/two_child_limit/households_affected", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/households_affected", @@ -3653,10 +3652,14 @@ "metric_name": "dwp/uc/two_child_limit/children_affected", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/children_affected", @@ -3687,10 +3690,14 @@ "metric_name": "dwp/uc/two_child_limit/children_in_affected_households", "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/children_in_affected_households", @@ -3730,10 +3737,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/3_children_households", @@ -3773,10 +3779,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/3_children_households_total_children", @@ -3816,10 +3826,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/4_children_households", @@ -3859,10 +3868,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/4_children_households_total_children", @@ -3902,10 +3915,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/5_children_households", @@ -3945,10 +3957,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/5_children_households_total_children", @@ -3988,10 +4004,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Publisher's top band is 6-or-more children." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Publisher's top band is 6-or-more children. Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/6_children_households", @@ -4031,10 +4046,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "children_in_affected_households", - "value_variable": "children_count", + "value_variable": "is_child", + "value_reduction": { + "variable": "is_child", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161)." + "notes": "Total children (not only affected children) in affected households of this size (compute/benefits.py:158-161). Row counts: children_in_affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/6_children_households_total_children", @@ -4074,10 +4093,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/adult_pip_households", @@ -4117,10 +4135,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/adult_pip_children", @@ -4160,10 +4182,9 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_households", "value_variable": "household_count", "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_households." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/disabled_child_element_households", @@ -4203,10 +4224,14 @@ ], "kind": "baseline_flag_crosstab", "affected_flag_variable": "uc_is_child_limit_affected", - "count_of": "affected_children", - "value_variable": "children_count", + "value_variable": "uc_is_child_limit_affected", + "value_reduction": { + "variable": "uc_is_child_limit_affected", + "entity": "person", + "reduce": "sum" + }, "map_to": "household", - "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148)." + "notes": "Affected children are persons flagged by uc_is_child_limit_affected (household round-trip) who are children in UC-receiving households; an affected household contains at least one such child (compute/benefits.py:130-148). Row counts: affected_children." }, "axiom": { "metric_name": "dwp/uc/two_child_limit/disabled_child_element_children", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index ba309d80b2..360485af82 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -315,6 +315,20 @@ load_uk_national_frame, write_uk_national_frame, ) +from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, + UKNationalSolveDoctrine, + uk_doctrine_with_overrides, + uk_national_target_loss_weights, +) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, UKStagingProvenance, @@ -657,8 +671,20 @@ "UKHMRCIncomeStageTransform", "UKHMRCTargetMaterialization", "UKNationalBuildResult", + "UKNationalSolveDoctrine", "UKStagingProvenance", + "UK_NATIONAL_L0_LAMBDA", + "UK_NATIONAL_LEARNING_RATE", + "UK_NATIONAL_MASS_RULE", + "UK_NATIONAL_MAX_WEIGHT_RATIO", "UK_NATIONAL_SCHEMA", + "UK_NATIONAL_SEED", + "UK_NATIONAL_SOLVE_DOCTRINE", + "UK_NATIONAL_SOLVE_EPOCHS", + "UK_NATIONAL_TARGET_LOSS_CAP", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_doctrine_with_overrides", + "uk_national_target_loss_weights", "UKNationalStage", "UKRowwiseDatasetResult", "UKReleaseInputColumn", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py new file mode 100644 index 0000000000..6e5e947b04 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -0,0 +1,661 @@ +"""Production UK national calibration seam orchestration.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import os +import time +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np + +from microcosm.build.country_spec import GatesManifest, load_country_spec +from microcosm.build.gate_battery import ( + BlockingMode, + EvidenceContext, + GateBatteryRun, + gate_signing_key_env, +) +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.logbook_adoption import ( + AttemptState, + append_phase, + apply_error_verdict, + error_receipt_path, + git_code_pin, + local_artifact_reference, + record_terminal_attempt, + resolve_predecessor, + role_pins_digest, + write_error_receipt, +) +from microcosm.build.target_materialization import assert_calibration_input_finite +from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY +from microcosm.build.uk_runtime.diagnostics import ( + uk_target_geography_levels, + write_uk_calibration_diagnostics, +) +from microcosm.build.uk_runtime.national_calibration import UKNationalCalibrationStage +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry +from microcosm.frame import Frame + +_REPOSITORY = Path(__file__).resolve().parents[6] +# The FRS line's spine, staging, imputation and calibration stages share one +# hash chain (logbook/README.md): the dataset token names the base data, not +# the build mechanism, so calibration derives the ratified `uk/frs` scope. +_PIPELINE = "uk-frs-calibration" + + +@dataclass(frozen=True) +class UKCalibrationRunPaths: + input_h5: Path + staging_h5: Path + diagnostics_json: Path + build_record_json: Path + terminal_gate_json: Path + + +@dataclass(frozen=True) +class UKCalibrationRunResult: + frame: Frame + diagnostics_sha256: str + staging_sha256: str + build_record_sha256: str + terminal_gate_sha256: str + logbook_spool: Path + gate_report: Mapping[str, object] + build_record: Mapping[str, object] + + +UK_CALIBRATION_GATE_SCOPE = ( + "uk_target_fit", + "uk_weight_ratio", + "uk_weight_ess", + "uk_zero_weight_strata", + "uk_aggregate_admin", + "uk_calibration_reference_coverage", +) + + +def _scope_exclusions() -> dict[str, str]: + full = {entry.id for entry in load_country_spec("uk").gates.gates} + excluded = full - set(UK_CALIBRATION_GATE_SCOPE) + rationales: dict[str, str] = {} + for gate_id in sorted(excluded): + if "parity" in gate_id or gate_id in {"uk_export_surface", "uk_target_surface"}: + reason = "swap-acceptance evidence; produced by the swap lane, not the calibration seam." + elif gate_id in { + "uk_release_input_coverage_manifest_current", + "uk_release_family_build_stages", + "uk_release_input_coverage", + "uk_nonnegative_columns", + "uk_support", + "uk_take_up_signal", + "uk_brma_enum_domain", + "uk_degenerate_release_surface", + "uk_input_mass_parity", + "uk_qrf_tail_concentration", + "uk_weights_audit", + }: + reason = "spine-construction gate; owned by the spine build's own battery." + else: + reason = "outside the calibration seam's reviewed gate scope." + rationales[gate_id] = reason + if set(UK_CALIBRATION_GATE_SCOPE) | set(rationales) != full: + raise RuntimeError("UK calibration gate scope does not classify every gate id.") + return rationales + + +UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS = _scope_exclusions() + + +def run_uk_calibration( + *, + paths: UKCalibrationRunPaths, + input_sha256: str, + ledger_artifact: Any, + register_registry: TargetRegistry, + calibration_year: int, + exclusion_receipt: Mapping[str, Mapping[str, str]], + doctrine: Any, + doctrine_overrides: Mapping[str, Mapping[str, object]], + measure_resolver: object | None, + source_pins: Mapping[str, Mapping[str, object]], + run_config_extra: Mapping[str, object], + release_candidate: bool, + release_id: str, + logbook_prev_row_digest: str | None = None, +) -> UKCalibrationRunResult: + """Run the UK national calibration seam and write its sidecars.""" + + started_at = time.perf_counter() + started_ts = datetime.now(UTC) + code_pin = git_code_pin(_REPOSITORY) + # Predecessor configuration is validated before anything is written: a + # disagreeing chain must refuse with no artifact on disk, not after a + # staged H5, diagnostics and a signed gate report already exist. + predecessor = resolve_predecessor(logbook_prev_row_digest) + run_config = { + "pipeline": _PIPELINE, + "release_id": release_id, + "register_sha256": register_registry.version, + "calibration_year": int(calibration_year), + "doctrine": _doctrine_payload(doctrine), + "doctrine_overrides": dict(doctrine_overrides), + # The caller verifies the feed's facts and manifest digests; sealing + # the verified identity into run_config carries it through the + # identity digest, the build record and the Logbook row, so the run + # says which Ledger artifact it was measured against. + "ledger": _ledger_provenance(ledger_artifact), + **dict(run_config_extra), + } + state = AttemptState( + # Attempts are distinct rows even when they re-run one release: both + # the local chain and the store refuse a repeated build id. + build_id=_new_calibration_attempt_id(timestamp=started_ts), + identity_digest=hashlib.sha256(canonical_json_bytes(run_config)).hexdigest(), + input_pins_digest=role_pins_digest(source_pins), + phases_reached=["attempt_started"], + gate_verdicts={}, + ) + spool_dir = paths.staging_h5.parent / "logbook-spool" + try: + return _run_uk_calibration_attempt( + paths=paths, + input_sha256=input_sha256, + ledger_artifact=ledger_artifact, + register_registry=register_registry, + calibration_year=calibration_year, + exclusion_receipt=exclusion_receipt, + doctrine=doctrine, + doctrine_overrides=doctrine_overrides, + measure_resolver=measure_resolver, + source_pins=source_pins, + release_candidate=release_candidate, + release_id=release_id, + state=state, + run_config=run_config, + code_pin=code_pin, + started_at=started_at, + started_ts=started_ts, + predecessor=predecessor, + spool_dir=spool_dir, + ) + except BaseException as error: + # Every terminal disposition records a row — successful, failed, or + # refused (logbook/README.md). A refusal that left no row would be a + # silent gap in the chain the run is supposed to evidence. + _record_failed_attempt( + error=error, + state=state, + started_at=started_at, + started_ts=started_ts, + seed=getattr(doctrine, "seed", None), + code_pin=code_pin, + predecessor=predecessor, + receipt_base_dir=paths.staging_h5.parent, + spool_dir=spool_dir, + ) + raise + + +def _new_calibration_attempt_id(*, timestamp: datetime) -> str: + instant = timestamp.astimezone(UTC) + return ( + "uk-frs-calibration-attempt-" + f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" + ) + + +def _record_failed_attempt( + *, + error: BaseException, + state: AttemptState, + started_at: float, + started_ts: datetime, + seed: int | None, + code_pin: str, + predecessor: str | None, + receipt_base_dir: Path, + spool_dir: Path, +) -> None: + if state.spool_path is not None: + return + error_path = write_error_receipt( + error_receipt_path(receipt_base_dir, build_id=state.build_id), + state=state, + pipeline=_PIPELINE, + error=error, + ) + apply_error_verdict( + state, + f"{local_artifact_reference(error_path, repository_hint=_REPOSITORY)}" + "#/error_type", + ) + record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung="f100", + seed=seed, + code_pin=code_pin, + # An operator interrupt is a discarded attempt, not a failed one; the + # row says which so the chain reads honestly. + disposition=( + "discarded" if isinstance(error, KeyboardInterrupt) else "failed" + ), + predecessor=predecessor, + spool_dir=spool_dir, + ) + + +def _run_uk_calibration_attempt( + *, + paths: UKCalibrationRunPaths, + input_sha256: str, + ledger_artifact: Any, + register_registry: TargetRegistry, + calibration_year: int, + exclusion_receipt: Mapping[str, Mapping[str, str]], + doctrine: Any, + doctrine_overrides: Mapping[str, Mapping[str, object]], + measure_resolver: object | None, + source_pins: Mapping[str, Mapping[str, object]], + release_candidate: bool, + release_id: str, + state: AttemptState, + run_config: Mapping[str, object], + code_pin: str, + started_at: float, + started_ts: datetime, + predecessor: str | None, + spool_dir: Path, +) -> UKCalibrationRunResult: + measured_input_sha = _sha256_file(paths.input_h5) + if measured_input_sha != input_sha256: + raise ValueError( + "input H5 sha mismatch: " + f"measured {measured_input_sha}, pinned {input_sha256}" + ) + append_phase(state, "input_sha_verified") + frame, _provenance = load_uk_national_frame(paths.input_h5) + append_phase(state, "input_loaded") + assert_calibration_input_finite(frame) + append_phase(state, "input_finite") + + stage = UKNationalCalibrationStage( + register_registry, + # The declared calibration year the register was compiled at — the + # stage validates it; there is deliberately no fallback to the input + # frame's base-year time_period or any ambient default. + period=calibration_year, + doctrine=doctrine, + measure_resolver=measure_resolver, + ) + calibrated = stage(frame) + append_phase(state, "national_calibration_solved") + + build_block = { + "build_id": state.build_id, + "code_pin": code_pin, + "source_pins": dict(source_pins), + "ledger": run_config["ledger"], + "input_posture": { + "tier": "staging_candidate", + "sha256": measured_input_sha, + "size_bytes": paths.input_h5.stat().st_size, + }, + "doctrine": _doctrine_payload(doctrine), + "doctrine_overrides": dict(doctrine_overrides), + "measure_exclusions": dict(exclusion_receipt), + "measure_resolution": ( + stage.manifest.get("measure_resolution") + if isinstance(stage.manifest, Mapping) + else None + ), + "register": _register_census(register_registry, exclusion_receipt), + "score_vs_enhanced_frs": None, + } + write_uk_calibration_diagnostics( + stage.solve_result, + paths.diagnostics_json, + calibrated, + target_geography_levels=uk_target_geography_levels(stage.registry), + target_registry=stage.registry, + build=build_block, + ) + diagnostics_sha = _sha256_file(paths.diagnostics_json) + append_phase(state, "diagnostics_written") + + gate_report = _run_calibration_gate_battery( + calibrated, + stage, + paths.terminal_gate_json, + release_candidate=release_candidate, + release_id=release_id, + diagnostics_sha256=diagnostics_sha, + ) + append_phase(state, "calibration_gates_evaluated") + for gate_id, payload in gate_report["gates"].items(): + state.gate_verdicts[gate_id] = { + "verdict": payload["status"], + "receipt": f"local://{paths.terminal_gate_json.name}#/gates/{gate_id}", + } + + write_uk_national_frame(calibrated, paths.staging_h5) + staging_sha = _sha256_file(paths.staging_h5) + append_phase(state, "staging_h5_written") + + record = { + "schema_version": 1, + "pipeline": _PIPELINE, + "build_id": state.build_id, + "run_config": run_config, + "source_pins": dict(source_pins), + "role_pins_digest": role_pins_digest(source_pins), + "input_posture": build_block["input_posture"], + "register": build_block["register"], + "calibration": stage.manifest, + "gate_summary": _gate_summary(gate_report), + "shippable": False, + "shippable_reason": ( + "calibration-scoped battery; release certification is the " + "release-cut producer's job" + ), + "artifacts": { + "staging_h5": {"path": str(paths.staging_h5), "sha256": staging_sha}, + "diagnostics_json": { + "path": str(paths.diagnostics_json), + "sha256": diagnostics_sha, + }, + "terminal_gate_json": { + "path": str(paths.terminal_gate_json), + "sha256": _sha256_file(paths.terminal_gate_json), + }, + }, + } + _write_json(paths.build_record_json, record) + build_record_sha = _sha256_file(paths.build_record_json) + append_phase(state, "build_record_written") + state.artifact_location = local_artifact_reference( + paths.staging_h5, repository_hint=_REPOSITORY + ) + spool = record_terminal_attempt( + state=state, + started_at=started_at, + started_ts=started_ts, + pipeline=_PIPELINE, + rung="f100", + seed=getattr(doctrine, "seed", None), + code_pin=code_pin, + disposition="iterating", + predecessor=predecessor, + spool_dir=spool_dir, + ) + return UKCalibrationRunResult( + frame=calibrated, + diagnostics_sha256=diagnostics_sha, + staging_sha256=staging_sha, + build_record_sha256=build_record_sha, + terminal_gate_sha256=_sha256_file(paths.terminal_gate_json), + logbook_spool=spool, + gate_report=gate_report, + build_record=record, + ) + + +def _run_calibration_gate_battery( + frame: Frame, + stage: UKNationalCalibrationStage, + path: Path, + *, + release_candidate: bool, + release_id: str, + diagnostics_sha256: str, +) -> dict[str, object]: + manifest = _calibration_gate_manifest() + admin_totals, admin_receipt = _aggregate_admin_totals(frame, manifest) + artifacts = { + "national_calibration": stage.manifest, + "parity_evidence": SimpleNamespace( + target_relative_errors={ + str(row["name"]): float(row["relative_error"]) + for row in stage.diagnostics + } + ), + "aggregate_admin": admin_totals, + } + battery = GateBatteryRun( + manifest, + release_id=release_id, + report_path=path, + release_candidate=release_candidate, + registry=UK_GATE_REGISTRY, + release_evidence={"calibration_diagnostics_sha256": diagnostics_sha256}, + ) + battery.run_phase("terminal", EvidenceContext(frame=frame, artifacts=artifacts)) + battery.enforce("terminal", mode=BlockingMode.BLOCKS_ARTIFACT) + payload = battery.report_payload() + payload["posture"] = "calibration_seam" + payload["scope_exclusions"] = dict(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + payload["aggregate_admin_measurement"] = admin_receipt + _resign_gate_report(payload) + _write_json(path, payload) + return payload + + +def _calibration_gate_manifest() -> GatesManifest: + source = load_country_spec("uk").gates + entries = tuple( + entry for entry in source.gates if entry.id in UK_CALIBRATION_GATE_SCOPE + ) + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; calibration_seam_scope", + phases=("terminal",), + gates=entries, + ) + + +def _aggregate_admin_totals( + frame: Frame, manifest: GatesManifest +) -> tuple[dict[str, float], list[dict[str, object]]]: + """Measure every declared admin anchor, fail-loud on absent evidence. + + The anchor value's magnitude tells its statistic — NEED per-household + means are hundreds of pounds, the NHS anchor is a national total — the + same reviewed convention the first armed-run receipts used. Non-household + entities carry their household's weight through the person linkage. + """ + + anchors = [] + for entry in manifest.gates: + if entry.id == "uk_aggregate_admin": + anchors = list(entry.parameters.get("anchors", ())) + break + household_weights = np.asarray( + frame.weights_for("household").values, dtype=float + ) + totals: dict[str, float] = {} + receipt: list[dict[str, object]] = [] + for anchor in anchors: + entity = str(anchor.get("entity", "household")) + name = str(anchor.get("name", anchor.get("measure"))) + measure = str(anchor.get("measure", anchor.get("name"))) + table = frame.table(entity) + if measure not in table: + raise ValueError( + f"aggregate_admin anchor {name!r} needs {entity}.{measure}, " + "which the calibrated frame does not carry; refusing to " + "fabricate a measured value." + ) + if entity == "household": + weights = household_weights + elif entity == "person": + # `household_id` is a group-table id column, which the Frame + # kernel validates unique before any frame exists, and the CGT + # clone stage offsets its clones' ids for exactly that reason — + # so this lookup cannot silently drop a duplicate key. + person = frame.table("person") + lookup = dict( + zip( + frame.table("household")["household_id"].to_numpy(), + household_weights, + strict=True, + ) + ) + weights = np.asarray( + [lookup[key] for key in person["person_household_id"].to_numpy()], + dtype=float, + ) + else: + raise ValueError( + f"aggregate_admin anchor {name!r} declares entity {entity!r}; " + "the calibration seam measures household and person anchors " + "only." + ) + values = table[measure].to_numpy(dtype=float) + total = float(np.dot(values, weights)) + carriers = values != 0 + carrier_weight = float(weights[carriers].sum()) + mean_carriers = ( + float(np.dot(values[carriers], weights[carriers]) / carrier_weight) + if carrier_weight + else float("nan") + ) + declared = float(anchor["value"]) + measured = mean_carriers if abs(declared) < 1e6 else total + totals[name] = measured + receipt.append( + { + "anchor": name, + "entity": entity, + "measure": measure, + "measured": measured, + "weighted_total": total, + "weighted_mean_carriers": mean_carriers, + "statistic_convention": "assessed_by_anchor_magnitude", + } + ) + return totals, receipt + + +def _ledger_provenance(artifact: Any) -> dict[str, object]: + """The verified identity of the Ledger consumer feed this run compiled. + + A bare ``consumer_facts.jsonl`` feed carries no manifest, so its + Ledger-side provenance is recorded as absent rather than invented. + """ + + provenance: dict[str, object] = { + "facts_sha256": getattr(artifact, "facts_sha256", None), + "fact_row_count": getattr(artifact, "fact_row_count", None), + "manifest_sha256": getattr(artifact, "manifest_sha256", None), + } + manifest = getattr(artifact, "manifest", None) + if isinstance(manifest, Mapping): + provenance["manifest"] = { + key: manifest.get(key) + for key in ("artifact_id", "profile", "schema_version", "generated_at") + if manifest.get(key) is not None + } + return provenance + + +def _register_census( + registry: TargetRegistry, exclusions: Mapping[str, Mapping[str, str]] +) -> dict[str, object]: + return { + "country": registry.country, + "version": registry.version, + "compiled_count": len(registry.specs) + len(exclusions), + "excluded_count": len(exclusions), + "calibrated_count": len(registry.specs), + } + + +def _doctrine_payload(doctrine: Any) -> dict[str, object]: + return { + key: getattr(doctrine, key) + for key in ( + "epochs", + "learning_rate", + "max_weight_ratio", + "seed", + "target_loss_cap", + "scale_rule", + "target_weight_rule", + "mass_rule", + "l0_lambda", + ) + if hasattr(doctrine, key) + } + + +def _gate_summary(report: Mapping[str, object]) -> dict[str, object]: + gates = report.get("gates", {}) + if not isinstance(gates, Mapping): + return {} + return { + gate_id: payload.get("status") + for gate_id, payload in gates.items() + if isinstance(payload, Mapping) + } + + +def _resign_gate_report(payload: dict[str, object]) -> None: + attestation = payload.get("attestation") + if not isinstance(attestation, dict): + raise RuntimeError("gate report has no attestation block.") + env = gate_signing_key_env("uk") + encoded = os.environ.get(env) + if not encoded: + raise RuntimeError( + f"{env} must be set; unsigned full-scale calibration runs refuse to stage." + ) + try: + key = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + raise RuntimeError(f"{env} must be valid base64.") from exc + if len(key) != 32: + raise RuntimeError(f"{env} must decode to exactly 32 bytes.") + attestation.pop("signing_error", None) + attestation["signing_key_sha256"] = hashlib.sha256(key).hexdigest() + attestation["signature"] = None + attestation["signature"] = hmac.new( + key, canonical_json_bytes(payload), hashlib.sha256 + ).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _write_json(path: Path, payload: Mapping[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index 732a523356..edd620a448 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -19,6 +19,7 @@ import math import uuid from collections.abc import Mapping, Sequence +from importlib import resources as importlib_resources from pathlib import Path from typing import Any @@ -37,6 +38,7 @@ "UK_DIAGNOSTICS_SCHEMA_VERSION", "UK_TARGET_GEOGRAPHY_LEVELS", "uk_calibration_diagnostics_payload", + "uk_target_geography_levels", "uk_weight_summary", "uk_zero_weight_strata", "write_uk_calibration_diagnostics", @@ -259,6 +261,33 @@ def _normalize_geography_level(value: object) -> str: return level +def uk_target_geography_levels(registry: TargetRegistry) -> dict[str, str]: + """Map declared UK target row names to their contract geography level.""" + + payload = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + targets = {row["target_id"]: row for row in payload["targets"]} + levels: dict[str, str] = {} + for spec in registry.specs: + target_id = spec.metadata.get("contract_target_id") + target = targets.get(str(target_id)) + if target is None: + raise ValueError( + f"UK calibration target {spec.name!r} references unknown " + f"contract target {target_id!r}." + ) + geography_levels = tuple(target.get("geography_levels") or ()) + if not geography_levels: + raise ValueError( + f"UK calibration target {spec.name!r} has no geography level." + ) + levels[spec.to_target().row_name] = str(geography_levels[0]) + return levels + + def _geography_mapping( target_names: Sequence[str], values: Mapping[str, object], diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py index d67b823ff1..520bb36693 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/hmrc_restoration.py @@ -12,6 +12,7 @@ from __future__ import annotations import hashlib +import re from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path @@ -91,6 +92,8 @@ "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" ) CERTIFIED_UK_CANDIDATE_SIZE_BYTES = 1_315_880_118 +STAGING_CANDIDATE_TIER = "staging_candidate" +STAGING_CANDIDATE_REVISION = "non_certified_staging_candidate" _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN = object() @@ -494,6 +497,50 @@ def verify_certified_uk_candidate(path: str | Path) -> UKCertifiedCandidateIdent return identity +def verify_staging_candidate_uk_input( + path: str | Path, + *, + expected_sha256: str, +) -> UKCertifiedCandidateIdentity: + """Hash-gate a declared non-certified staging-candidate input.""" + + candidate = Path(path).expanduser().resolve() + if not candidate.is_file(): + raise FileNotFoundError(f"UK staging-candidate input not found: {candidate}.") + expected = str(expected_sha256) + if not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ValueError( + "--staging-candidate-input-sha256 must be a lowercase SHA-256." + ) + fingerprint_before = _uk_source_file_fingerprint(candidate) + digest = _sha256(candidate) + fingerprint_after = _uk_source_file_fingerprint(candidate) + if fingerprint_after != fingerprint_before: + raise RuntimeError( + "UK staging-candidate input changed while its SHA-256 was being verified." + ) + if digest != expected: + raise ValueError( + f"{candidate}: sha256 {digest} does not match declared " + f"staging-candidate input {expected}." + ) + identity = UKCertifiedCandidateIdentity( + path=candidate, + filename=candidate.name, + tier=STAGING_CANDIDATE_TIER, + revision=STAGING_CANDIDATE_REVISION, + sha256=digest, + size_bytes=fingerprint_after.size_bytes, + ) + object.__setattr__( + identity, + "_verification_token", + _CERTIFIED_CANDIDATE_VERIFICATION_TOKEN, + ) + object.__setattr__(identity, "_source_file_fingerprint", fingerprint_after) + return identity + + def restore_uk_hmrc_income_family( frame: Frame, *, @@ -801,6 +848,24 @@ def _validate_certified_candidate_identity( identity.sha256, identity.size_bytes, ) + if identity.tier == STAGING_CANDIDATE_TIER: + # A declared staging-candidate input is deliberately not the certified + # artifact, so the certified-pin equality below cannot apply. The bytes + # are still bound, by the two checks above rather than by this branch: + # the verification token is a module-private sentinel that only + # verify_certified_uk_candidate and verify_staging_candidate_uk_input + # stamp, and the latter hashes the file against a mandatory declared + # sha256 (re-reading the fingerprint to catch a mid-read swap) and + # refuses on mismatch; the source-file fingerprint is required of both + # tiers. What this tier declares is "the file the operator named", and + # the build record labels it non_certified_staging_candidate while + # --staging-candidate-input-sha256 is refused for release candidates. + if identity.revision != STAGING_CANDIDATE_REVISION: + raise ValueError( + "UK staging-candidate input identity has an invalid revision " + f"{identity.revision!r}." + ) + return if actual != expected: raise ValueError( "HMRC replay base identity does not match the certified Microcosm UK " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py index 62c70f9d10..50482bfcee 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/ledger_targets.py @@ -179,24 +179,65 @@ def counterfactual_delta( f"{metric_name!r}" ) - def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: - entity = str(condition.get("entity") or "household") + def _household_ids_for(self, entity: str) -> Any: + """The household each row of ``entity`` belongs to, row-aligned.""" + source = self.tables[entity] if entity == "household": - household_ids = source["household_id"] + return source["household_id"] + if entity == "person": + # People sit directly in a household; only group entities need the + # membership lookup below, whose column would be the nonexistent + # "person_person_id" here. + return source["person_household_id"] + people = self.tables["person"] + entity_membership = f"person_{entity}_id" + if entity_membership not in people: + raise KeyError(entity_membership) + group_to_household = ( + people[[entity_membership, "person_household_id"]] + .drop_duplicates() + .set_index(entity_membership)["person_household_id"] + ) + if group_to_household.index.has_duplicates: + raise ValueError(f"UK {entity} groups span multiple households.") + return source[f"{entity}_id"].map(group_to_household) + + def entity_reduction(self, reduction: Mapping[str, Any]) -> np.ndarray: + """Reduce a member-level variable to a household-grain value. + + The numeric sibling of :meth:`household_condition`: where that answers + "does this household satisfy the predicate", this answers "what does + this household's members sum to". A count target declared over a + boolean member variable — the number of children, not whether the + household has any — is only honest through this path; collapsing the + boolean with ``any`` would publish an indicator against a count. + """ + + entity = str(reduction.get("entity") or "person") + source = self.tables[entity] + variable = str(reduction["variable"]) + if variable not in source: + raise KeyError(f"{entity}.{variable}") + household_ids = self._household_ids_for(entity) + reduce = str(reduction.get("reduce") or "sum") + values = source[variable] + if reduce == "sum": + aggregate = values.astype(float).groupby(household_ids).sum() + elif reduce == "count": + aggregate = values.groupby(household_ids).count() else: - people = self.tables["person"] - entity_membership = f"person_{entity}_id" - if entity_membership not in people: - raise KeyError(entity_membership) - group_to_household = ( - people[[entity_membership, "person_household_id"]] - .drop_duplicates() - .set_index(entity_membership)["person_household_id"] - ) - if group_to_household.index.has_duplicates: - raise ValueError(f"UK {entity} groups span multiple households.") - household_ids = source[f"{entity}_id"].map(group_to_household) + raise ValueError(f"Unsupported UK entity reduction {reduce!r}.") + households = self.tables["household"] + return np.asarray( + households["household_id"].map(aggregate).fillna(0.0), + dtype=float, + ) + + def household_condition(self, condition: Mapping[str, Any]) -> np.ndarray: + entity = str(condition.get("entity") or "household") + source = self.tables[entity] + household_ids = self._household_ids_for(entity) reduce = str(condition["reduce"]) variable = str(condition["variable"]) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py new file mode 100644 index 0000000000..b28beee1f6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -0,0 +1,250 @@ +"""UK simulated-measure inputs for the national calibration seam.""" + +from __future__ import annotations + +import json +from importlib import resources as importlib_resources +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry + +_ENTITY_LINK = {"benunit": "person_benunit_id", "household": "person_household_id"} +_ENTITY_ID = {"person": "person_id", "benunit": "benunit_id", "household": "household_id"} + + +def compute_uk_measure_input( + frame: Any, simulation: Any, entity: str, variable: str, year: int +) -> tuple[np.ndarray, str]: + """Compute one policyengine-uk variable at the requested entity grain.""" + + definition = simulation.tax_benefit_system.variables.get(variable) + if definition is None: + raise KeyError(f"policyengine-uk has no variable {variable!r}") + native = definition.entity.key + table = frame.table(entity) + raw = simulation.calculate(variable, year) + + if native == entity: + values = _values(raw) + route = "native" + else: + native_values = _values(raw) + if native_values.dtype.kind in {"O", "U", "S", "b"}: + if entity == "person" and native in ("benunit", "household"): + native_table = frame.table(native) + lookup = pd.Series( + native_values, index=native_table[_ENTITY_ID[native]].to_numpy() + ) + keys = frame.table("person")[_ENTITY_LINK[native]].to_numpy() + values = lookup.loc[keys].to_numpy() + route = f"categorical_broadcast_{native}_to_person" + elif native == "person" and native_values.dtype.kind == "b": + person = frame.table("person") + collapsed = ( + pd.Series(native_values.astype(bool)) + .groupby(person[_ENTITY_LINK[entity]].to_numpy()) + .max() + ) + keys = table[_ENTITY_ID[entity]].to_numpy() + values = collapsed.loc[keys].to_numpy().astype(float) + route = f"bool_any_collapse_person_to_{entity}" + else: + raise KeyError( + f"no categorical mapping from {native} to {entity} " + f"for {variable!r}" + ) + else: + mapped = simulation.calculate(variable, year, map_to=entity) + values = _values(mapped) + route = f"map_to_{entity}" + + if len(values) != len(table): + raise ValueError( + f"{variable!r} produced {len(values)} values for {len(table)} " + f"{entity} rows." + ) + if values.dtype.kind not in {"O", "U", "S"}: + values = values.astype(float) + return values, route + + +class UKMeasureResolver: + """B2 measure provider backed by a policyengine-uk Microsimulation.""" + + def __init__( + self, + *, + simulation_source: Path | None, + scratch_dir: Path, + year: int, + frame: Any, + microsimulation_factory: Any | None = None, + ): + self.year = int(year) + policyengine_uk = _policyengine_uk_module() + factory = ( + microsimulation_factory + if microsimulation_factory is not None + else policyengine_uk.Microsimulation + ) + if simulation_source is None: + if frame is None: + raise ValueError("scratch-mode UKMeasureResolver requires a frame.") + scratch_dir.mkdir(parents=True, exist_ok=True) + source_path = scratch_dir / "simulation-input.h5" + write_uk_national_frame(frame, source_path) + mode = "scratch_frame_export" + else: + source_path = Path(simulation_source) + mode = "direct_h5" + if frame is None: + frame, _provenance = load_uk_national_frame(source_path) + self.frame = frame + self.simulation = factory(dataset=str(source_path)) + self._receipt = { + "mode": mode, + "source_path": str(source_path), + "policyengine_uk_version": str( + getattr(policyengine_uk, "__version__", "unknown") + ), + } + self.contract_targets = _uk_contract_targets() + + def knows(self, entity: str, variable: str) -> bool: + """Whether a route in :func:`compute_uk_measure_input` reaches here. + + The resolution loop turns a False into a named refusal; anything this + answers True to must actually compute, or the operator gets a wrapped + provider exception instead of the fence's message. + """ + + definition = self.simulation.tax_benefit_system.variables.get(variable) + if definition is None or entity not in _ENTITY_ID: + return False + native = definition.entity.key + if native == entity: + return True + if "person" in (native, entity): + # Person-to-group and group-to-person are covered by the + # broadcast, any-collapse, and map_to routes alike. + return True + # Group to group has only the numeric map_to route: a categorical or + # boolean variable has no mapping between benunit and household and + # refuses here rather than deep inside the provider. + return getattr(definition, "value_type", None) in (int, float) + + def entity_for(self, variable: str) -> str | None: + definition = self.simulation.tax_benefit_system.variables.get(variable) + if definition is None: + return None + return str(definition.entity.key) + + def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: + return compute_uk_measure_input( + self.frame, self.simulation, entity, variable, self.year + ) + + def receipt(self) -> dict[str, str]: + return dict(self._receipt) + + +def load_uk_calibration_measure_exclusions( + path: Path | None = None, +) -> tuple[dict[str, str], ...]: + """Load and validate the reviewed UK calibration-measure exclusions.""" + + if path is None: + text = ( + importlib_resources.files("microcosm.build.uk") + .joinpath("calibration_measure_exclusions.json") + .read_text(encoding="utf-8") + ) + else: + text = Path(path).read_text(encoding="utf-8") + payload = json.loads(text) + allowed_top = {"schema_version", "exclusions"} + unknown = sorted(set(payload) - allowed_top) + if unknown: + raise ValueError(f"unknown top-level exclusion key(s): {unknown}") + if payload.get("schema_version") != 1: + raise ValueError("UK calibration measure exclusions schema_version must be 1.") + exclusions = payload.get("exclusions") + if not isinstance(exclusions, list): + raise ValueError("UK calibration measure exclusions must contain a list.") + seen: set[str] = set() + loaded: list[dict[str, str]] = [] + for entry in exclusions: + if not isinstance(entry, dict): + raise ValueError("UK calibration measure exclusion entries must be objects.") + name = str(entry.get("name", "")) + reason = str(entry.get("reason", "")) + tracking = str(entry.get("tracking", "")) + if name in seen: + raise ValueError(f"duplicate UK calibration measure exclusion {name!r}.") + if not reason.strip(): + raise ValueError(f"UK calibration measure exclusion {name!r} has empty reason.") + if not tracking.strip(): + raise ValueError( + f"UK calibration measure exclusion {name!r} has empty tracking; " + "a narrowed target surface must name where it is being resolved." + ) + seen.add(name) + loaded.append({"name": name, "reason": reason, "tracking": tracking}) + return tuple(loaded) + + +def apply_uk_calibration_measure_exclusions( + registry: TargetRegistry, exclusions: tuple[dict[str, str], ...] +) -> tuple[TargetRegistry, dict[str, dict[str, str]]]: + """Remove reviewed excluded references from a UK target registry. + + The receipt carries every field the register declares, tracking included: + an exclusion narrows the calibrated target surface, so the run evidence + must say where each narrowing is being resolved, not only why. + """ + + declared = {entry["name"]: entry for entry in exclusions} + matched = {spec.name for spec in registry.specs if spec.name in declared} + stale = sorted(set(declared) - matched) + if stale: + raise ValueError( + "UK calibration measure exclusion matched zero registry specs: " + + ", ".join(stale) + ) + kept = [spec for spec in registry.specs if spec.name not in declared] + receipt = { + name: { + "reason": declared[name]["reason"], + "tracking": declared[name]["tracking"], + } + for name in sorted(matched) + } + return TargetRegistry(kept, country=registry.country), receipt + + +def _values(raw: Any) -> np.ndarray: + return np.asarray(raw.values if hasattr(raw, "values") else raw) + + +def _policyengine_uk_module() -> Any: + import policyengine_uk + + return policyengine_uk + + +def _uk_contract_targets() -> dict[str, Any]: + payload = ( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + contract = json.loads(payload) + return {target["target_id"]: target for target in contract["targets"]} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py index 03744f7936..0c479d29cd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_build.py @@ -19,15 +19,13 @@ from __future__ import annotations import json -import uuid from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date from pathlib import Path +from types import SimpleNamespace from typing import Any -import pandas as pd - import microcosm.build.uk_runtime.national_frame as _national_frame from microcosm.build.country_spec import load_country_spec from microcosm.build.frame_sampling import ( @@ -47,16 +45,52 @@ from microcosm.build.plan import StagePlan, StageRecord from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY from microcosm.build.uk_runtime.national_frame import ( - UKStagingProvenance, + UK_HOUSEHOLD_WEIGHT_KIND_ATTR as UK_HOUSEHOLD_WEIGHT_KIND_ATTR, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_MASS_LOG_ATTR as UK_MASS_LOG_ATTR, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_H5_TABLES as UK_NATIONAL_H5_TABLES, +) +from microcosm.build.uk_runtime.national_frame import ( + UKStagingProvenance as UKStagingProvenance, +) +from microcosm.build.uk_runtime.national_frame import ( + _mass_log_from_stored as _mass_log_from_stored, +) +from microcosm.build.uk_runtime.national_frame import ( + _read_uk_national_tables as _read_uk_national_tables, +) +from microcosm.build.uk_runtime.national_frame import ( + _read_weight_metadata as _read_weight_metadata, +) +from microcosm.build.uk_runtime.national_frame import ( + _weight_kind_from_stored as _weight_kind_from_stored, +) +from microcosm.build.uk_runtime.national_frame import ( + _write_uk_single_year_tables as _write_uk_single_year_tables, +) +from microcosm.build.uk_runtime.national_frame import ( + _write_weight_metadata as _write_weight_metadata, +) +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame as load_uk_national_frame, +) +from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.national_frame import ( + write_uk_national_frame as write_uk_national_frame, +) from microcosm.build.uk_runtime.national_sampling import ( UK_SAMPLE_SEED_DEFAULT, sample_uk_national_frame, ) +from microcosm.build.uk_runtime.parity_reference import EfrsParityReference from microcosm.build.uk_runtime.release_input_coverage import ( PolicyEngineUKCoverageEngine, ) @@ -66,14 +100,7 @@ exclusion_evaluation_date, ) from microcosm.calibrate import TargetRegistry -from microcosm.frame import ( - Frame, - MassChangeRecord, - WeightKind, - engine_tables, - put_frame_table, - read_frame_table, -) +from microcosm.frame import Frame __all__ = [ "UKNationalBuildResult", @@ -88,11 +115,6 @@ "write_uk_national_frame", ] -UK_NATIONAL_H5_TABLES = ("person", "benunit", "household", "time_period") -UK_HOUSEHOLD_WEIGHT_KIND_ATTR = "populace_household_weight_kind" -UK_MASS_LOG_ATTR = "populace_mass_log_json" - - # The fingerprint pair moved to national_frame with the Frame carrier; the # re-import keeps this module's existing consumers (hmrc_restoration binds # certified candidates to it) on their current import path. @@ -173,169 +195,6 @@ def input_coverage_path(self) -> Path: return self.terminal_gate_path -def _read_uk_national_tables( - path: str | Path, -) -> tuple[dict[str, Any], _UKSourceFileFingerprint, Path]: - """Read a compact UK single-year H5's payload with the race guard. - - Shared body of both loaders: suffix check, symlink resolution, and the - fingerprint-before/after guard binding the returned tables to one stable - set of bytes. - """ - - requested_path = Path(path).expanduser() - if requested_path.suffix != ".h5": - raise ValueError("UK national dataset path must end with '.h5'.") - # Hugging Face cache entries retain the requested ``.h5`` name as a - # symlink whose content-addressed blob target has no suffix. Validate the - # caller-facing artifact name before resolving it, while binding all - # provenance and stable-byte checks to the actual opened file. - input_path = requested_path.resolve() - if not input_path.is_file(): - raise FileNotFoundError(f"UK national dataset not found: {input_path}.") - - fingerprint_before = _uk_source_file_fingerprint(input_path) - stored_kind, stored_mass_log = _read_weight_metadata(input_path) - with pd.HDFStore(input_path, mode="r") as store: - keys = {key.lstrip("/") for key in store.keys()} - missing = sorted(set(UK_NATIONAL_H5_TABLES) - keys) - if missing: - raise ValueError(f"UK national dataset is missing table(s): {missing}.") - raw_period = store["time_period"] - if len(raw_period) != 1: - raise ValueError( - "UK national dataset time_period must contain exactly one value." - ) - payload = { - "person": read_frame_table(store, "person"), - "benunit": read_frame_table(store, "benunit"), - "household": read_frame_table(store, "household"), - "time_period": str(raw_period.iloc[0]), - "household_weight_kind": _weight_kind_from_stored(stored_kind), - "mass_log": _mass_log_from_stored(stored_mass_log), - } - fingerprint_after = _uk_source_file_fingerprint(input_path) - if fingerprint_after != fingerprint_before: - raise RuntimeError( - "UK national source H5 changed while it was being loaded; refusing " - "to bind mixed or stale bytes to build stages." - ) - return payload, fingerprint_after, input_path - - -def load_uk_national_frame( - path: str | Path, -) -> tuple[Frame, UKStagingProvenance]: - """Load a compact UK single-year H5 as a validated Frame plus provenance. - - Frame construction is where the structural invariants are enforced - (linkage in both directions, sorted group ids, column uniqueness, weight - health); :func:`validate_uk_national_frame` adds the UK residue. The - provenance record travels beside the frame — it is the same source-path - and fingerprint identity the shadow carrier smuggled in private fields. - """ - - payload, fingerprint, input_path = _read_uk_national_tables(path) - frame = uk_national_frame( - person=payload["person"], - benunit=payload["benunit"], - household=payload["household"], - time_period=payload["time_period"], - weight_kind=payload["household_weight_kind"], - mass_log=payload["mass_log"], - ) - validate_uk_national_frame(frame) - return frame, UKStagingProvenance(source_h5=input_path, fingerprint=fingerprint) - - -def _write_uk_single_year_tables( - *, - person: pd.DataFrame, - benunit: pd.DataFrame, - household: pd.DataFrame, - time_period: str, - weight_kind: WeightKind, - mass_log: tuple[MassChangeRecord, ...], - path: Path, -) -> Path: - """The one physical writer for every UK single-year H5. - - Tables and the weight-kind/mass-log attrs must land together: writing - them into a temporary file and renaming keeps a metadata failure from - leaving a complete-looking attr-less H5 that would silently default to - DESIGN semantics on the next read. - """ - - path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp.h5") - try: - with pd.HDFStore(temporary_path) as store: - put_frame_table( - store, - "person", - person, - preferred_format="table", - data_columns=True, - ) - put_frame_table( - store, - "benunit", - benunit, - preferred_format="table", - data_columns=True, - ) - put_frame_table( - store, - "household", - household, - preferred_format="table", - data_columns=True, - ) - store.put( - "time_period", - pd.Series([time_period]), - format="table", - data_columns=True, - ) - _write_weight_metadata( - temporary_path, weight_kind=weight_kind, mass_log=mass_log - ) - temporary_path.replace(path) - finally: - temporary_path.unlink(missing_ok=True) - return path - - -def write_uk_national_frame(frame: Frame, path: str | Path) -> Path: - """Atomically write a validated UK national Frame as a staging H5. - - The engine-facing payload is materialized through the shared - :func:`microcosm.frame.engine_tables`, so the typed weights are - authoritative and the ``household_weight`` column is overwritten in - place (preserving its position, and therefore the artifact's column - order). - """ - - validate_uk_national_frame(frame) - output_path = Path(path) - if output_path.suffix != ".h5": - raise ValueError("UK national staging path must end with '.h5'.") - # The export contract is pinned, not inherited: household weights are the - # one materialized vector, so a frame that somehow reached this point - # with other typed weights cannot grow reserved columns the UK loader - # rejects (validate_uk_national_frame refuses such frames anyway). - tables = engine_tables(frame, weighted_entities=("household",)) - return _write_uk_single_year_tables( - person=tables["person"], - benunit=tables["benunit"], - household=tables["household"], - time_period=uk_time_period(frame), - weight_kind=uk_household_weight_kind(frame), - mass_log=frame.mass_log, - path=output_path, - ) - - def build_uk_national_dataset( *, input_h5: str | Path, @@ -360,6 +219,7 @@ def build_uk_national_dataset( now: date | None = None, gate_registry: Mapping[str, GateBinding] | None = None, ledger_target_registry: Mapping[int | str, TargetRegistry] | None = None, + parity_reference: EfrsParityReference | None = None, ) -> UKNationalBuildResult: """Run ordered national stages, hard-gate the result, and stage an H5. @@ -555,6 +415,13 @@ def build_uk_national_dataset( calibration_evidence = _stage_calibration_evidence(materialized_stages) if calibration_evidence is not None: artifacts["national_calibration"] = calibration_evidence + parity_evidence = _stage_parity_evidence( + materialized_stages, + frame=frame, + parity_reference=parity_reference, + ) + if parity_evidence is not None: + artifacts["parity_evidence"] = parity_evidence if input_mass_reference is not None: artifacts["input_mass_reference"] = input_mass_reference if reviewed_input_mass_exclusions is not None: @@ -828,7 +695,80 @@ def _stage_calibration_evidence( for stage in stages: if stage.name == "national_calibration": manifest = getattr(stage.transform, "manifest", None) - return dict(manifest) if isinstance(manifest, Mapping) else {} + if not isinstance(manifest, Mapping): + raise RuntimeError( + "national_calibration stage did not produce a manifest; " + "refusing to build calibration gate evidence." + ) + return dict(manifest) + return None + + +def _stage_parity_evidence( + stages: tuple[PlanStage, ...], + *, + frame: Frame, + parity_reference: EfrsParityReference | None, +) -> object | None: + """Build the parity-trio evidence, side by side, never aliased. + + Two different comparisons travel in one object, and they are not equally + strong: + + * The **column** surfaces are independently sourced — the candidate's from + the staged frame, the reference's from the frozen parity instrument's + declared input entities. + * The **target** surfaces are not. The parity instrument carries source + identity and incumbent input-column shares only; it holds no incumbent + target surface. So the reference side is the declared registry and the + candidate side is the solve's realized diagnostics — which proves that + the solve bound every declared target at the declared period, and does + **not** prove agreement with any incumbent-derived surface. + + Neither side is ever copied from the other; a copied reference would make + the trio pass by construction. Sourcing the target side from an + incumbent-bound instrument needs an instrument that carries one, which is + release-cut work (#757). + """ + + for stage in stages: + if stage.name != "national_calibration": + continue + if parity_reference is None: + return None + diagnostics = getattr(stage.transform, "diagnostics", None) + if not isinstance(diagnostics, tuple): + raise RuntimeError( + "national_calibration stage did not produce target diagnostics; " + "refusing to build parity evidence." + ) + registry = getattr(stage.transform, "registry", None) + specs = getattr(registry, "specs", None) + if specs is None: + raise RuntimeError( + "national_calibration stage carries no compiled registry; " + "refusing to build parity evidence." + ) + target_relative_errors = { + str(row["name"]): float(row["relative_error"]) for row in diagnostics + } + return SimpleNamespace( + candidate_columns={ + f"{entity}.{column}" + for entity in frame.entities + for column in frame.table(entity).columns + }, + reference_columns={ + f"{entity}.{name}" + for name, entity in parity_reference.input_entities.items() + }, + candidate_targets=set(target_relative_errors), + # Solver diagnostics label rows as ``name@period``; the declared + # side is compared at the same labeled grain so a target bound at + # the wrong period cannot satisfy the surface. + reference_targets={f"{spec.name}@{spec.period}" for spec in specs}, + target_relative_errors=target_relative_errors, + ) return None @@ -877,89 +817,3 @@ def _write_input_coverage_diagnostic(path: Path, gate: GateResult) -> None: encoding="utf-8", ) temporary_path.replace(path) - - -def _weight_kind_from_stored(value: object) -> WeightKind: - if isinstance(value, bytes): - value = value.decode("utf-8") - try: - return WeightKind(str(value)) - except ValueError as exc: - raise ValueError(f"Unknown stored UK household weight kind {value!r}.") from exc - - -def _read_weight_metadata(path: Path) -> tuple[object, object]: - try: - import h5py - except ImportError as exc: # pragma: no cover - UK H5 runtime dependency - raise RuntimeError("h5py is required to read UK national metadata.") from exc - with h5py.File(path, mode="r") as file: - return ( - file.attrs.get( - UK_HOUSEHOLD_WEIGHT_KIND_ATTR, - WeightKind.DESIGN.value, - ), - file.attrs.get(UK_MASS_LOG_ATTR, "[]"), - ) - - -def _write_weight_metadata( - path: Path, - *, - weight_kind: WeightKind, - mass_log: tuple[MassChangeRecord, ...], -) -> None: - try: - import h5py - except ImportError as exc: # pragma: no cover - UK H5 runtime dependency - raise RuntimeError("h5py is required to write UK national metadata.") from exc - with h5py.File(path, mode="r+") as file: - file.attrs[UK_HOUSEHOLD_WEIGHT_KIND_ATTR] = weight_kind.value - file.attrs[UK_MASS_LOG_ATTR] = json.dumps( - [_mass_change_record_payload(record) for record in mass_log], - sort_keys=True, - ) - - -def _mass_log_from_stored(value: object) -> tuple[MassChangeRecord, ...]: - if isinstance(value, bytes): - value = value.decode("utf-8") - try: - payload = json.loads(str(value)) - except json.JSONDecodeError as exc: - raise ValueError("Stored UK microcosm mass log is not valid JSON.") from exc - if not isinstance(payload, list): - raise ValueError("Stored UK microcosm mass log must be a JSON list.") - records: list[MassChangeRecord] = [] - for entry in payload: - if not isinstance(entry, dict): - raise ValueError("Stored UK microcosm mass-log entries must be objects.") - try: - records.append( - MassChangeRecord( - entity=str(entry["entity"]), - old_total=float(entry["old_total"]), - new_total=float(entry["new_total"]), - declared_factor=( - None - if entry.get("declared_factor") is None - else float(entry["declared_factor"]) - ), - reason=str(entry["reason"]), - ) - ) - except (KeyError, TypeError, ValueError) as exc: - raise ValueError( - "Stored UK microcosm mass-log entry is malformed." - ) from exc - return tuple(records) - - -def _mass_change_record_payload(record: MassChangeRecord) -> dict[str, object]: - return { - "entity": record.entity, - "old_total": record.old_total, - "new_total": record.new_total, - "declared_factor": record.declared_factor, - "reason": record.reason, - } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py index 6d24e65ec0..4ff938bb09 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_calibration.py @@ -5,17 +5,37 @@ from collections.abc import Mapping, Sequence from typing import Any -from microcosm.build.ledger_targets import compile_ledger_target_references +import numpy as np + from microcosm.build.plan import Stage +from microcosm.build.target_materialization import ( + MeasureResolution, + resolve_target_measures, +) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.ledger_targets import ( UKFrameTargetAdapter, + UKLedgerTargetCompilation, materialize_uk_ledger_targets, ) -from microcosm.calibrate import calibrate, effective_sample_size -from microcosm.frame import Frame +from microcosm.build.uk_runtime.national_doctrine import ( + UK_NATIONAL_SOLVE_DOCTRINE, + UKNationalSolveDoctrine, + uk_national_target_loss_weights, +) +from microcosm.calibrate import ( + CalibrationResult, + TargetRegistry, + calibrate, + effective_sample_size, +) +from microcosm.frame import Frame, WeightKind, Weights -__all__ = ["UKNationalCalibrationStage", "uk_national_calibration_stage"] +__all__ = [ + "UKNationalCalibrationStage", + "national_calibration_mass_reason", + "uk_national_calibration_stage", +] class UKNationalCalibrationStage: @@ -23,68 +43,91 @@ class UKNationalCalibrationStage: def __init__( self, - facts: Sequence[Mapping[str, Any]], + registry: UKLedgerTargetCompilation | TargetRegistry, *, - references: Sequence[object] | None = None, - epochs: int = 256, - learning_rate: float = 0.02, - max_weight_ratio: float = 10.0, - seed: int = 0, + period: int, + doctrine: UKNationalSolveDoctrine = UK_NATIONAL_SOLVE_DOCTRINE, + measure_resolver: object | None = None, ) -> None: - from microcosm.build.country_spec import load_country_spec - - self.facts = tuple(facts) - self.references = tuple( - load_country_spec("uk").target_references - if references is None - else references - ) - self.epochs = epochs - self.learning_rate = learning_rate - self.max_weight_ratio = max_weight_ratio - self.seed = seed + self.compilation = ( + registry + if isinstance(registry, UKLedgerTargetCompilation) + else UKLedgerTargetCompilation(registry=registry, unsupported=()) + ) + self.registry = self.compilation.registry + # The materialization period is the declared calibration year the + # registry was compiled at — never the input frame's base-year + # time_period, which lags it (survey 2024, calibration 2025). + if not isinstance(period, int) or isinstance(period, bool) or period <= 0: + raise ValueError( + f"period must be the declared calibration year, got {period!r}." + ) + self.period = period + self.doctrine = doctrine + self.measure_resolver = measure_resolver self.manifest: dict[str, object] | None = None self.diagnostics: tuple[dict[str, object], ...] = () + self.solve_result: CalibrationResult | None = None self.output_content_identity: str | None = None def __call__(self, frame: Frame) -> Frame: - registry = compile_ledger_target_references( - self.facts, self.references, country="uk" - ) - declared = len(self.references) - resolved = len(registry.specs) - if resolved != declared: + declared = len(self.registry.specs) + len(self.compilation.unsupported) + resolved = len(self.registry.specs) + if self.compilation.unsupported: raise RuntimeError( "UK national calibration resolved " - f"{resolved} of {declared} activated target references." + f"{resolved} of {declared} activated target references; " + f"unsupported={self.compilation.unsupported!r}." ) + adapter = _CalibrationFrameAdapter(frame) + original_columns = { + entity: set(table.columns) for entity, table in adapter.tables.items() + } + measure_resolution = self._resolve_measures(frame) + if measure_resolution is not None: + _inject_measure_inputs(adapter, measure_resolution.measure_inputs) materialized = materialize_uk_ledger_targets( - UKFrameTargetAdapter(frame), - registry, - period=_registry_materialization_period(registry.specs), + adapter, + self.registry, + period=self.period, ) if materialized.skipped: - skipped = [ - { - "name": item.name, - "measure": item.measure, - "reason": item.reason, - } - for item in materialized.skipped - ] + skipped = [skip.__dict__ for skip in materialized.skipped] raise RuntimeError( - "UK national calibration could not materialize every " - f"activated reference measure: {skipped}." + "UK national calibration could not materialize every activated " + f"target reference: skipped={skipped}." + ) + if measure_resolution is not None: + _drop_injected_measure_inputs( + adapter, + measure_resolution.measure_inputs, + original_columns, ) - prepared = materialized.adapter.to_frame() + prepared = adapter.prepared_frame() + mass_reason = national_calibration_mass_reason( + spec.family for spec in self.registry.specs + ) + mass_log_records_before_calibration = len(frame.mass_log) + # Target-set rows follow registry spec order, so the doctrine weight + # vector aligns positionally; under the default "uniform" rule this + # is None — the kernel's own equal weighting. + target_loss_weights = uk_national_target_loss_weights( + [spec.family for spec in self.registry.specs], + rule=self.doctrine.target_weight_rule, + ) result = calibrate( prepared, - registry.to_target_set(), + self.registry.to_target_set(), weight_entity="household", - epochs=self.epochs, - learning_rate=self.learning_rate, - max_weight_ratio=self.max_weight_ratio, - seed=self.seed, + epochs=self.doctrine.epochs, + learning_rate=self.doctrine.learning_rate, + mass=self.doctrine.mass_rule, + mass_reason=mass_reason, + max_weight_ratio=self.doctrine.max_weight_ratio, + seed=self.doctrine.seed, + l0_lambda=self.doctrine.l0_lambda, + target_loss_cap=self.doctrine.target_loss_cap, + target_loss_weights=target_loss_weights, ) if result.skipped or len(result.problem.names) != declared: skipped = [item.name for item in result.skipped] @@ -93,6 +136,13 @@ def __call__(self, frame: Frame) -> Frame: f"reference: declared={declared}, rows={len(result.problem.names)}, " f"skipped={skipped}." ) + clean_frame = adapter.restore(result.frame) + self.solve_result = result + calibration_record = _post_solve_calibration_record( + frame, + clean_frame, + before_count=mass_log_records_before_calibration, + ) self.diagnostics = tuple( { "name": row.name, @@ -103,17 +153,69 @@ def __call__(self, frame: Frame) -> Frame: for row in result.diagnostics ) ratios = result.weights / result.initial_weights - self.manifest = { + old_total = float(calibration_record.old_total) + new_total = float(calibration_record.new_total) + before_kind = frame.weights_for("household").kind + after_kind = clean_frame.weights_for("household").kind + manifest = { "activated_reference_count": declared, "resolved_reference_count": resolved, "matrix_target_count": len(result.problem.names), "loss": result.final_loss, "effective_sample_size": effective_sample_size(result.weights), "max_weight_ratio": float(ratios.max()), - "max_weight_ratio_bound": self.max_weight_ratio, + "max_weight_ratio_bound": self.doctrine.max_weight_ratio, + "target_materialization": materialized.report(), + "weights": { + "household_weight_kind": after_kind.value, + "household_weight_kind_chain": [ + {"stage": "staging", "kind": before_kind.value}, + {"stage": "national_calibration", "kind": after_kind.value}, + ], + "mass_log_records_before_calibration": ( + mass_log_records_before_calibration + ), + "mass_log_records": len(clean_frame.mass_log), + "calibration_mass_change": { + "entity": str(calibration_record.entity), + "old_total": old_total, + "new_total": new_total, + "relative_shift": (new_total - old_total) / old_total, + "declared_factor": calibration_record.declared_factor, + "reason": str(calibration_record.reason), + }, + }, + "solve": { + "n_targets": len(result.problem.names), + "n_households": len(clean_frame.table("household")), + "initial_loss": float(result.initial_loss), + "final_loss": float(result.final_loss), + "n_nonzero": int(np.count_nonzero(result.weights)), + }, + "parameters": {"doctrine": _doctrine_bounds(self.doctrine)}, } - self.output_content_identity = uk_frame_content_identity(result.frame) - return result.frame + if measure_resolution is not None: + manifest["measure_resolution"] = dict(measure_resolution.receipt) + self.manifest = manifest + self.output_content_identity = uk_frame_content_identity(clean_frame) + return clean_frame + + def _resolve_measures(self, frame: Frame) -> MeasureResolution | None: + if self.measure_resolver is None: + return None + resolve = getattr(self.measure_resolver, "resolve", None) + if callable(resolve): + return resolve( + lambda: _CalibrationFrameAdapter(frame), + self.registry, + period=self.period, + ) + return resolve_target_measures( + lambda: _CalibrationFrameAdapter(frame), + self.registry, + self.measure_resolver, + period=self.period, + ) def checkpoint_metadata(self) -> Mapping[str, object]: if self.manifest is None: @@ -165,21 +267,169 @@ def resume_from_checkpoint( def uk_national_calibration_stage( - facts: Sequence[Mapping[str, Any]], **kwargs: Any + registry: UKLedgerTargetCompilation, **kwargs: Any ) -> Stage: """Return the named ordered-stage entry for national calibration.""" - transform = UKNationalCalibrationStage(facts, **kwargs) + transform = UKNationalCalibrationStage(registry, **kwargs) return Stage(name="national_calibration", transform=transform) -def _registry_materialization_period(specs: Sequence[object]) -> int | str: - periods = {spec.period for spec in specs} - if not periods: - raise RuntimeError("UK national calibration has no target specs to materialize.") - if len(periods) != 1: +def national_calibration_mass_reason(bound_families: Sequence[str]) -> str: + """The mass-record reason a national doctrine calibration declares.""" + + families = sorted({str(name) for name in bound_families}) + if not families or any(not name.strip() for name in families): + raise ValueError("bound_families must name at least one target family.") + return ( + "National doctrine calibration to bound target family(ies) " + f"{', '.join(families)}; total household mass moved with the targets." + ) + + +def _post_solve_calibration_record( + before: Frame, + after: Frame, + *, + before_count: int, +): + if after.weights_for("household").kind is not WeightKind.CALIBRATED: + raise RuntimeError( + "UK national calibration returned household weights whose kind is " + f"{after.weights_for('household').kind.value!r}, not 'calibrated'." + ) + if len(after.mass_log) != before_count + 1: + raise RuntimeError( + "UK national calibration must append exactly one mass record; " + f"before={before_count}, after={len(after.mass_log)}." + ) + if before.mass_log != after.mass_log[:before_count]: + raise RuntimeError( + "UK national calibration changed pre-existing mass-log records." + ) + record = after.mass_log[-1] + if record.entity != "household" or "calibration" not in record.reason: + raise RuntimeError( + "UK national calibration latest mass record is not the calibration " + f"record: entity={record.entity!r}, reason={record.reason!r}." + ) + return record + + +def _doctrine_bounds(doctrine: UKNationalSolveDoctrine) -> dict[str, object]: + return { + "epochs": doctrine.epochs, + "learning_rate": doctrine.learning_rate, + "max_weight_ratio": doctrine.max_weight_ratio, + "seed": doctrine.seed, + "target_loss_cap": doctrine.target_loss_cap, + "scale_rule": doctrine.scale_rule, + "target_weight_rule": doctrine.target_weight_rule, + "mass_rule": doctrine.mass_rule, + "l0_lambda": doctrine.l0_lambda, + } + + +def _inject_measure_inputs( + adapter: UKFrameTargetAdapter, + measure_inputs: Mapping[tuple[str, str], np.ndarray], +) -> None: + for (entity, variable), values in measure_inputs.items(): + adapter.tables[entity][variable] = values + + +def _drop_injected_measure_inputs( + adapter: UKFrameTargetAdapter, + measure_inputs: Mapping[tuple[str, str], np.ndarray], + original_columns: Mapping[str, set[str]], +) -> None: + for entity, variable in measure_inputs: + if variable not in original_columns[entity]: + adapter.tables[entity].drop(columns=[variable], inplace=True) + + +class _CalibrationFrameAdapter(UKFrameTargetAdapter): + """The shared UK adapter plus the prepared-frame/restore lifecycle. + + Prepared measure columns are scratch state: they exist for constraint + compilation only, and ``restore`` rebuilds pristine entity tables around + the calibrated weights so the staged frame survives the HDFStore writer + (slash-named scratch columns crash it). + """ + + def __init__(self, frame: Frame) -> None: + super().__init__(frame) + self._source_frame = frame + self._original_tables = { + name: table.copy() for name, table in self.tables.items() + } + + def prepared_frame(self) -> Frame: + return Frame( + {**self.tables, **self.link_tables}, + self._source_frame.schema, + { + entity: self._source_frame.weights_for(entity) + for entity in self._source_frame.weighted_entities + }, + self._source_frame.strata, + mass_log=self._source_frame.mass_log, + metadata=self._source_frame.metadata, + ) + + def restore(self, calibrated: Frame) -> Frame: + tables = {name: table.copy() for name, table in self._original_tables.items()} + tables.update({name: table.copy() for name, table in self.link_tables.items()}) + return Frame( + tables, + calibrated.schema, + { + entity: Weights( + calibrated.weights_for(entity).values, + calibrated.weights_for(entity).kind, + ) + for entity in calibrated.weighted_entities + }, + calibrated.strata, + mass_log=calibrated.mass_log, + metadata=calibrated.metadata, + ) + + +def prepare_uk_target_frame( + frame: Frame, + registry: TargetRegistry, + *, + period: int | str, + measure_resolver: object | None, +) -> tuple[Frame, Mapping[str, Any] | None]: + """Materialize a registry's measures onto a frame, for scoring. + + The same resolve-inject-materialize route the calibration stage takes, + without the solve: scoring a UK register against a raw exported H5 cannot + work, because every packaged reference binds a slash-named prepared + measure that calibration deliberately strips before export. A skipped + target refuses rather than quietly shrinking the surface both sides are + compared on. + """ + + resolution = None + if measure_resolver is not None: + resolution = resolve_target_measures( + lambda: _CalibrationFrameAdapter(frame), + registry, + measure_resolver, + period=period, + ) + adapter = _CalibrationFrameAdapter(frame) + if resolution is not None: + _inject_measure_inputs(adapter, resolution.measure_inputs) + materialized = materialize_uk_ledger_targets(adapter, registry, period=period) + if materialized.skipped: raise RuntimeError( - "UK national calibration cannot materialize mixed-period target " - f"specs: {sorted(str(period) for period in periods)}." + "target measures did not materialize for scoring: " + f"{[skip.__dict__ for skip in materialized.skipped]}." ) - return next(iter(periods)) + return adapter.prepared_frame(), ( + None if resolution is None else dict(resolution.receipt) + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py new file mode 100644 index 0000000000..a17ac1928e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_doctrine.py @@ -0,0 +1,222 @@ +"""UK national calibration doctrine: reviewed solve constants for #623. + +The national release path exposes calibration knobs only as receipted per-run +overrides on the calibration seam driver; defaults remain the reviewed +constants. Every solver option that the first Ledger-backed national +calibration currently inherits is declared here so a future default change +must edit this module and its pinned tests, forcing review before any armed +run. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass, fields, replace + +import numpy as np + +__all__ = [ + "UK_NATIONAL_LEARNING_RATE", + "UK_NATIONAL_L0_LAMBDA", + "UK_NATIONAL_MASS_RULE", + "UK_NATIONAL_MAX_WEIGHT_RATIO", + "UK_NATIONAL_SEED", + "UK_NATIONAL_SOLVE_DOCTRINE", + "UK_NATIONAL_SOLVE_EPOCHS", + "UK_NATIONAL_TARGET_LOSS_CAP", + "UK_NATIONAL_TARGET_WEIGHT_RULE", + "uk_doctrine_with_overrides", + "uk_national_target_loss_weights", + "UKNationalSolveDoctrine", +] + +UK_NATIONAL_SOLVE_EPOCHS = 256 +UK_NATIONAL_LEARNING_RATE = 0.02 +UK_NATIONAL_MAX_WEIGHT_RATIO = 10.0 +UK_NATIONAL_SEED = 0 +UK_NATIONAL_TARGET_LOSS_CAP = 10.0 +UK_NATIONAL_L0_LAMBDA = 0.0 +UK_NATIONAL_MASS_RULE = "free" +UK_NATIONAL_TARGET_WEIGHT_RULE = "uniform" + +_ALLOWED_SCALE_RULES = ("default_target_loss_scales",) + +# "family_equal" gives every declared target family one equal share of the +# objective, split evenly within the family — the reviewed rule name the +# local doctrine reserved for exactly this ("a future family-level weighting +# would be a new reviewed rule name here — never a per-target vector"). +# +# Vocabulary admitted on 2026-08-23 from the first armed-run receipts: under +# "uniform" the hmrc SPI income-band family is 57% of the target surface and +# supplied 101 of the 102 references that start past the loss cap; pursuing +# them annihilated 24 population and OBR targets that were never past cap +# themselves (objective gain from annihilating a segment: +4.48 uniform vs +# +0.68 family_equal at cap 10, negative at cap 2). María's ruling +# (2026-08-24): the default stays "uniform" — family_equal is reachable only +# as an explicit, receipted per-run override while the weighting doctrine is +# still being measured (its own run-9 receipts show a 9-member family +# carrying 5.5x the per-reference weight of a 50-member one). +_ALLOWED_TARGET_WEIGHT_RULES = ("uniform", "family_equal") +_ALLOWED_MASS_RULES = ("free",) +_OVERRIDABLE_FIELDS = frozenset( + {"epochs", "learning_rate", "target_weight_rule", "target_loss_cap"} +) + + +@dataclass(frozen=True) +class UKNationalSolveDoctrine: + """The declared constants of the UK national calibration solve.""" + + epochs: int = UK_NATIONAL_SOLVE_EPOCHS + learning_rate: float = UK_NATIONAL_LEARNING_RATE + max_weight_ratio: float | None = UK_NATIONAL_MAX_WEIGHT_RATIO + seed: int = UK_NATIONAL_SEED + target_loss_cap: float = UK_NATIONAL_TARGET_LOSS_CAP + scale_rule: str = "default_target_loss_scales" + target_weight_rule: str = UK_NATIONAL_TARGET_WEIGHT_RULE + mass_rule: str = UK_NATIONAL_MASS_RULE + l0_lambda: float = UK_NATIONAL_L0_LAMBDA + + def __post_init__(self) -> None: + if ( + not isinstance(self.epochs, int) + or isinstance(self.epochs, bool) + or self.epochs <= 0 + ): + raise ValueError( + f"doctrine epochs must be a positive integer, got {self.epochs!r}." + ) + if ( + not isinstance(self.learning_rate, int | float) + or isinstance(self.learning_rate, bool) + or not np.isfinite(self.learning_rate) + or self.learning_rate <= 0 + ): + raise ValueError( + "doctrine learning_rate must be a positive finite number, " + f"got {self.learning_rate!r}." + ) + if self.max_weight_ratio is not None and ( + not isinstance(self.max_weight_ratio, int | float) + or isinstance(self.max_weight_ratio, bool) + or not np.isfinite(self.max_weight_ratio) + or self.max_weight_ratio <= 1 + ): + raise ValueError( + "doctrine max_weight_ratio must be None or a finite number " + f"greater than 1, got {self.max_weight_ratio!r}." + ) + if ( + not isinstance(self.seed, int) + or isinstance(self.seed, bool) + or self.seed < 0 + ): + raise ValueError( + f"doctrine seed must be a non-negative integer, got {self.seed!r}." + ) + if ( + not isinstance(self.target_loss_cap, int | float) + or isinstance(self.target_loss_cap, bool) + or not np.isfinite(self.target_loss_cap) + or self.target_loss_cap <= 0 + ): + raise ValueError( + "doctrine target_loss_cap must be a positive finite number, " + f"got {self.target_loss_cap!r}." + ) + if self.scale_rule not in _ALLOWED_SCALE_RULES: + raise ValueError( + f"doctrine scale_rule must be one of {_ALLOWED_SCALE_RULES}, " + f"got {self.scale_rule!r}." + ) + if self.target_weight_rule not in _ALLOWED_TARGET_WEIGHT_RULES: + raise ValueError( + "doctrine target_weight_rule must be one of " + f"{_ALLOWED_TARGET_WEIGHT_RULES}, got " + f"{self.target_weight_rule!r}." + ) + if self.mass_rule not in _ALLOWED_MASS_RULES: + raise ValueError( + f"doctrine mass_rule must be one of {_ALLOWED_MASS_RULES}, " + f"got {self.mass_rule!r}." + ) + if ( + not isinstance(self.l0_lambda, int | float) + or isinstance(self.l0_lambda, bool) + or not np.isfinite(self.l0_lambda) + or self.l0_lambda < 0 + ): + raise ValueError( + "doctrine l0_lambda must be a non-negative finite number, " + f"got {self.l0_lambda!r}." + ) + + +UK_NATIONAL_SOLVE_DOCTRINE = UKNationalSolveDoctrine() + + +def uk_doctrine_with_overrides( + **overrides: object, +) -> tuple[UKNationalSolveDoctrine, dict[str, dict[str, object]]]: + """Return the national doctrine plus a receipt of effective overrides.""" + + fields_by_name = {field.name for field in fields(UKNationalSolveDoctrine)} + unknown = sorted(set(overrides) - fields_by_name) + if unknown: + raise ValueError( + "unknown UK national doctrine field(s) " + f"{unknown}; frozen fields are {sorted(fields_by_name)}." + ) + frozen = sorted(set(overrides) - _OVERRIDABLE_FIELDS) + if frozen: + raise ValueError( + "UK national doctrine field(s) are reviewed constants, not knobs: " + f"{frozen}; overridable fields are {sorted(_OVERRIDABLE_FIELDS)}." + ) + doctrine = replace(UK_NATIONAL_SOLVE_DOCTRINE, **overrides) + receipt: dict[str, dict[str, object]] = {} + for name in sorted(_OVERRIDABLE_FIELDS): + default = getattr(UK_NATIONAL_SOLVE_DOCTRINE, name) + effective = getattr(doctrine, name) + if effective != default: + receipt[name] = {"default": default, "effective": effective} + return doctrine, receipt + + +def uk_national_target_loss_weights( + families: Sequence[str], + *, + rule: str = UK_NATIONAL_TARGET_WEIGHT_RULE, +) -> np.ndarray | None: + """The declared target-weight vector for one compiled register. + + ``uniform`` returns ``None`` — the kernel's own equal weighting, and the + identity this doctrine shipped with. ``family_equal`` gives each declared + family one equal share of the objective, divided evenly among its + members, so an over-supplied family cannot outvote the rest of the + surface by sheer count. + + The vector is derived from the compiled register's declared families and + nothing else: it is a rule, not a per-target vector, which is the + property the local doctrine requires of any weighting. + """ + + if rule not in _ALLOWED_TARGET_WEIGHT_RULES: + raise ValueError( + f"target_weight_rule must be one of {_ALLOWED_TARGET_WEIGHT_RULES}, " + f"got {rule!r}." + ) + if rule == "uniform": + return None + labels = [str(family) for family in families] + if not labels: + raise ValueError("family_equal weighting requires a non-empty register.") + if any(not label for label in labels): + raise ValueError("every target must declare a family for family_equal.") + counts = Counter(labels) + family_count = len(counts) + return np.asarray( + [1.0 / (family_count * counts[label]) for label in labels], + dtype=np.float64, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 2690adfb93..3d3a0640b3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -24,9 +24,12 @@ from __future__ import annotations +import json +import uuid from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -38,23 +41,34 @@ WeightKind, Weights, engine_tables, + put_frame_table, + read_frame_table, ) __all__ = [ "UK_EXPORTED_WEIGHT_COLUMNS", + "UK_HOUSEHOLD_WEIGHT_KIND_ATTR", + "UK_MASS_LOG_ATTR", + "UK_NATIONAL_H5_TABLES", "UK_NATIONAL_SCHEMA", "UK_TIME_PERIOD_METADATA_KEY", "UKStagingProvenance", + "load_uk_national_frame", "uk_household_weight_kind", "uk_national_frame", "uk_time_period", "validate_uk_national_frame", + "write_uk_national_frame", ] UK_NATIONAL_SCHEMA = EntitySchema(group_entities=("benunit", "household")) UK_TIME_PERIOD_METADATA_KEY = "time_period" +UK_NATIONAL_H5_TABLES = ("person", "benunit", "household", "time_period") +UK_HOUSEHOLD_WEIGHT_KIND_ATTR = "populace_household_weight_kind" +UK_MASS_LOG_ATTR = "populace_mass_log_json" + # The persisted weight columns of the materialized export contract above: # real columns on the tables, but plumbing rather than input mass. Consumers # that total column mass (the input-mass parity surface, its reference @@ -115,6 +129,92 @@ def _uk_source_file_fingerprint(path: Path) -> _UKSourceFileFingerprint: ) +def _weight_kind_from_stored(value: object) -> WeightKind: + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return WeightKind(str(value)) + except ValueError as exc: + raise ValueError(f"Unknown stored UK household weight kind {value!r}.") from exc + + +def _read_weight_metadata(path: Path) -> tuple[object, object]: + try: + import h5py + except ImportError as exc: # pragma: no cover - UK H5 runtime dependency + raise RuntimeError("h5py is required to read UK national metadata.") from exc + with h5py.File(path, mode="r") as file: + return ( + file.attrs.get( + UK_HOUSEHOLD_WEIGHT_KIND_ATTR, + WeightKind.DESIGN.value, + ), + file.attrs.get(UK_MASS_LOG_ATTR, "[]"), + ) + + +def _write_weight_metadata( + path: Path, + *, + weight_kind: WeightKind, + mass_log: tuple[MassChangeRecord, ...], +) -> None: + try: + import h5py + except ImportError as exc: # pragma: no cover - UK H5 runtime dependency + raise RuntimeError("h5py is required to write UK national metadata.") from exc + with h5py.File(path, mode="r+") as file: + file.attrs[UK_HOUSEHOLD_WEIGHT_KIND_ATTR] = weight_kind.value + file.attrs[UK_MASS_LOG_ATTR] = json.dumps( + [_mass_change_record_payload(record) for record in mass_log], + sort_keys=True, + ) + + +def _mass_log_from_stored(value: object) -> tuple[MassChangeRecord, ...]: + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + payload = json.loads(str(value)) + except json.JSONDecodeError as exc: + raise ValueError("Stored UK microcosm mass log is not valid JSON.") from exc + if not isinstance(payload, list): + raise ValueError("Stored UK microcosm mass log must be a JSON list.") + records: list[MassChangeRecord] = [] + for entry in payload: + if not isinstance(entry, dict): + raise ValueError("Stored UK microcosm mass-log entries must be objects.") + try: + records.append( + MassChangeRecord( + entity=str(entry["entity"]), + old_total=float(entry["old_total"]), + new_total=float(entry["new_total"]), + declared_factor=( + None + if entry.get("declared_factor") is None + else float(entry["declared_factor"]) + ), + reason=str(entry["reason"]), + ) + ) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError( + "Stored UK microcosm mass-log entry is malformed." + ) from exc + return tuple(records) + + +def _mass_change_record_payload(record: MassChangeRecord) -> dict[str, object]: + return { + "entity": record.entity, + "old_total": record.old_total, + "new_total": record.new_total, + "declared_factor": record.declared_factor, + "reason": record.reason, + } + + @dataclass(frozen=True) class UKStagingProvenance: """Where a UK national frame's bytes came from, carried beside it. @@ -127,6 +227,65 @@ class UKStagingProvenance: fingerprint: _UKSourceFileFingerprint +def _read_uk_national_tables( + path: str | Path, +) -> tuple[dict[str, Any], _UKSourceFileFingerprint, Path]: + """Read a compact UK single-year H5's payload with the race guard.""" + + requested_path = Path(path).expanduser() + if requested_path.suffix != ".h5": + raise ValueError("UK national dataset path must end with '.h5'.") + input_path = requested_path.resolve() + if not input_path.is_file(): + raise FileNotFoundError(f"UK national dataset not found: {input_path}.") + + fingerprint_before = _uk_source_file_fingerprint(input_path) + stored_kind, stored_mass_log = _read_weight_metadata(input_path) + with pd.HDFStore(input_path, mode="r") as store: + keys = {key.lstrip("/") for key in store.keys()} + missing = sorted(set(UK_NATIONAL_H5_TABLES) - keys) + if missing: + raise ValueError(f"UK national dataset is missing table(s): {missing}.") + raw_period = store["time_period"] + if len(raw_period) != 1: + raise ValueError( + "UK national dataset time_period must contain exactly one value." + ) + payload = { + "person": read_frame_table(store, "person"), + "benunit": read_frame_table(store, "benunit"), + "household": read_frame_table(store, "household"), + "time_period": str(raw_period.iloc[0]), + "household_weight_kind": _weight_kind_from_stored(stored_kind), + "mass_log": _mass_log_from_stored(stored_mass_log), + } + fingerprint_after = _uk_source_file_fingerprint(input_path) + if fingerprint_after != fingerprint_before: + raise RuntimeError( + "UK national source H5 changed while it was being loaded; refusing " + "to bind mixed or stale bytes to build stages." + ) + return payload, fingerprint_after, input_path + + +def load_uk_national_frame( + path: str | Path, +) -> tuple[Frame, UKStagingProvenance]: + """Load a compact UK single-year H5 as a validated Frame plus provenance.""" + + payload, fingerprint, input_path = _read_uk_national_tables(path) + frame = uk_national_frame( + person=payload["person"], + benunit=payload["benunit"], + household=payload["household"], + time_period=payload["time_period"], + weight_kind=payload["household_weight_kind"], + mass_log=payload["mass_log"], + ) + validate_uk_national_frame(frame) + return frame, UKStagingProvenance(source_h5=input_path, fingerprint=fingerprint) + + def uk_national_frame( *, person: pd.DataFrame, @@ -194,6 +353,77 @@ def uk_household_weight_kind(frame: Frame) -> WeightKind: return frame.weights_for("household").kind +def _write_uk_single_year_tables( + *, + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + time_period: str, + weight_kind: WeightKind, + mass_log: tuple[MassChangeRecord, ...], + path: Path, +) -> Path: + """The one physical writer for every UK single-year H5.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp.h5") + try: + with pd.HDFStore(temporary_path) as store: + put_frame_table( + store, + "person", + person, + preferred_format="table", + data_columns=True, + ) + put_frame_table( + store, + "benunit", + benunit, + preferred_format="table", + data_columns=True, + ) + put_frame_table( + store, + "household", + household, + preferred_format="table", + data_columns=True, + ) + store.put( + "time_period", + pd.Series([time_period]), + format="table", + data_columns=True, + ) + _write_weight_metadata( + temporary_path, weight_kind=weight_kind, mass_log=mass_log + ) + temporary_path.replace(path) + finally: + temporary_path.unlink(missing_ok=True) + return path + + +def write_uk_national_frame(frame: Frame, path: str | Path) -> Path: + """Atomically write a validated UK national Frame as a staging H5.""" + + validate_uk_national_frame(frame) + output_path = Path(path) + if output_path.suffix != ".h5": + raise ValueError("UK national staging path must end with '.h5'.") + tables = engine_tables(frame, weighted_entities=("household",)) + return _write_uk_single_year_tables( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + mass_log=frame.mass_log, + path=output_path, + ) + + @dataclass(frozen=True) class _UKGateSurface: """The duck-attr evidence surface the legacy UK gate modules read. diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index fa82cf720b..ed0f12d19c 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -288,6 +288,7 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "calibration_measure_exclusions.json", "hmrc_cgt_size_bands.json", "advani_summers_capital_gains_distribution.json", "salary_sacrifice_anchor.json", @@ -363,6 +364,7 @@ def test_uk_package_loads(self) -> None: "frs_release.json", "gates.json", "brma_rent_counts.json", + "calibration_measure_exclusions.json", "hmrc_cgt_size_bands.json", "advani_summers_capital_gains_distribution.json", "salary_sacrifice_anchor.json", diff --git a/packages/microcosm-build/tests/test_frame_serializer_registry.py b/packages/microcosm-build/tests/test_frame_serializer_registry.py index 04f5df7da4..11edf2aeb7 100644 --- a/packages/microcosm-build/tests/test_frame_serializer_registry.py +++ b/packages/microcosm-build/tests/test_frame_serializer_registry.py @@ -20,7 +20,7 @@ HDF_WRITE_EXCLUSIONS, FrameSerializerSpec, ) -from microcosm.build.uk_runtime.national_build import ( +from microcosm.build.uk_runtime.national_frame import ( _read_uk_national_tables, _write_uk_single_year_tables, ) diff --git a/packages/microcosm-build/tests/test_score_uk_national_candidate.py b/packages/microcosm-build/tests/test_score_uk_national_candidate.py new file mode 100644 index 0000000000..657667506e --- /dev/null +++ b/packages/microcosm-build/tests/test_score_uk_national_candidate.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import json + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.national_build import write_uk_national_frame +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind +from tools.score_uk_national_candidate import ( + _load_registry, + _sha256_file, + main, + score_uk_national_candidate, +) + + +def _registry() -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="target_a", + entity="household", + measure="measure_a", + value=10.0, + source="synthetic", + family="family_a", + ), + TargetSpec( + name="target_b", + entity="household", + measure="measure_b", + value=20.0, + source="synthetic", + family="family_b", + ), + ], + country="uk", + ) + + +def _write(path, *, measure_a, measure_b) -> None: + n = 2 + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_household_id": np.arange(n), + "person_benunit_id": np.arange(n), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = pd.DataFrame( + { + "household_id": np.arange(n), + "measure_a": measure_a, + "measure_b": measure_b, + } + ) + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2025", + weight_kind=WeightKind.CALIBRATED, + household_weights=np.ones(n), + ) + write_uk_national_frame(frame, path) + + +def test_score_uk_national_candidate_scores_synthetic_twins(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + + score = score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + ) + + assert score["artifacts"]["candidate"]["sha256"] == _sha256_file(candidate) + assert score["artifacts"]["incumbent"]["sha256"] == _sha256_file(incumbent) + assert score["target_drift"] == [ + { + "target": "target_a@0", + "family": "family_a", + "candidate_relative_error": pytest.approx(0.0), + "incumbent_relative_error": pytest.approx(-0.2), + "winner": "candidate", + }, + { + "target": "target_b@0", + "family": "family_b", + "candidate_relative_error": pytest.approx(-0.5), + "incumbent_relative_error": pytest.approx(0.0), + "winner": "incumbent", + }, + ] + assert score["candidate_full_loss"] == pytest.approx(0.25) + assert score["incumbent_full_loss"] == pytest.approx(0.1) + assert score["candidate_train_loss"] == score["candidate_full_loss"] + # With no declared split, the holdout keys report absence rather than + # repeating the fitted loss under a name that means generalization. + assert score["candidate_holdout_loss"] is None + assert score["incumbent_holdout_loss"] is None + assert score["holdout_basis"] == "none_declared" + assert score["loss"]["train_equals_full"] is True + assert score["candidate_target_wins"] == 1 + assert score["incumbent_target_wins"] == 1 + assert score["target_wins_by_family"] == { + "family_a": { + "candidate_target_wins": 1, + "incumbent_target_wins": 0, + "ties": 0, + }, + "family_b": { + "candidate_target_wins": 0, + "incumbent_target_wins": 1, + "ties": 0, + }, + } + + +def test_score_uk_national_candidate_cli_writes_score_block(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + registry_json = tmp_path / "registry.json" + output_json = tmp_path / "score.json" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + _registry().to_json(registry_json) + + assert ( + main( + [ + "--candidate-h5", + str(candidate), + "--candidate-sha256", + _sha256_file(candidate), + "--incumbent-h5", + str(incumbent), + "--incumbent-sha256", + _sha256_file(incumbent), + "--registry-json", + str(registry_json), + "--output-json", + str(output_json), + "--calibration-year", + "2025", + "--no-measure-resolution", + ] + ) + == 0 + ) + + payload = json.loads(output_json.read_text(encoding="utf-8")) + score = payload["score_vs_enhanced_frs"] + assert score["holdout_basis"] == "none_declared" + # An undeclared holdout reports absence, never the fitted loss wearing a + # holdout name: June's fixture holds a genuinely different holdout value, + # so a copied one would read as generalization that was never measured. + assert score["candidate_holdout_loss"] is None + assert score["incumbent_holdout_loss"] is None + assert isinstance(score["candidate_train_loss"], float) + assert isinstance(score["incumbent_full_loss"], float) + assert _load_registry(registry_json).version == _registry().version + + +def test_scorer_refuses_artifacts_that_do_not_match_their_pins(tmp_path) -> None: + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + + with pytest.raises(ValueError, match="candidate artifact sha mismatch"): + score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256="0" * 64, + incumbent_sha256=_sha256_file(incumbent), + target_registry=_registry(), + calibration_year=2025, + ) + + +def test_scorer_refuses_a_register_that_is_not_a_validated_artifact(tmp_path) -> None: + # The loose loader accepted any object with a specs list, so a + # hand-edited surface could decide rule 1 unnoticed. + loose = tmp_path / "loose.json" + loose.write_text( + json.dumps( + { + "country": "uk", + "specs": [spec.__dict__ for spec in _registry().specs], + } + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="not a microcosm target registry artifact"): + _load_registry(loose) + + tampered = tmp_path / "tampered.json" + _registry().to_json(tampered) + payload = json.loads(tampered.read_text(encoding="utf-8")) + payload["specs"][0]["value"] = 999.0 + tampered.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="corrupt"): + _load_registry(tampered) + + +def test_scorer_refuses_prepared_measures_it_cannot_materialize(tmp_path) -> None: + """A production-shaped register cannot be scored off raw exported columns. + + Every packaged UK reference binds a slash-named prepared measure, and + calibration strips those before export — so scoring without measure + resolution has to refuse rather than report on whatever happens to bind. + """ + + pytest.importorskip("tables") + candidate = tmp_path / "candidate.h5" + incumbent = tmp_path / "incumbent.h5" + _write(candidate, measure_a=[5.0, 5.0], measure_b=[5.0, 5.0]) + _write(incumbent, measure_a=[4.0, 4.0], measure_b=[10.0, 10.0]) + production_shaped = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=20.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + with pytest.raises(RuntimeError, match="did not materialize for scoring"): + score_uk_national_candidate( + candidate_h5=candidate, + incumbent_h5=incumbent, + candidate_sha256=_sha256_file(candidate), + incumbent_sha256=_sha256_file(incumbent), + target_registry=production_shaped, + calibration_year=2025, + ) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5ede29d29b..cc672a1599 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -42,7 +42,7 @@ ), ( "uk", - "e3b171bdaae1bdfe842e61c129fd279a6b29001cdef4c18de51823768911ec8c", + "0c85845b4d463638ae3e5c5a25e17de8b720794e3653c5991dc4f069d95762d3", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_target_materialization.py b/packages/microcosm-build/tests/test_target_materialization.py index 7bcf2bd211..b4af50fb48 100644 --- a/packages/microcosm-build/tests/test_target_materialization.py +++ b/packages/microcosm-build/tests/test_target_materialization.py @@ -1,6 +1,13 @@ import numpy as np +import pandas as pd +import pytest -from microcosm.build.target_materialization import materialize_target_bindings +from microcosm.build.target_materialization import ( + MeasureResolutionError, + assert_calibration_input_finite, + materialize_target_bindings, + resolve_target_measures, +) from microcosm.calibrate import TargetRegistry, TargetSpec @@ -20,6 +27,11 @@ def __init__(self): } def column(self, entity, variable): + # Counts are adapter concerns (the UK adapter's convention): a + # *_count variable is the all-ones indicator over the entity. + if variable in {"person_count", "household_count", f"{entity}_count"}: + first = next(iter(self.tables[entity].values())) + return np.ones(len(first), dtype=float) return self.tables[entity][variable] def set_column(self, entity, variable, values): @@ -36,6 +48,83 @@ def counterfactual_delta(self, binding, period): return np.array([0.0, -1.0, -2.0]) +class ResolutionAdapter: + def __init__(self, source): + self.source = source + self.tables = {entity: table.copy() for entity, table in source.items()} + + def column(self, entity, variable): + if variable not in self.tables[entity]: + raise KeyError(f"{entity}.{variable}") + return np.asarray(self.tables[entity][variable]) + + def set_column(self, entity, variable, values): + self.tables[entity][variable] = np.asarray(values) + + +class StubMeasureProvider: + contract_targets = { + "needs_input_a": { + "bindings": {"policyengine": {"value_variable": "input_a"}} + }, + "needs_input_a_and_b": { + "bindings": { + "policyengine": {"value_expression": "input_a + input_b"} + } + }, + "needs_input_a_again": { + "bindings": {"policyengine": {"value_variable": "input_a"}} + }, + } + + def __init__(self): + self.calls = [] + + def knows(self, entity, variable): + return (entity, variable) in { + ("person", "input_a"), + ("person", "input_b"), + } + + def compute(self, entity, variable): + self.calls.append((entity, variable)) + values = { + ("person", "input_a"): np.array([1.0, 2.0, 3.0]), + ("person", "input_b"): np.array([10.0, 20.0, 30.0]), + }[(entity, variable)] + return values, f"stub:{entity}.{variable}" + + def receipt(self): + return {"provider": "stub"} + + +def _resolution_source(): + return { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "base": [5.0, 6.0, 7.0], + } + ) + } + + +def _resolution_registry(*names): + specs = [] + for name in names: + specs.append( + TargetSpec( + name=name, + entity="person", + measure=f"{name}_measure", + value=1.0, + source="test", + metadata={"contract_target_id": name}, + ) + ) + return TargetRegistry(specs, country="uk") + + def test_prepared_column_path_materializes_filtered_values(): registry = TargetRegistry( [ @@ -170,9 +259,7 @@ def test_missing_materialization_inputs_are_reported_as_skips(): country="uk", ) contract = { - "missing": { - "bindings": {"policyengine": {"value_variable": "not_present"}} - } + "missing": {"bindings": {"policyengine": {"value_variable": "not_present"}}} } result = materialize_target_bindings(StubAdapter(), registry, contract, period=2025) @@ -180,3 +267,509 @@ def test_missing_materialization_inputs_are_reported_as_skips(): assert len(result.skipped) == 1 assert result.skipped[0].name == "missing" assert result.report()["skipped_count"] == 1 + + +def test_resolve_target_measures_converges_over_multiple_rounds_without_frame_mutation(): + source = _resolution_source() + before = {entity: table.copy(deep=True) for entity, table in source.items()} + provider = StubMeasureProvider() + + resolution = resolve_target_measures( + lambda: ResolutionAdapter(source), + _resolution_registry("needs_input_a_and_b"), + provider, + period=2025, + ) + + assert set(resolution.measure_inputs) == { + ("person", "input_a"), + ("person", "input_b"), + } + assert len(resolution.receipt["rounds"]) == 3 + assert resolution.receipt["attached"] == { + "person.input_a": "stub:person.input_a", + "person.input_b": "stub:person.input_b", + } + for entity, table in source.items(): + pd.testing.assert_frame_equal(table, before[entity]) + + +def test_resolve_target_measures_computes_shared_missing_key_once_per_round(): + source = _resolution_source() + provider = StubMeasureProvider() + + resolve_target_measures( + lambda: ResolutionAdapter(source), + _resolution_registry("needs_input_a", "needs_input_a_again"), + provider, + period=2025, + ) + + assert provider.calls == [("person", "input_a")] + + +def test_resolve_target_measures_raises_when_provided_key_still_fails(): + class IgnoringAdapter(ResolutionAdapter): + def column(self, entity, variable): + if variable == "input_a": + raise KeyError(f"{entity}.{variable}") + return super().column(entity, variable) + + provider = StubMeasureProvider() + + with pytest.raises( + MeasureResolutionError, match="remained unmaterializable" + ) as error: + resolve_target_measures( + lambda: IgnoringAdapter(_resolution_source()), + _resolution_registry("needs_input_a"), + provider, + period=2025, + ) + + assert error.value.receipt["attached"] == { + "person.input_a": "stub:person.input_a" + } + assert error.value.receipt["skips"][-1]["name"] == "needs_input_a" + + +def test_resolve_target_measures_raises_on_no_progress_and_max_rounds(): + provider = StubMeasureProvider() + + with pytest.raises(MeasureResolutionError, match="does not know"): + resolve_target_measures( + lambda: ResolutionAdapter(_resolution_source()), + TargetRegistry( + [ + TargetSpec( + name="unknown", + entity="person", + measure="unknown_measure", + value=1.0, + source="test", + metadata={"contract_target_id": "unknown"}, + ) + ], + country="uk", + ), + provider, + period=2025, + contract_targets={ + "unknown": { + "bindings": { + "policyengine": {"value_variable": "not_provider_known"} + } + } + }, + ) + + with pytest.raises(MeasureResolutionError, match="did not converge"): + resolve_target_measures( + lambda: ResolutionAdapter(_resolution_source()), + _resolution_registry("needs_input_a"), + provider, + period=2025, + max_rounds=0, + ) + + +def test_assert_calibration_input_finite_reports_every_nan_float_column(): + class FrameLike: + entities = ("person", "household") + + def __init__(self): + self._tables = { + "person": pd.DataFrame( + { + "ok": [1.0, 2.0], + "bad": [np.nan, 3.0], + "also_bad": [np.nan, np.nan], + "label": ["x", None], + } + ), + "household": pd.DataFrame({"bad": [np.nan]}), + } + + def table(self, entity): + return self._tables[entity] + + with pytest.raises(ValueError) as error: + assert_calibration_input_finite(FrameLike()) + + message = str(error.value) + assert "person.bad: 1" in message + assert "person.also_bad: 2" in message + assert "household.bad: 1" in message + assert "person.label" not in message + + +def test_assert_calibration_input_finite_noops_on_clean_frame(): + class CleanFrame: + entities = ("person",) + + def table(self, entity): + assert entity == "person" + return pd.DataFrame({"value": [1.0], "label": ["x"]}) + + assert_calibration_input_finite(CleanFrame()) + + +def test_count_aliases_and_in_predicates_materialize_prepared_columns(): + registry = TargetRegistry( + [ + TargetSpec( + name="affected_households", + entity="household", + measure="affected_households_measure", + value=2.0, + source="test", + metadata={"contract_target_id": "affected_households"}, + ) + ], + country="uk", + ) + contract = { + "affected_households": { + "bindings": { + "policyengine": { + "value_variable": "household_count", + "filters": [ + { + "variable": "children", + "operator": "in", + "value": [3.0, 4.0], + } + ], + } + } + } + } + adapter = StubAdapter() + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + assert adapter.tables["household"]["affected_households_measure"].tolist() == [ + 1.0, + 0.0, + 1.0, + ] + + +def _banded_registry(*, count_measure: bool = True) -> TargetRegistry: + """Three adjacent income bands over one contract target.""" + + specs = [] + for lower, label in ((0, "0"), (20, "20"), (40, "40")): + specs.append( + TargetSpec( + name=f"band_{label}", + entity="person", + measure=f"income_band_{label}", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + "ledger_filter_total_income_lower_bound": str(lower), + }, + ) + ) + return TargetRegistry(specs, country="uk") + + +_BANDED_CONTRACT = { + "spi.income_by_band": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + } + } + } +} + + +def test_bands_slice_the_population_and_partition_it(): + adapter = StubAdapter() + registry = _banded_registry() + + result = materialize_target_bindings( + adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + # income is [10, 20, 30]; bands are [0,20), [20,40), [40,inf). + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] + assert list(adapter.tables["person"]["income_band_40"]) == [0.0, 0.0, 0.0] + # Every record lands in exactly one band: the bands partition the surface. + total = sum( + adapter.tables["person"][f"income_band_{label}"] + for label in ("0", "20", "40") + ) + assert list(total) == [1.0, 1.0, 1.0] + + +def test_adjacent_bands_are_not_identical(): + # The regression that would have caught the unsliced-measure defect: + # before banding was implemented every band returned the same unsliced + # column, so two adjacent bands compared equal. + adapter = StubAdapter() + materialize_target_bindings( + adapter, _banded_registry(), _BANDED_CONTRACT, period=2025 + ) + + assert not np.array_equal( + adapter.tables["person"]["income_band_0"], + adapter.tables["person"]["income_band_20"], + ) + + +def test_published_range_labels_band_in_model_units(): + adapter = StubAdapter() + # Monthly published bands, annual model values: "£1.01 to £2.00" x12 + # covers [12.12, 24.12) and the sibling opens at 24.12. + registry = TargetRegistry( + [ + TargetSpec( + name="award_low", + entity="person", + measure="award_low", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_family_type": "Single, no children", + "ledger_filter_monthly_award_bands": "£1.01 to £2.00", + }, + ), + TargetSpec( + name="award_high", + entity="person", + measure="award_high", + value=1.0, + source="test", + family="dwp_universal_credit", + metadata={ + "contract_target_id": "uc.award_bands", + "ledger_filter_monthly_award_bands": "£2.01 to £3.00", + }, + ), + ], + country="uk", + ) + contract = { + "uc.award_bands": { + "bindings": { + "policyengine": { + "value_variable": "person_count", + "groupby_variable": "income", + "from_entity": "person", + "band_period_factor": 12, + } + } + } + } + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + # income is [10, 20, 30]: only 20 falls inside [12.12, 24.12). + assert list(adapter.tables["person"]["award_low"]) == [0.0, 1.0, 0.0] + assert list(adapter.tables["person"]["award_high"]) == [0.0, 0.0, 1.0] + + +def test_unreadable_band_is_skipped_not_silently_unsliced(): + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="mystery_band", + entity="person", + measure="mystery_band", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + "ledger_filter_total_income_band": "not a range", + }, + ) + ], + country="uk", + ) + + result = materialize_target_bindings( + adapter, registry, _BANDED_CONTRACT, period=2025 + ) + + assert [skip.name for skip in result.skipped] == ["mystery_band"] + assert "no readable band edge" in result.skipped[0].reason + assert "mystery_band" not in adapter.tables["person"] + + +def test_entity_count_binding_yields_the_unit_indicator(): + # An entity-count value_variable with no count_of column counts records: + # the shape every two-child-limit household reference needs. Those + # references previously put a prose label in count_of, which the provider + # then looked up as a column and failed on. + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="affected", + entity="household", + measure="affected_households", + value=1.0, + source="test", + family="dwp_two_child_limit", + metadata={"contract_target_id": "tcl.households"}, + ) + ], + country="uk", + ) + contract = { + "tcl.households": { + "bindings": { + "policyengine": { + "kind": "baseline_flag_crosstab", + "affected_flag_variable": "affected", + "value_variable": "household_count", + "from_entity": "household", + } + } + } + } + + result = materialize_target_bindings(adapter, registry, contract, period=2025) + + assert result.skipped == () + assert list(adapter.tables["household"]["affected_households"]) == [1.0, 0.0, 1.0] + + +def test_crosstab_counts_a_real_value_variable_per_record(): + adapter = StubAdapter() + registry = TargetRegistry( + [ + TargetSpec( + name="children", + entity="household", + measure="affected_children", + value=1.0, + source="test", + family="dwp_two_child_limit", + metadata={"contract_target_id": "tcl.children"}, + ) + ], + country="uk", + ) + contract = { + "tcl.children": { + "bindings": { + "policyengine": { + "kind": "baseline_flag_crosstab", + "affected_flag_variable": "affected", + "count_of": "affected_children", + "value_variable": "children", + "from_entity": "household", + } + } + } + } + + materialize_target_bindings(adapter, registry, contract, period=2025) + + # children is [3, 2, 4], masked by affected [1, 0, 1]. + assert list(adapter.tables["household"]["affected_children"]) == [3.0, 0.0, 4.0] + + +def _two_filter_registry(*, matching: bool) -> TargetRegistry: + """Bands whose specs carry a second, competing band-like filter. + + ``matching`` decides whether the income filter's key relates to the + binding's ``groupby_variable``; when it does not, neither candidate can be + attributed and the edge must refuse rather than be guessed. + """ + + income_key = ( + "ledger_filter_income_lower_bound" + if matching + else "ledger_filter_total_income_lower_bound" + ) + specs = [] + for lower, age_lower, label in ((0, 16, "0"), (20, 30, "20")): + specs.append( + TargetSpec( + name=f"band_{label}", + entity="person", + measure=f"income_band_{label}", + value=1.0, + source="test", + family="hmrc_spi", + metadata={ + "contract_target_id": "spi.income_by_band", + income_key: str(lower), + "ledger_filter_age_lower_bound": str(age_lower), + }, + ) + ) + return TargetRegistry(specs, country="uk") + + +def test_band_edges_follow_the_groupby_variable_not_alphabetical_order(): + # "age" sorts before "income", so the alphabetically-first key would have + # sliced the income measure on age edges: distinct, plausible, and wrong. + adapter = StubAdapter() + + result = materialize_target_bindings( + adapter, _two_filter_registry(matching=True), _BANDED_CONTRACT, period=2025 + ) + + assert result.skipped == () + # income is [10, 20, 30]; income bands are [0,20) and [20,inf). + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] + + +def test_unattributable_band_filters_refuse_rather_than_guess(): + adapter = StubAdapter() + + with pytest.raises(ValueError) as error: + materialize_target_bindings( + adapter, + _two_filter_registry(matching=False), + _BANDED_CONTRACT, + period=2025, + ) + + message = str(error.value) + assert "band-like Ledger filters" in message + assert "ledger_filter_age_lower_bound" in message + assert "band_filter_dimension" in message + + +def test_declared_band_filter_dimension_breaks_the_tie(): + adapter = StubAdapter() + contract = { + "spi.income_by_band": { + "bindings": { + "policyengine": { + **_BANDED_CONTRACT["spi.income_by_band"]["bindings"][ + "policyengine" + ], + "band_filter_dimension": "total_income", + } + } + } + } + + result = materialize_target_bindings( + adapter, _two_filter_registry(matching=False), contract, period=2025 + ) + + assert result.skipped == () + assert list(adapter.tables["person"]["income_band_0"]) == [1.0, 0.0, 0.0] + assert list(adapter.tables["person"]["income_band_20"]) == [0.0, 1.0, 1.0] diff --git a/packages/microcosm-build/tests/test_uk_calibration_run.py b/packages/microcosm-build/tests/test_uk_calibration_run.py new file mode 100644 index 0000000000..e77037d938 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_calibration_run.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.logbook import canonical_json_bytes +from microcosm.build.uk_runtime import calibration_run +from microcosm.build.uk_runtime.calibration_run import ( + UK_CALIBRATION_GATE_SCOPE, + UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, + UKCalibrationRunPaths, + run_uk_calibration, +) +from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine +from microcosm.build.uk_runtime.national_frame import ( + load_uk_national_frame, + uk_national_frame, + write_uk_national_frame, +) +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame import WeightKind + +SIGNING_KEY = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" + + +@pytest.fixture(autouse=True) +def _signing_key(monkeypatch): + monkeypatch.setenv("MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY", SIGNING_KEY) + + +def _frame(): + ids = np.arange(4, dtype="int64") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "nhs_spending": [50.0, 50.0, 50.0, 50.0], + } + ), + benunit=pd.DataFrame( + {"benunit_id": ids, "universal_credit": [1.0, 1.0, 0.0, 0.0]} + ), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": [10.0, 10.0, 10.0, 10.0], + "household_is_spi_synthetic": [False, False, False, False], + "household_is_capital_gains_clone": [False, False, False, False], + "electricity_consumption": [1.0, 1.0, 1.0, 1.0], + "gas_consumption": [1.0, 1.0, 1.0, 1.0], + } + ), + time_period="2023", + weight_kind=WeightKind.DESIGN, + ) + + +def _registry(): + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=20.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _admin_anchor_values(): + values = {} + for entry in load_country_spec("uk").gates.gates: + if entry.id == "uk_aggregate_admin": + for anchor in entry.parameters["anchors"]: + values[str(anchor["name"])] = float(anchor["value"]) + return values + + +def test_gate_scope_classifies_every_uk_gate(): + all_ids = {entry.id for entry in load_country_spec("uk").gates.gates} + assert set(UK_CALIBRATION_GATE_SCOPE) | set(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) == all_ids + assert set(UK_CALIBRATION_GATE_SCOPE).isdisjoint(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + assert all(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS.values()) + + +def test_import_hygiene_does_not_load_national_build_in_fresh_subprocess(): + source = Path(calibration_run.__file__).read_text(encoding="utf-8") + assert "microcosm.build.uk_runtime.national_build" not in source + assert "from microcosm.build.uk_runtime.national_build" not in source + + +def test_run_uk_calibration_writes_cross_pinned_outputs(monkeypatch, tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + source_pins = { + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size}, + "ledger_facts": {"sha256": "a" * 64, "size_bytes": 1}, + } + + result = run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins=source_pins, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="test-run", + ) + + assert paths.staging_h5.exists() + assert paths.diagnostics_json.exists() + assert paths.build_record_json.exists() + assert paths.terminal_gate_json.exists() + assert result.build_record["artifacts"]["staging_h5"]["sha256"] == _sha(paths.staging_h5) + assert result.build_record["artifacts"]["diagnostics_json"]["sha256"] == _sha(paths.diagnostics_json) + assert result.build_record["artifacts"]["terminal_gate_json"]["sha256"] == _sha(paths.terminal_gate_json) + staged, _ = load_uk_national_frame(paths.staging_h5) + assert staged.weights_for("household").kind is WeightKind.CALIBRATED + report = json.loads(paths.terminal_gate_json.read_text()) + assert report["posture"] == "calibration_seam" + assert set(report["scope_exclusions"]) == set(UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS) + attestation = report["attestation"] + signature = attestation["signature"] + attestation["signature"] = None + key = b"0123456789abcdef0123456789abcdef" + assert hmac.new(key, canonical_json_bytes(report), hashlib.sha256).hexdigest() == signature + assert result.logbook_spool.exists() + + +def test_run_uk_calibration_refuses_input_sha_before_outputs(tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + with pytest.raises(ValueError, match="sha mismatch"): + run_uk_calibration( + paths=paths, + input_sha256="0" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="bad-sha", + ) + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + + +def test_seam_never_modifies_data_variables(monkeypatch, tmp_path: Path): + """The seam's defining invariant: weights move, data never does. + + Every data column of every entity table in the staged H5 must be + byte-identical to the input; only the household weights, the weight + kind, and exactly one appended mass record may differ. + """ + + pytest.importorskip("tables") # pandas HDF backend + + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + # Target 30 against an initial weighted UC count of 20, so the solve + # genuinely has to move weights while the data stays untouched. + pulling_registry = TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=30.0, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + run_uk_calibration( + paths=paths, + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=pulling_registry, + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=50), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={}, + release_candidate=False, + release_id="invariant-run", + ) + + source, _ = load_uk_national_frame(input_h5) + staged, _ = load_uk_national_frame(paths.staging_h5) + for entity in ("person", "benunit", "household"): + left = source.table(entity) + right = staged.table(entity) + assert list(left.columns) == list(right.columns), entity + for column in left.columns: + pd.testing.assert_series_equal(left[column], right[column]) + assert staged.weights_for("household").kind is WeightKind.CALIBRATED + assert not np.allclose( + staged.weights_for("household").values, + source.weights_for("household").values, + ) + assert len(staged.mass_log) == len(source.mass_log) + 1 + + +def test_aggregate_admin_measurement_convention_and_refusals(): + frame = _frame() + manifest = calibration_run._calibration_gate_manifest() + + totals, receipt = calibration_run._aggregate_admin_totals(frame, manifest) + + # Small anchors (NEED means) measure as the weighted mean over carriers; + # the NHS total measures as the person total under mapped household + # weights: 4 persons x 50.0 x weight 10.0. + assert totals["need_electricity_mean_spending"] == pytest.approx(1.0) + assert totals["need_gas_mean_spending"] == pytest.approx(1.0) + assert totals["nhs_spending_total"] == pytest.approx(2000.0) + by_anchor = {row["anchor"]: row for row in receipt} + assert by_anchor["nhs_spending_total"]["entity"] == "person" + assert ( + by_anchor["need_electricity_mean_spending"]["statistic_convention"] + == "assessed_by_anchor_magnitude" + ) + + stripped = _frame() + stripped.table("household").drop(columns=["electricity_consumption"], inplace=True) + with pytest.raises(ValueError, match="household.electricity_consumption"): + calibration_run._aggregate_admin_totals(stripped, manifest) + + +def test_seam_pipeline_derives_a_ratified_logbook_scope(): + """The seam appends to the FRS line's chain, not a new unratified one.""" + + logbook_tool = _load_logbook_tool() + + scope = logbook_tool._chain_scope(calibration_run._PIPELINE) + + assert scope == "uk/frs" + assert scope in logbook_tool.DECLARED_SCOPES + + +def _load_logbook_tool(): + import importlib.util + + path = Path(__file__).resolve().parents[3] / "tools" / "logbook.py" + spec = importlib.util.spec_from_file_location("_logbook_tool", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_refusal_records_a_failed_attempt_and_stages_nothing(tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + paths = UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ) + + with pytest.raises(ValueError, match="sha mismatch"): + run_uk_calibration( + paths=paths, + input_sha256="0" * 64, + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=1), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": { + "sha256": _sha(input_h5), + "size_bytes": input_h5.stat().st_size, + } + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="refused-run", + ) + + # Every terminal disposition is a row; a refusal that left the chain + # silent would hide the attempt entirely. + spooled = sorted((tmp_path / "logbook-spool").rglob("*.json")) + assert spooled, "refusal recorded no Logbook row" + rows = [json.loads(path.read_text()) for path in spooled] + assert [row["disposition"] for row in rows] == ["failed"] + assert rows[0]["pipeline"] == calibration_run._PIPELINE + # The refusal is explained on disk, not only in the row. + receipts = sorted((tmp_path / "logbook-receipts").rglob("error.json")) + assert len(receipts) == 1 + assert not paths.staging_h5.exists() + assert not paths.diagnostics_json.exists() + assert not paths.terminal_gate_json.exists() + + +def test_attempt_ids_are_unique_across_reruns_of_one_release( + monkeypatch, tmp_path: Path +): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + source_pins = { + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + } + build_ids = [] + for attempt in ("a", "b"): + run_dir = tmp_path / attempt + run_dir.mkdir() + result = run_uk_calibration( + paths=UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=run_dir / "staged.h5", + diagnostics_json=run_dir / "diagnostics.json", + build_record_json=run_dir / "build_record.json", + terminal_gate_json=run_dir / "terminal_gates.json", + ), + input_sha256=_sha(input_h5), + ledger_artifact=object(), + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins=source_pins, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="one-release-id", + ) + build_ids.append(result.build_record["build_id"]) + + # Both the local chain and the store reject a duplicate build id, so one + # release re-run twice must not collide. + assert build_ids[0] != build_ids[1] + assert all(value.startswith("uk-frs-calibration-attempt-") for value in build_ids) + + +def test_verified_ledger_identity_reaches_the_run_evidence(monkeypatch, tmp_path: Path): + pytest.importorskip("tables") # pandas HDF backend + monkeypatch.setattr( + calibration_run, + "_aggregate_admin_totals", + lambda frame, manifest: (_admin_anchor_values(), []), + ) + input_h5 = tmp_path / "input.h5" + write_uk_national_frame(_frame(), input_h5) + artifact = SimpleNamespace( + facts_sha256="d" * 64, + fact_row_count=107_550, + manifest_sha256="e" * 64, + manifest={ + "artifact_id": "chronicle-uk-artifact-1cab809", + "profile": "uk-national", + "schema_version": 1, + "unrelated": "not carried", + }, + ) + + result = run_uk_calibration( + paths=UKCalibrationRunPaths( + input_h5=input_h5, + staging_h5=tmp_path / "staged.h5", + diagnostics_json=tmp_path / "diagnostics.json", + build_record_json=tmp_path / "build_record.json", + terminal_gate_json=tmp_path / "terminal_gates.json", + ), + input_sha256=_sha(input_h5), + ledger_artifact=artifact, + register_registry=_registry(), + calibration_year=2025, + exclusion_receipt={}, + doctrine=UKNationalSolveDoctrine(epochs=5), + doctrine_overrides={}, + measure_resolver=None, + source_pins={ + "input_h5": {"sha256": _sha(input_h5), "size_bytes": input_h5.stat().st_size} + }, + run_config_extra={"calibration_year": 2025}, + release_candidate=False, + release_id="ledger-identity", + ) + + ledger = result.build_record["run_config"]["ledger"] + assert ledger["facts_sha256"] == "d" * 64 + assert ledger["manifest_sha256"] == "e" * 64 + assert ledger["fact_row_count"] == 107_550 + assert ledger["manifest"] == { + "artifact_id": "chronicle-uk-artifact-1cab809", + "profile": "uk-national", + "schema_version": 1, + } + # A bare feed carries no manifest, and that absence is recorded rather + # than invented. + assert calibration_run._ledger_provenance(object())["manifest_sha256"] is None diff --git a/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py new file mode 100644 index 0000000000..78deb7789d --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_calibration_seam_driver.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from microcosm.build.uk_runtime.ledger_targets import UKLedgerTargetCompilation +from microcosm.calibrate import TargetRegistry, TargetSpec + + +def _load_driver_module(): + root = Path(__file__).resolve().parents[3] + path = root / "tools" / "calibrate_uk_national_dataset.py" + spec = importlib.util.spec_from_file_location("calibrate_uk_national_dataset", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _registry(): + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=1.0, + source="test", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + +def _args(tmp_path: Path) -> list[str]: + paths = { + "input": tmp_path / "input.h5", + "ledger": tmp_path / "ledger", + "staging": tmp_path / "staging.h5", + "diagnostics": tmp_path / "diagnostics.json", + "record": tmp_path / "record.json", + } + paths["ledger"].mkdir(exist_ok=True) + for key, path in paths.items(): + if key != "ledger": + path.write_bytes(key.encode()) + return [ + "--input-h5", + str(paths["input"]), + "--input-sha256", + "a" * 64, + "--ledger-facts", + str(paths["ledger"]), + "--ledger-facts-sha256", + "b" * 64, + "--ledger-manifest-sha256", + "c" * 64, + "--staging-h5", + str(paths["staging"]), + "--diagnostics-json", + str(paths["diagnostics"]), + "--build-record-json", + str(paths["record"]), + "--release-id", + "dev-calibration", + ] + + +def test_driver_refuses_release_candidate_with_each_override(tmp_path: Path): + driver = _load_driver_module() + override_flags = [ + ["--epochs", "128"], + ["--target-weight-rule", "family_equal"], + ["--learning-rate", "0.01"], + ["--target-loss-cap", "5"], + ] + for flag in override_flags: + with pytest.raises(SystemExit): + driver._parse_args(_args(tmp_path) + ["--release-candidate", *flag]) + + +def test_driver_refuses_release_candidate_with_operator_exclusions(tmp_path: Path): + # The scoped battery carries no target-surface gate, so an operator + # register could narrow what a release candidate was measured against + # without anything noticing. + driver = _load_driver_module() + exclusions = tmp_path / "operator.json" + exclusions.write_text("{}", encoding="utf-8") + + with pytest.raises(SystemExit): + driver._parse_args( + _args(tmp_path) + + ["--release-candidate", "--measure-exclusions", str(exclusions)] + ) + + # Without the release-candidate posture the same register is accepted. + parsed = driver._parse_args( + _args(tmp_path) + ["--measure-exclusions", str(exclusions)] + ) + assert parsed.measure_exclusions == exclusions + + +def test_driver_refuses_bad_sha_and_path_alias(tmp_path: Path): + driver = _load_driver_module() + base_args = _args(tmp_path) + with pytest.raises(SystemExit): + driver._parse_args([*base_args[:3], "not-a-sha", *base_args[4:]]) + + args = _args(tmp_path) + staging_index = args.index("--staging-h5") + 1 + args[staging_index] = args[args.index("--input-h5") + 1] + with pytest.raises(SystemExit, match="distinct paths"): + driver._parse_args(args) + + +def test_driver_threads_registry_exclusions_resolver_and_overrides(monkeypatch, tmp_path, capsys): + driver = _load_driver_module() + calls = [] + registry = _registry() + artifact = SimpleNamespace( + path=tmp_path / "ledger", + facts=({"fact": 1},), + facts_sha256="b" * 64, + manifest_sha256="c" * 64, + ) + artifact.path.mkdir() + (artifact.path / "consumer_facts.jsonl").write_text("{}", encoding="utf-8") + monkeypatch.setattr(driver, "load_ledger_consumer_artifact", lambda *a, **k: artifact) + monkeypatch.setattr( + driver, + "compile_uk_target_registry", + lambda facts, target_period: UKLedgerTargetCompilation(registry, ()), + ) + monkeypatch.setattr( + driver, + "load_uk_frs_release", + lambda: SimpleNamespace(calibration_year=2025), + ) + monkeypatch.setattr(driver, "load_uk_calibration_measure_exclusions", lambda path: ()) + monkeypatch.setattr( + driver, + "apply_uk_calibration_measure_exclusions", + lambda reg, exclusions: (reg, {"excluded": "reviewed"}), + ) + + class FakeResolver: + def __init__(self, **kwargs): + self.kwargs = kwargs + + monkeypatch.setattr(driver, "UKMeasureResolver", FakeResolver) + + def fake_run(**kwargs): + calls.append(kwargs) + return SimpleNamespace( + staging_sha256="1" * 64, + diagnostics_sha256="2" * 64, + terminal_gate_sha256="3" * 64, + build_record_sha256="4" * 64, + build_record={"gate_summary": {"uk_target_fit": "passed"}}, + ) + + monkeypatch.setattr(driver, "run_uk_calibration", fake_run) + + argv = _args(tmp_path) + # The driver verifies the input pin before the resolver reads the H5, so + # the threading test must pin the fixture file's real digest. + argv[argv.index("--input-sha256") + 1] = driver._sha256_file( + Path(argv[argv.index("--input-h5") + 1]) + ) + result = driver.main(argv + ["--epochs", "128"]) + + assert result == 0 + call = calls[0] + assert call["register_registry"] is registry + assert call["calibration_year"] == 2025 + assert call["exclusion_receipt"] == {"excluded": "reviewed"} + assert call["doctrine"].epochs == 128 + assert call["doctrine_overrides"] == {"epochs": {"default": 256, "effective": 128}} + assert isinstance(call["measure_resolver"], FakeResolver) + assert call["measure_resolver"].kwargs["simulation_source"] == call["paths"].input_h5 + assert call["source_pins"]["ledger_facts"] == {"sha256": "b" * 64, "size_bytes": 2} + assert "uk_target_fit" in capsys.readouterr().out diff --git a/packages/microcosm-build/tests/test_uk_diagnostics.py b/packages/microcosm-build/tests/test_uk_diagnostics.py index 9dc8f65c79..af25c584f5 100644 --- a/packages/microcosm-build/tests/test_uk_diagnostics.py +++ b/packages/microcosm-build/tests/test_uk_diagnostics.py @@ -21,6 +21,7 @@ CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, TargetRegistry, TargetSpec, + diagnostics_payload, score_targets, ) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights @@ -344,6 +345,32 @@ def test_payload_preserves_common_schema_and_adds_versioned_uk_evidence() -> Non assert json.loads(json.dumps(payload, allow_nan=False)) == payload +def test_uk_payload_shared_layer_matches_shared_diagnostics_format() -> None: + result, frame, registry, geography = _diagnostics_case() + build = {"release_id": "fixture"} + + shared = diagnostics_payload(result, target_registry=registry, build=build) + uk = uk_calibration_diagnostics_payload( + result, + frame, + target_geography_levels=geography, + target_registry=registry, + build=build, + ) + + uk_shared_layer = { + key: value for key, value in uk.items() if key != "uk_diagnostics" + } + assert uk_shared_layer == shared + assert set(uk) == {*shared.keys(), "uk_diagnostics"} + assert set(uk["uk_diagnostics"]) == { + "schema_version", + "weights", + "zero_weight_rows_by_stratum", + "target_pass_rates_by_geography_level", + } + + def test_payload_requires_exact_explicit_geography_mapping() -> None: result, frame, registry, geography = _diagnostics_case() missing = dict(geography) diff --git a/packages/microcosm-build/tests/test_uk_ledger_targets.py b/packages/microcosm-build/tests/test_uk_ledger_targets.py index 023ddb7f82..793222f25d 100644 --- a/packages/microcosm-build/tests/test_uk_ledger_targets.py +++ b/packages/microcosm-build/tests/test_uk_ledger_targets.py @@ -15,23 +15,52 @@ class StubUKAdapter: + """Seven people across three households, flagged child by flagged child. + + The flag and the count are deliberately distinct here: household 0 holds + three flagged children and household 2 holds two, so a household-grain + read of the boolean (1/0/1) can never be mistaken for the child counts + (3/0/2) the contract's count rows declare. + """ + def __init__(self): + person_household = np.array([0, 0, 0, 0, 1, 2, 2]) + child_flags = np.array([True, True, True, False, False, True, True]) self.tables = { "person": { - "capital_gains": np.array([0.0, 5_000.0, 20_000.0]), + "capital_gains": np.array( + [0.0, 0.0, 0.0, 5_000.0, 0.0, 0.0, 20_000.0] + ), + "person_household_id": person_household, + "uc_is_child_limit_affected": child_flags, }, "household": { + "household_id": np.array([0, 1, 2]), + # The household-grain flag is the any-collapse: "this + # household contains at least one flagged child". "uc_is_child_limit_affected": np.array([1.0, 0.0, 1.0]), - "children_count": np.array([3.0, 2.0, 4.0]), }, } - def column(self, entity, variable): return self.tables[entity][variable] def set_column(self, entity, variable, values): self.tables.setdefault(entity, {})[variable] = np.asarray(values) + def entity_reduction(self, reduction): + assert reduction["reduce"] == "sum" + assert reduction["entity"] == "person" + variable = reduction["variable"] + values = self.tables["person"][variable].astype(float) + households = self.tables["person"]["person_household_id"] + return np.asarray( + [ + values[households == household_id].sum() + for household_id in self.tables["household"]["household_id"] + ], + dtype=float, + ) + def parameter(self, name, period): assert name == "gov.hmrc.cgt.annual_exempt_amount" assert period == 2025 @@ -92,7 +121,90 @@ def test_materialize_uk_ledger_targets_with_stub_adapter(): result = materialize_uk_ledger_targets(adapter, registry, period=2025) assert result.skipped == () - assert adapter.tables["person"]["hmrc/cgt_taxpayers"].tolist() == [0.0, 0.0, 1.0] + assert adapter.tables["person"]["hmrc/cgt_taxpayers"].tolist() == [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + ] + # Child counts, not household indicators: the declared value_reduction + # sums the flag over each household's people. A boolean any-collapse + # would have published [1.0, 0.0, 1.0] against a count target. assert adapter.tables["household"][ "dwp/uc/two_child_limit/children_affected" - ].tolist() == [3.0, 0.0, 4.0] + ].tolist() == [3.0, 0.0, 2.0] + + +def _composition_frame(): + """Two households: one lone parent with a child, one older couple.""" + + import pandas as pd + + from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + return Frame( + { + "person": pd.DataFrame( + { + "person_id": np.arange(5), + "person_benunit_id": [0, 0, 1, 2, 2], + "person_household_id": [0, 0, 0, 1, 1], + "is_child": [0.0, 1.0, 0.0, 0.0, 0.0], + "age": [40.0, 8.0, 30.0, 70.0, 72.0], + } + ), + "benunit": pd.DataFrame( + { + "benunit_id": np.arange(3), + "family_type": ["LONE_PARENT", "SINGLE", "COUPLE_NO_CHILDREN"], + } + ), + "household": pd.DataFrame({"household_id": np.arange(2)}), + }, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(np.array([10.0, 20.0]), WeightKind.DESIGN)}, + metadata={"time_period": "2025"}, + ) + + +def test_household_condition_reduces_person_level_conditions(): + # Person-level conditions collapse through person_household_id. Building + # the group-membership column here would ask for "person_person_id", + # which cannot exist — the failure that excluded every ONS + # household-composition reference from calibration. + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + adapter = UKFrameTargetAdapter(_composition_frame()) + + children = adapter.household_condition( + { + "variable": "is_child", + "entity": "person", + "reduce": "sum", + "operator": ">=", + "value": 1, + } + ) + + assert list(children) == [True, False] + + +def test_household_condition_still_reduces_group_entities(): + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + adapter = UKFrameTargetAdapter(_composition_frame()) + + single = adapter.household_condition( + { + "variable": "family_type", + "entity": "benunit", + "reduce": "any", + "operator": "==", + "value": "SINGLE", + } + ) + + assert list(single) == [True, False] diff --git a/packages/microcosm-build/tests/test_uk_measure_simulation.py b/packages/microcosm-build/tests/test_uk_measure_simulation.py new file mode 100644 index 0000000000..ae7bb44b60 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_measure_simulation.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import measure_simulation +from microcosm.build.uk_runtime.measure_simulation import ( + UKMeasureResolver, + apply_uk_calibration_measure_exclusions, + compute_uk_measure_input, + load_uk_calibration_measure_exclusions, +) +from microcosm.calibrate import TargetRegistry, TargetSpec + + +class FrameStub: + def __init__(self): + self._tables = { + "person": pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_benunit_id": [10, 10, 20], + "person_household_id": [100, 100, 200], + } + ), + "benunit": pd.DataFrame({"benunit_id": [10, 20]}), + "household": pd.DataFrame({"household_id": [100, 200]}), + } + + def table(self, entity): + return self._tables[entity] + + +def _stub_value_type(values): + """The policyengine ``value_type`` a real variable definition would carry.""" + + return {"f": float, "i": int, "b": bool}.get( + np.asarray(values).dtype.kind, str + ) + + +class SimulationStub: + def __init__(self, values): + self.values = values + variables = { + name: SimpleNamespace( + entity=SimpleNamespace(key=entity), + value_type=_stub_value_type(stub_values), + ) + for name, (entity, stub_values) in values.items() + } + self.tax_benefit_system = SimpleNamespace(variables=variables) + self.calls = [] + + def calculate(self, variable, year, map_to=None): + self.calls.append((variable, year, map_to)) + key = (variable, map_to) + if key in self.values: + return self.values[key][1] + return self.values[variable][1] + + +def test_compute_uk_measure_input_native_route(): + sim = SimulationStub({"income_tax": ("person", np.array([1.0, 2.0, 3.0]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "person", "income_tax", 2025 + ) + + assert route == "native" + assert values.tolist() == [1.0, 2.0, 3.0] + + +def test_compute_uk_measure_input_categorical_broadcast_group_to_person(): + sim = SimulationStub({"family_type": ("benunit", np.array(["couple", "single"]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "person", "family_type", 2025 + ) + + assert route == "categorical_broadcast_benunit_to_person" + assert values.tolist() == ["couple", "couple", "single"] + + +def test_compute_uk_measure_input_bool_any_collapse_person_to_group(): + sim = SimulationStub({"is_disabled": ("person", np.array([False, True, False]))}) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "household", "is_disabled", 2025 + ) + + assert route == "bool_any_collapse_person_to_household" + assert values.tolist() == [1.0, 0.0] + + +def test_compute_uk_measure_input_numeric_map_to_entity(): + sim = SimulationStub( + { + "benefit": ("person", np.array([1.0, 2.0, 3.0])), + ("benefit", "household"): ("household", np.array([3.0, 3.0])), + } + ) + + values, route = compute_uk_measure_input( + FrameStub(), sim, "household", "benefit", 2025 + ) + + assert route == "map_to_household" + assert values.tolist() == [3.0, 3.0] + assert sim.calls[-1] == ("benefit", 2025, "household") + + +def test_compute_uk_measure_input_refuses_unknown_and_length_mismatch(): + with pytest.raises(KeyError, match="no variable"): + compute_uk_measure_input(FrameStub(), SimulationStub({}), "person", "missing", 2025) + + sim = SimulationStub({"income_tax": ("person", np.array([1.0]))}) + with pytest.raises(ValueError, match="produced 1 values"): + compute_uk_measure_input(FrameStub(), sim, "person", "income_tax", 2025) + + +def _write_exclusions(path: Path, payload: dict) -> Path: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def test_exclusion_loader_refusals(tmp_path: Path): + base = { + "schema_version": 1, + "exclusions": [ + {"name": "a", "reason": "reviewed reason", "tracking": "microcosm#623"} + ], + } + + with pytest.raises(ValueError, match="schema_version"): + load_uk_calibration_measure_exclusions( + _write_exclusions(tmp_path / "bad-version.json", {**base, "schema_version": 2}) + ) + with pytest.raises(ValueError, match="unknown top-level"): + load_uk_calibration_measure_exclusions( + _write_exclusions(tmp_path / "unknown.json", {**base, "extra": True}) + ) + with pytest.raises(ValueError, match="empty reason"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "empty.json", + {**base, "exclusions": [{"name": "a", "reason": "", "tracking": "x"}]}, + ) + ) + with pytest.raises(ValueError, match="duplicate"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "duplicate.json", + { + **base, + "exclusions": [ + {"name": "a", "reason": "one", "tracking": "x"}, + {"name": "a", "reason": "two", "tracking": "x"}, + ], + }, + ) + ) + + +def test_exclusion_applier_returns_pruned_registry_and_receipt(): + registry = TargetRegistry( + [ + TargetSpec(name="drop", entity="person", measure="drop", value=1.0, source="test"), + TargetSpec(name="keep", entity="person", measure="keep", value=1.0, source="test"), + ], + country="uk", + ) + + pruned, receipt = apply_uk_calibration_measure_exclusions( + registry, + ({"name": "drop", "reason": "reviewed", "tracking": "microcosm#623"},), + ) + + assert [spec.name for spec in pruned.specs] == ["keep"] + assert receipt == { + "drop": {"reason": "reviewed", "tracking": "microcosm#623"} + } + with pytest.raises(ValueError, match="matched zero"): + apply_uk_calibration_measure_exclusions( + registry, + ({"name": "stale", "reason": "reviewed", "tracking": "microcosm#623"},), + ) + + +def test_packaged_exclusions_load(): + exclusions = load_uk_calibration_measure_exclusions() + assert {entry["name"] for entry in exclusions} == { + "hmrc.salary_sacrifice.it_relief_basic_rate", + "hmrc.salary_sacrifice.it_relief_higher_rate", + "hmrc.salary_sacrifice.it_relief_additional_rate", + "hmrc.salary_sacrifice.nics_relief_employee", + "hmrc.salary_sacrifice.nics_relief_employer", + } + + +def test_measure_resolver_direct_and_scratch_receipts(monkeypatch, tmp_path: Path): + created = [] + + class FakeMicrosimulation: + def __init__(self, *, dataset): + created.append(dataset) + self.tax_benefit_system = SimpleNamespace(variables={}) + + fake_policyengine_uk = SimpleNamespace( + __version__="9.9.9", Microsimulation=FakeMicrosimulation + ) + monkeypatch.setitem(sys.modules, "policyengine_uk", fake_policyengine_uk) + writes = [] + monkeypatch.setattr( + measure_simulation, + "write_uk_national_frame", + lambda frame, path: writes.append((frame, path)) or path, + ) + + frame = FrameStub() + direct = UKMeasureResolver( + simulation_source=tmp_path / "input.h5", + scratch_dir=tmp_path, + year=2025, + frame=frame, + ) + scratch = UKMeasureResolver( + simulation_source=None, + scratch_dir=tmp_path, + year=2025, + frame=frame, + ) + + assert direct.receipt()["mode"] == "direct_h5" + assert direct.receipt()["policyengine_uk_version"] == "9.9.9" + assert scratch.receipt()["mode"] == "scratch_frame_export" + assert writes == [(frame, tmp_path / "simulation-input.h5")] + assert created == [str(tmp_path / "input.h5"), str(tmp_path / "simulation-input.h5")] + + +def _resolver_over(sim, monkeypatch, tmp_path: Path) -> UKMeasureResolver: + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace(__version__="9.9.9", Microsimulation=lambda *, dataset: sim), + ) + return UKMeasureResolver( + simulation_source=tmp_path / "input.h5", + scratch_dir=tmp_path, + year=2025, + frame=FrameStub(), + ) + + +def test_knows_answers_only_for_routes_compute_can_take(monkeypatch, tmp_path: Path): + sim = SimulationStub( + { + "income_tax": ("person", np.array([1.0, 2.0, 3.0])), + "is_disabled": ("person", np.array([False, True, False])), + "family_type": ("benunit", np.array(["couple", "single"])), + "benunit_income": ("benunit", np.array([1.0, 2.0])), + } + ) + resolver = _resolver_over(sim, monkeypatch, tmp_path) + + assert resolver.knows("person", "income_tax") + assert resolver.knows("household", "income_tax") + assert resolver.knows("household", "is_disabled") + assert resolver.knows("person", "family_type") + # Numeric group-to-group rides the map_to route… + assert resolver.knows("household", "benunit_income") + # …but a categorical one has no benunit-to-household mapping at all, and + # the fence must say so rather than let compute raise mid-resolution. + assert not resolver.knows("household", "family_type") + assert not resolver.knows("person", "no_such_variable") + assert not resolver.knows("firm", "income_tax") + + +def test_unknown_categorical_mapping_refuses_through_the_fence( + monkeypatch, tmp_path: Path +): + sim = SimulationStub({"family_type": ("benunit", np.array(["couple", "single"]))}) + resolver = _resolver_over(sim, monkeypatch, tmp_path) + + assert not resolver.knows("household", "family_type") + with pytest.raises(KeyError, match="no categorical mapping"): + resolver.compute("household", "family_type") + + +def test_exclusion_loader_requires_tracking(tmp_path: Path): + # An exclusion narrows the calibrated target surface; the register has to + # say where each narrowing is being resolved, not only why. + with pytest.raises(ValueError, match="empty tracking"): + load_uk_calibration_measure_exclusions( + _write_exclusions( + tmp_path / "untracked.json", + { + "schema_version": 1, + "exclusions": [ + {"name": "a", "reason": "reviewed reason", "tracking": ""} + ], + }, + ) + ) diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index 1a9766e68f..59d845fae8 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -31,15 +31,21 @@ from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, ) +from microcosm.build.uk_runtime.national_doctrine import UKNationalSolveDoctrine from microcosm.build.uk_runtime.national_frame import ( _uk_gate_surface, uk_household_weight_kind, uk_national_frame, uk_time_period, ) +from microcosm.build.uk_runtime.parity_reference import ( + EfrsParityReference, + EfrsParitySource, +) from microcosm.build.uk_runtime.release_input_coverage import ( uk_release_input_coverage_gate, ) +from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import Frame, MassChangeRecord, WeightKind TEST_UK_RELEASE_ID = "populace-uk-2023-frs-k535080" @@ -355,6 +361,13 @@ def _registry_with_calibration() -> dict[str, UKGateBinding]: return registry +def _registry_with_calibration_and_parity_trio() -> dict[str, UKGateBinding]: + registry = _registry_with_calibration() + for name in ("export_surface", "target_surface", "target_fit"): + registry[name] = UK_GATE_REGISTRY[name] + return registry + + def _uc_reference(**overrides) -> LedgerTargetReference: values = { "name": "dwp.uc.households", @@ -390,6 +403,22 @@ def _calibration_fact(value: float = 60.0) -> dict: } +def _calibration_registry(value: float = 60.0) -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=value, + source="test", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + def test_driver_validates_the_uk_residue_after_each_stage( monkeypatch, tmp_path ) -> None: @@ -523,16 +552,16 @@ def test_resumed_national_calibration_feeds_reference_coverage_gate( _write_two_row_h5(input_h5, include_calibration_columns=True) frame, _provenance = load_uk_national_frame(input_h5) stage = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) staged = stage(frame) metadata = json.loads(json.dumps(stage.checkpoint_metadata())) resumed = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) resumed.resume_from_checkpoint(metadata, staged) @@ -1205,6 +1234,108 @@ def test_national_build_parity_trio_is_evidence_absent( assert gates["uk_qrf_tail_concentration"]["status"] == "failed" +def _fixture_parity_reference(input_h5) -> EfrsParityReference: + """A synthetic frozen instrument matching the fixture's export surface. + + Reference and candidate stay independently derived in production; the + test constructs the reference from the *input* artifact, before any + stage runs, so a stage that leaked scratch columns would still fail + the export-surface comparison. + """ + + frame, _provenance = load_uk_national_frame(input_h5) + input_entities = { + str(column): entity + for entity in frame.entities + for column in frame.table(entity).columns + } + return EfrsParityReference( + source=EfrsParitySource( + repo_id="example/synthetic", + repo_type="model", + filename="synthetic_reference.h5", + revision="0" * 40, + sha256="0" * 64, + url="https://example.invalid/synthetic_reference.h5", + vintage="2024_25", + period="2024", + size_bytes=1, + ), + nonzero_shares={name: 1.0 for name in input_entities}, + input_entities=input_entities, + ) + + +def test_national_build_parity_trio_evaluates_for_armed_calibration( + tmp_path, +) -> None: + pytest.importorskip("tables") + + input_h5 = tmp_path / "healthy.h5" + terminal_json = tmp_path / "terminal_gates.json" + _write_two_row_h5(input_h5, include_calibration_columns=True) + + _run_national_build( + input_h5=input_h5, + staging_h5=tmp_path / "staging.h5", + stages=( + Stage( + name="national_calibration", + transform=UKNationalCalibrationStage( + _calibration_registry(55.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ), + ), + ), + coverage_engine=object(), + terminal_gate_path=terminal_json, + gate_registry=_registry_with_calibration_and_parity_trio(), + parity_reference=_fixture_parity_reference(input_h5), + ) + + gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] + for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + assert gates[entry_id]["status"] == "passed", entry_id + assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + + +def test_armed_calibration_without_parity_reference_stays_evidence_absent( + tmp_path, +) -> None: + """No frozen instrument, no trio evidence — never a copied reference.""" + + pytest.importorskip("tables") + + input_h5 = tmp_path / "healthy.h5" + terminal_json = tmp_path / "terminal_gates.json" + _write_two_row_h5(input_h5, include_calibration_columns=True) + + _run_national_build( + input_h5=input_h5, + staging_h5=tmp_path / "staging.h5", + stages=( + Stage( + name="national_calibration", + transform=UKNationalCalibrationStage( + _calibration_registry(55.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), + ), + ), + ), + coverage_engine=object(), + terminal_gate_path=terminal_json, + gate_registry=_registry_with_calibration_and_parity_trio(), + ) + + gates = json.loads(terminal_json.read_text(encoding="utf-8"))["gates"] + for entry_id in ("uk_export_surface", "uk_target_surface", "uk_target_fit"): + assert gates[entry_id]["status"] == "evidence_absent", entry_id + assert gates[entry_id]["reason"] == "missing evidence: parity_evidence" + assert gates["uk_calibration_reference_coverage"]["status"] == "passed" + + def test_national_build_rejects_both_gate_path_names_and_h5_collisions( tmp_path, ) -> None: @@ -1467,7 +1598,7 @@ def test_atomic_writer_cleans_temporary_h5_after_write_failure( monkeypatch, tmp_path ) -> None: pytest.importorskip("tables") - from microcosm.build.uk_runtime import national_build + from microcosm.build.uk_runtime import national_frame input_h5 = tmp_path / "base.h5" staging_h5 = tmp_path / "staging.h5" @@ -1479,10 +1610,10 @@ def fail_store(path, *_args, **_kwargs): Path(path).write_bytes(b"partial") raise OSError("simulated HDF write failure") - monkeypatch.setattr(national_build.pd, "HDFStore", fail_store) + monkeypatch.setattr(national_frame.pd, "HDFStore", fail_store) with pytest.raises(OSError, match="simulated HDF write failure"): - national_build.write_uk_national_frame(frame, staging_h5) + national_frame.write_uk_national_frame(frame, staging_h5) assert staging_h5.read_bytes() == b"previous-good-artifact" assert list(tmp_path.glob(".staging.h5.*.tmp.h5")) == [] @@ -1529,15 +1660,19 @@ class _CountingCalibrationStage: def __init__(self, calls: list[str]) -> None: self.calls = calls self.inner = UKNationalCalibrationStage( - [_calibration_fact()], - references=[_uc_reference()], - epochs=1, + _calibration_registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) @property def manifest(self) -> dict[str, object] | None: return self.inner.manifest + @property + def diagnostics(self) -> tuple[dict[str, object], ...]: + return self.inner.diagnostics + def __call__(self, frame: Frame) -> Frame: self.calls.append("national_calibration") return self.inner(frame) diff --git a/packages/microcosm-build/tests/test_uk_national_build_driver.py b/packages/microcosm-build/tests/test_uk_national_build_driver.py index 54e39138eb..1f793c12ec 100644 --- a/packages/microcosm-build/tests/test_uk_national_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_national_build_driver.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import importlib.util import json import sys @@ -12,6 +13,7 @@ from microcosm.build.gate_battery import GateBatteryBlockedError from microcosm.build.logbook import LOGBOOK_ROW_FIELDS, load_spool_rows +from microcosm.build.logbook_adoption import role_pins_digest from microcosm.build.uk_runtime.national_frame import ( UKStagingProvenance, _uk_source_file_fingerprint, @@ -927,6 +929,40 @@ def test_national_driver_accepts_legacy_input_coverage_path_alias( assert args.logbook_prev_row_digest == "d" * 64 +def test_national_driver_accepts_staging_candidate_input_sha( + monkeypatch, + tmp_path, +) -> None: + builder = _load_builder_module() + declared_sha = "a" * 64 + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_national_dataset.py", + *_IDENTITY_CLI_ARGUMENTS, + "--input-h5", + "base.h5", + "--staging-h5", + "staging.h5", + "--staging-candidate-input-sha256", + declared_sha, + "--frs-raw-dir", + "frs_2023_24", + "--spi-tab", + "put2223uk.tab", + "--hmrc-ods", + "hmrc.ods", + "--cgt-ods", + "cgt.ods", + ], + ) + + args = builder._parse_args() + + assert args.staging_candidate_input_sha256 == declared_sha + + def test_national_driver_forwards_legacy_output_to_compatibility_serializer( monkeypatch, tmp_path, @@ -1283,6 +1319,60 @@ def test_national_driver_refuses_release_candidate_with_the_legacy_alias( assert "signed schema-4 report" in capsys.readouterr().err +def test_national_driver_refuses_release_candidate_with_staging_candidate_input( + monkeypatch, + capsys, +) -> None: + builder = _load_builder_module() + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_national_dataset.py", + *_IDENTITY_CLI_ARGUMENTS, + "--input-h5", + "base.h5", + "--staging-h5", + "staging.h5", + "--staging-candidate-input-sha256", + "a" * 64, + "--frs-raw-dir", + "frs_2023_24", + "--spi-tab", + "put2223uk.tab", + "--hmrc-ods", + "hmrc.ods", + "--cgt-ods", + "cgt.ods", + "--release-candidate", + ], + ) + + with pytest.raises(SystemExit): + builder._parse_args() + assert "non-certified spine" in capsys.readouterr().err + + +def test_staging_candidate_input_verifier_requires_declared_sha(tmp_path) -> None: + builder = _load_builder_module() + source = tmp_path / "synthetic-spine.h5" + source.write_bytes(b"synthetic") + digest = hashlib.sha256(b"synthetic").hexdigest() + + identity = builder.verify_staging_candidate_uk_input( + source, + expected_sha256=digest, + ) + + assert identity.tier == "staging_candidate" + assert identity.sha256 == digest + with pytest.raises(ValueError, match="does not match declared"): + builder.verify_staging_candidate_uk_input( + source, + expected_sha256="0" * 64, + ) + + def test_staging_run_config_pins_the_sampling_identity(monkeypatch, tmp_path) -> None: builder = _load_builder_module() for name in ("adult.tab", "benefits.tab", "put2223uk.tab", "hmrc.ods", "cgt.ods"): @@ -1495,3 +1585,44 @@ def test_invalid_logbook_predecessor_refuses_before_sidecar_cleanup( for sidecar in sidecars: assert sidecar.read_text() == '{"stale": true}\n' assert not (tmp_path / "logbook-spool").exists() + + +@pytest.mark.parametrize("feed_layout", ["file", "directory"]) +def test_ledger_facts_role_pin_is_digestible(tmp_path, feed_layout) -> None: + """The armed run's Ledger role pin must survive role_pins_digest. + + First-armed-run finding (2026-08-23): _source_pins stored the full Ledger + provenance block under the 'ledger_facts' role, but role_pins_digest + accepts exactly {sha256, size_bytes}, so every armed build raised + ValueError at input_pins_digest before its first stage. Neither PR branch + fired it alone because only --ledger-facts reaches this path. + """ + + builder = _load_builder_module() + facts_body = b'{"fact": 1}\n' + if feed_layout == "file": + feed_path = tmp_path / "consumer_facts.jsonl" + feed_path.write_bytes(facts_body) + else: + feed_path = tmp_path / "feed" + feed_path.mkdir() + (feed_path / "consumer_facts.jsonl").write_bytes(facts_body) + artifact = SimpleNamespace( + path=feed_path, + facts_sha256="a" * 64, + provenance=lambda: { + "facts_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + "profile": "uk-national", + }, + ) + + pin = builder._ledger_facts_pin(artifact) + + assert set(pin) == {"sha256", "size_bytes"} + assert pin["sha256"] == "a" * 64 + assert pin["size_bytes"] == len(facts_body) + digest = role_pins_digest({"ledger_facts": pin}) + assert len(digest) == 64 + with pytest.raises(ValueError): + role_pins_digest({"ledger_facts": dict(artifact.provenance())}) diff --git a/packages/microcosm-build/tests/test_uk_national_calibration.py b/packages/microcosm-build/tests/test_uk_national_calibration.py index f6eb79022d..bd547a8e13 100644 --- a/packages/microcosm-build/tests/test_uk_national_calibration.py +++ b/packages/microcosm-build/tests/test_uk_national_calibration.py @@ -10,10 +10,32 @@ import pytest from microcosm.build.ledger_targets import LedgerTargetReference +from microcosm.build.uk_runtime import ( + UK_NATIONAL_L0_LAMBDA, + UK_NATIONAL_LEARNING_RATE, + UK_NATIONAL_MASS_RULE, + UK_NATIONAL_MAX_WEIGHT_RATIO, + UK_NATIONAL_SEED, + UK_NATIONAL_SOLVE_DOCTRINE, + UK_NATIONAL_SOLVE_EPOCHS, + UK_NATIONAL_TARGET_LOSS_CAP, + UK_NATIONAL_TARGET_WEIGHT_RULE, + UKNationalSolveDoctrine, + uk_doctrine_with_overrides, + uk_national_target_loss_weights, +) +from microcosm.build.uk_runtime.ledger_targets import ( + UKLedgerTargetCompilation, + _uk_contract_targets, +) +from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_calibration import ( UKNationalCalibrationStage, + _post_solve_calibration_record, + national_calibration_mass_reason, ) from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame +from microcosm.calibrate import TargetRegistry, TargetSpec from microcosm.frame import EntitySchema, Frame, WeightKind, Weights ACTIVE_REFERENCE_COUNT = 408 @@ -59,6 +81,23 @@ def _fact( } +def _registry(*, value: float = 30.0) -> TargetRegistry: + return TargetRegistry( + [ + TargetSpec( + name="dwp.uc.households", + entity="benunit", + measure="dwp/uc/households", + value=value, + source="test", + family="dwp_universal_credit", + metadata={"contract_target_id": "dwp.uc.households"}, + ) + ], + country="uk", + ) + + def _frame() -> Frame: ids = np.arange(4, dtype="int64") return Frame( @@ -81,6 +120,71 @@ def _frame() -> Frame: ) +def _frame_without_uc_column() -> Frame: + frame = _frame() + return Frame( + { + "person": frame.table("person"), + "benunit": frame.table("benunit").drop(columns=["universal_credit"]), + "household": frame.table("household"), + }, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +class StubMeasureResolver: + contract_targets = { + "dwp.uc.households": { + "bindings": { + "policyengine": { + "from_entity": "benunit", + "value_variable": "universal_credit", + } + } + } + } + + def __init__(self): + self.calls = [] + + def knows(self, entity, variable): + return (entity, variable) == ("benunit", "universal_credit") + + def compute(self, entity, variable): + self.calls.append((entity, variable)) + return np.array([1.0, 1.0, 0.0, 0.0]), "stub_uc" + + def receipt(self): + return {"provider": "stub_uc"} + + +class StubCrosstabResolver: + """Supplies the household-grain affected flag, as production does. + + ``uc_is_child_limit_affected`` is person-native in policyengine-uk, and + the Frame's global column-uniqueness rule forbids the same name on two + entity tables — so the household-grain flag can only ever arrive as a + table-scoped adapter injection. That is the production route, and it is + the route exercised here. + """ + + contract_targets = None + + def knows(self, entity, variable): + return (entity, variable) == ("household", "uc_is_child_limit_affected") + + def compute(self, entity, variable): + assert (entity, variable) == ("household", "uc_is_child_limit_affected") + return np.array([1.0, 0.0, 1.0]), "stub_any_collapse_person_to_household" + + def receipt(self): + return {"provider": "stub_crosstab_flag"} + + def _nested_frame() -> Frame: return Frame( { @@ -97,9 +201,7 @@ def _nested_frame() -> Frame: "universal_credit": [1.0, 0.0, 1.0, 1.0], } ), - "household": pd.DataFrame( - {"household_id": np.arange(3, dtype="int64")} - ), + "household": pd.DataFrame({"household_id": np.arange(3, dtype="int64")}), }, EntitySchema(group_entities=("benunit", "household")), {"household": Weights(np.array([10.0, 20.0, 30.0]), WeightKind.DESIGN)}, @@ -152,19 +254,19 @@ def _materialization_binding_frame( "household_id": np.arange(3, dtype="int64"), "esa_income": [10.0, 20.0, 0.0], "esa_contrib": [1.0, 2.0, 0.0], - "uc_is_child_limit_affected": [1.0, 0.0, 1.0], - "children_count": [2.0, 1.0, 3.0], } ) salary_sacrifice_metric = "hmrc/salary_sacrifice_it_relief_basic_rate" person_columns = { - "person_id": np.arange(4, dtype="int64"), - "person_benunit_id": [0, 0, 1, 2], - "person_household_id": [0, 0, 1, 2], - "capital_gains": [0.0, 7_000.0, 12_000.0, 500.0], + "person_id": np.arange(6, dtype="int64"), + "person_benunit_id": [0, 0, 1, 2, 2, 2], + "person_household_id": [0, 0, 1, 2, 2, 2], + "capital_gains": [0.0, 7_000.0, 12_000.0, 500.0, 0.0, 0.0], + # Two flagged children in household 0, none in 1, three in 2. + "uc_is_child_limit_affected": [True, True, False, True, True, True], } if include_counterfactual_delta: - person_columns[salary_sacrifice_metric] = [1.0, 2.0, 0.0, 0.0] + person_columns[salary_sacrifice_metric] = [1.0, 2.0, 0.0, 0.0, 0.0, 0.0] return Frame( { "person": pd.DataFrame(person_columns), @@ -185,7 +287,9 @@ def _materialization_binding_frame( def test_uc_calibration_compiles_and_moves_weighted_count_towards_fact() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=200, learning_rate=0.05 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=200, learning_rate=0.05), ) result = stage(frame) @@ -197,12 +301,27 @@ def test_uc_calibration_compiles_and_moves_weighted_count_towards_fact() -> None assert stage.manifest["resolved_reference_count"] == 1 assert stage.manifest["matrix_target_count"] == 1 assert stage.diagnostics[0]["target"] == 30.0 + assert result.weights_for("household").kind is WeightKind.CALIBRATED + assert len(result.mass_log) == 1 + mass_change = stage.manifest["weights"]["calibration_mass_change"] + assert mass_change["entity"] == "household" + assert "National doctrine calibration" in mass_change["reason"] + assert stage.manifest["weights"]["household_weight_kind_chain"] == [ + {"stage": "staging", "kind": "design"}, + {"stage": "national_calibration", "kind": "calibrated"}, + ] + assert stage.manifest["weights"]["mass_log_records_before_calibration"] == 0 + assert stage.manifest["weights"]["mass_log_records"] == 1 + assert stage.manifest["solve"]["n_targets"] == 1 + assert stage.manifest["solve"]["n_households"] == 4 def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() -> None: frame = _nested_frame() stage = UKNationalCalibrationStage( - [_fact(value=60.0)], references=[_uc_reference()], epochs=5 + _registry(value=60.0), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) result = stage(frame) @@ -215,12 +334,59 @@ def test_uc_calibration_stage_accepts_benunit_grain_reference_on_nested_frame() validate_uk_national_frame(result) -def test_activated_unresolvable_reference_aborts_loudly() -> None: +def test_stage_measure_resolver_injects_columns_then_restores_pristine_output() -> None: + frame = _frame_without_uc_column() + resolver = StubMeasureResolver() + original_columns = {entity: set(frame.table(entity).columns) for entity in frame.entities} + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + measure_resolver=resolver, + ) + + result = stage(frame) + + assert resolver.calls == [("benunit", "universal_credit")] + assert stage.manifest["measure_resolution"]["provider"] == {"provider": "stub_uc"} + assert stage.manifest["measure_resolution"]["attached"] == { + "benunit.universal_credit": "stub_uc" + } + for entity in frame.entities: + assert set(result.table(entity).columns) == original_columns[entity] + assert "universal_credit" not in result.table("benunit") + validate_uk_national_frame(result) + + +def test_stage_manifest_omits_measure_resolution_without_resolver() -> None: + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + ) + + stage(_frame()) + + assert "measure_resolution" not in stage.manifest + + +def test_activated_unresolvable_compiled_reference_aborts_loudly() -> None: stage = UKNationalCalibrationStage( - [_fact(concept="different")], references=[_uc_reference()], epochs=1 + UKLedgerTargetCompilation( + registry=TargetRegistry([], country="uk"), + unsupported=( + { + "name": "dwp.uc.households", + "period": "2025", + "reason": "did not match a Ledger fact selector", + }, + ), + ), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) - with pytest.raises(ValueError, match="did not match a Ledger fact selector"): + with pytest.raises(RuntimeError, match="did not match a Ledger fact selector"): stage(_frame()) @@ -284,40 +450,79 @@ def test_packaged_binding_classes_materialize_through_national_stage() -> None: strict=True, ) ] + from microcosm.build.ledger_targets import compile_ledger_target_references + from microcosm.build.uk_runtime.ledger_targets import ( + UKFrameTargetAdapter, + materialize_uk_ledger_targets, + ) + + registry = compile_ledger_target_references(facts, references, country="uk") + resolver = StubCrosstabResolver() + resolver.contract_targets = _uk_contract_targets() stage = UKNationalCalibrationStage( - facts, - references=references, - epochs=1, - learning_rate=0.01, + registry, + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1, learning_rate=0.01), + measure_resolver=resolver, ) - result = stage(_materialization_binding_frame()) + input_frame = _materialization_binding_frame() + original_columns = { + entity: set(input_frame.table(entity).columns) + for entity in input_frame.entities + } + + result = stage(input_frame) assert stage.manifest["activated_reference_count"] == len(selected_names) assert stage.manifest["resolved_reference_count"] == len(selected_names) assert stage.manifest["matrix_target_count"] == len(selected_names) + # The staged frame stays writer-clean: no prepared scratch column survives + # onto the returned tables (adjudicated lifecycle; slash-named scratch + # crashes the HDFStore staging writer). Original input columns — including + # the fixture's precomputed counterfactual delta — are exactly preserved. + for entity in input_frame.entities: + assert set(result.table(entity).columns) == original_columns[entity] + # The binding classes produce the right prepared values on the adapter… + adapter = UKFrameTargetAdapter(_materialization_binding_frame()) + # The same table-scoped injection the resolution loop performs. + adapter.tables["household"]["uc_is_child_limit_affected"] = np.array( + [1.0, 0.0, 1.0] + ) + materialize_uk_ledger_targets(adapter, registry, period=2025) materialized = { ("benunit", "dwp/uc/households"): [1.0, 0.0, 1.0], ("household", "obr/esa"): [11.0, 22.0, 0.0], - ("person", "hmrc/cgt_taxpayers"): [0.0, 1.0, 1.0, 0.0], + ("person", "hmrc/cgt_taxpayers"): [0.0, 1.0, 1.0, 0.0, 0.0, 0.0], + # Counts of flagged children, reduced from the person rows — not the + # household indicator [1.0, 0.0, 1.0] a same-grain boolean read gives. ("household", "dwp/uc/two_child_limit/children_affected"): [2.0, 0.0, 3.0], ("person", "hmrc/salary_sacrifice_it_relief_basic_rate"): [ 1.0, 2.0, 0.0, 0.0, + 0.0, + 0.0, ], } for (entity, measure), expected in materialized.items(): - assert result.table(entity)[measure].tolist() == expected + assert adapter.tables[entity][measure].tolist() == expected def test_packaged_materialization_skip_aborts_national_stage() -> None: + from microcosm.build.ledger_targets import compile_ledger_target_references + reference = _reference_by_name("hmrc.salary_sacrifice.it_relief_basic_rate") - stage = UKNationalCalibrationStage( + registry = compile_ledger_target_references( [_fact_for_reference(reference, 3.0)], - references=[reference], - epochs=1, + [reference], + country="uk", + ) + stage = UKNationalCalibrationStage( + registry, + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=1), ) with pytest.raises(RuntimeError, match="could not materialize every"): @@ -327,7 +532,9 @@ def test_packaged_materialization_skip_aborts_national_stage() -> None: def test_calibration_preserves_entity_ids_and_national_integrity() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) result = stage(frame) @@ -341,14 +548,18 @@ def test_calibration_preserves_entity_ids_and_national_integrity() -> None: def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: frame = _frame() stage = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) staged = stage(frame) metadata = json.loads(json.dumps(stage.checkpoint_metadata())) resumed = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) resumed.resume_from_checkpoint(metadata, staged) @@ -357,13 +568,17 @@ def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: assert resumed.output_content_identity == metadata["output_content_identity"] drifted = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="drifted record"): drifted.resume_from_checkpoint(metadata, frame) empty = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="calibration counts"): empty.resume_from_checkpoint({}, staged) @@ -378,7 +593,271 @@ def test_checkpoint_metadata_round_trips_calibration_evidence() -> None: empty.resume_from_checkpoint(missing_count, staged) unrun = UKNationalCalibrationStage( - [_fact()], references=[_uc_reference()], epochs=5 + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), ) with pytest.raises(RuntimeError, match="has not run"): unrun.checkpoint_metadata() + + +def test_prepared_slash_columns_are_not_returned_to_the_writer(tmp_path) -> None: + pytest.importorskip("tables") + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5), + ) + + result = stage(_frame()) + + assert "dwp/uc/households" not in result.table("benunit") + write_uk_national_frame(result, tmp_path / "staging.h5") + + +def test_national_calibration_mass_reason_is_canonical() -> None: + assert national_calibration_mass_reason( + ["dwp_universal_credit", "hmrc", "dwp_universal_credit"] + ) == ( + "National doctrine calibration to bound target family(ies) " + "dwp_universal_credit, hmrc; total household mass moved with the targets." + ) + with pytest.raises(ValueError, match="bound_families"): + national_calibration_mass_reason([]) + + +def test_post_solve_fence_requires_calibrated_kind_and_mass_record() -> None: + before = _frame() + uncalibrated = Frame( + {entity: before.table(entity) for entity in before.entities}, + before.schema, + {"household": Weights(np.full(4, 10.0), WeightKind.DESIGN)}, + before.strata, + mass_log=before.mass_log, + metadata=before.metadata, + ) + + with pytest.raises(RuntimeError, match="not 'calibrated'"): + _post_solve_calibration_record(before, uncalibrated, before_count=0) + + calibrated_without_record = Frame( + {entity: before.table(entity) for entity in before.entities}, + before.schema, + {"household": Weights(np.full(4, 10.0), WeightKind.CALIBRATED)}, + before.strata, + mass_log=before.mass_log, + metadata=before.metadata, + ) + with pytest.raises(RuntimeError, match="exactly one mass record"): + _post_solve_calibration_record( + before, + calibrated_without_record, + before_count=0, + ) + + +def test_national_doctrine_constants_are_the_declared_contract() -> None: + assert UK_NATIONAL_SOLVE_EPOCHS == 256 + assert UK_NATIONAL_LEARNING_RATE == 0.02 + assert UK_NATIONAL_MAX_WEIGHT_RATIO == 10.0 + assert UK_NATIONAL_SEED == 0 + assert UK_NATIONAL_TARGET_LOSS_CAP == 10.0 + assert UK_NATIONAL_L0_LAMBDA == 0.0 + assert UK_NATIONAL_MASS_RULE == "free" + # María's ruling (2026-08-24): family_equal is vocabulary, never the + # default — she passes it as an explicit per-run override. + assert UK_NATIONAL_TARGET_WEIGHT_RULE == "uniform" + assert UK_NATIONAL_SOLVE_DOCTRINE == UKNationalSolveDoctrine() + assert UK_NATIONAL_SOLVE_DOCTRINE.scale_rule == "default_target_loss_scales" + assert UK_NATIONAL_SOLVE_DOCTRINE.target_weight_rule == "uniform" + assert UKNationalSolveDoctrine(target_weight_rule="family_equal") + + +def test_uk_doctrine_with_overrides_receipts_effective_diffs_only() -> None: + doctrine, receipt = uk_doctrine_with_overrides() + assert doctrine == UK_NATIONAL_SOLVE_DOCTRINE + assert receipt == {} + + doctrine, receipt = uk_doctrine_with_overrides( + epochs=UK_NATIONAL_SOLVE_EPOCHS, + target_weight_rule=UK_NATIONAL_TARGET_WEIGHT_RULE, + ) + assert doctrine == UK_NATIONAL_SOLVE_DOCTRINE + assert receipt == {} + + doctrine, receipt = uk_doctrine_with_overrides( + epochs=128, + learning_rate=0.01, + target_weight_rule="family_equal", + target_loss_cap=5.0, + ) + assert doctrine.epochs == 128 + assert doctrine.learning_rate == 0.01 + assert doctrine.target_weight_rule == "family_equal" + assert doctrine.target_loss_cap == 5.0 + assert receipt == { + "epochs": {"default": 256, "effective": 128}, + "learning_rate": {"default": 0.02, "effective": 0.01}, + "target_loss_cap": {"default": 10.0, "effective": 5.0}, + "target_weight_rule": {"default": "uniform", "effective": "family_equal"}, + } + + +def test_uk_doctrine_with_overrides_refuses_invalid_or_frozen_fields() -> None: + with pytest.raises(ValueError, match="target_weight_rule"): + uk_doctrine_with_overrides(target_weight_rule="per_target") + with pytest.raises(ValueError, match="epochs"): + uk_doctrine_with_overrides(epochs=0) + with pytest.raises(ValueError, match="reviewed constants"): + uk_doctrine_with_overrides(seed=1) + with pytest.raises(ValueError, match="unknown"): + uk_doctrine_with_overrides(not_a_field=1) + + +def test_family_equal_gives_each_family_one_equal_share() -> None: + weights = uk_national_target_loss_weights( + ["hmrc"] * 3 + ["obr"], rule="family_equal" + ) + assert weights is not None + assert weights.sum() == pytest.approx(1.0) + # Three hmrc rows share half the objective; the single obr row holds the + # other half, so an over-supplied family cannot outvote by count. + assert weights[:3].sum() == pytest.approx(0.5) + assert weights[3] == pytest.approx(0.5) + + +def test_uniform_rule_defers_to_the_kernel_default() -> None: + assert uk_national_target_loss_weights(["hmrc", "obr"], rule="uniform") is None + + +def test_family_equal_refuses_an_undeclared_family() -> None: + with pytest.raises(ValueError, match="must declare a family"): + uk_national_target_loss_weights(["hmrc", ""], rule="family_equal") + + +def test_national_doctrine_rejects_tampered_bounds() -> None: + with pytest.raises(ValueError, match="epochs"): + UKNationalSolveDoctrine(epochs=0) + with pytest.raises(ValueError, match="learning_rate"): + UKNationalSolveDoctrine(learning_rate=0.0) + with pytest.raises(ValueError, match="max_weight_ratio"): + UKNationalSolveDoctrine(max_weight_ratio=1.0) + with pytest.raises(ValueError, match="seed"): + UKNationalSolveDoctrine(seed=-1) + with pytest.raises(ValueError, match="target_loss_cap"): + UKNationalSolveDoctrine(target_loss_cap=float("nan")) + with pytest.raises(ValueError, match="scale_rule"): + UKNationalSolveDoctrine(scale_rule="bespoke") + with pytest.raises(ValueError, match="target_weight_rule"): + UKNationalSolveDoctrine(target_weight_rule="per_target") + with pytest.raises(ValueError, match="mass_rule"): + UKNationalSolveDoctrine(mass_rule="conserve") + with pytest.raises(ValueError, match="l0_lambda"): + UKNationalSolveDoctrine(l0_lambda=-0.1) + + +@pytest.mark.parametrize( + ("rule", "expected"), + [("uniform", None), ("family_equal", [1.0])], +) +def test_doctrine_target_weight_rule_reaches_the_solver( + monkeypatch, rule, expected +) -> None: + """The declared rule must reach calibrate(), not just the manifest echo. + + First-armed-run finding (2026-08-23): the doctrine declared a + target_weight_rule the stage never passed to the kernel, so the solve + silently ran uniform whatever the doctrine said. The doctrine vector now + travels explicitly; "uniform" maps to None — the kernel's own default — + so the shipped identity is unchanged under the default rule. + """ + + from microcosm.calibrate import calibrate as real_calibrate + + captured: dict[str, object] = {} + + def capturing_calibrate(*args, **kwargs): + captured["target_loss_weights"] = kwargs["target_loss_weights"] + return real_calibrate(*args, **kwargs) + + monkeypatch.setattr( + "microcosm.build.uk_runtime.national_calibration.calibrate", + capturing_calibrate, + ) + stage = UKNationalCalibrationStage( + _registry(), + period=2025, + doctrine=UKNationalSolveDoctrine(epochs=5, target_weight_rule=rule), + ) + + stage(_frame()) + + weights = captured["target_loss_weights"] + if expected is None: + assert weights is None + else: + assert weights is not None + assert weights.tolist() == expected + + +def test_measure_resolution_never_touches_the_source_frame() -> None: + """Injection lands on adapter table copies, not on the caller's frame. + + A test that inspects the frame after ``restore`` would pass either way, + so this one holds the property where it actually lives: the source frame + gains no column while the resolution loop probes round after round. + """ + + from microcosm.build.target_materialization import resolve_target_measures + from microcosm.build.uk_runtime.ledger_targets import UKFrameTargetAdapter + + frame = _materialization_binding_frame() + before = { + entity: list(frame.table(entity).columns) for entity in frame.entities + } + + class ProbeResolver: + contract_targets = { + "probe.target": { + "bindings": { + "policyengine": { + "from_entity": "household", + "value_variable": "probe_measure", + } + } + } + } + + def knows(self, entity, variable): + return (entity, variable) == ("household", "probe_measure") + + def compute(self, entity, variable): + return np.array([1.0, 2.0, 3.0]), "probe" + + def receipt(self): + return {"provider": "probe"} + + registry = TargetRegistry( + [ + TargetSpec( + name="probe.target", + entity="household", + measure="probe/measure", + value=6.0, + source="test", + metadata={"contract_target_id": "probe.target"}, + ) + ], + country="uk", + ) + + resolution = resolve_target_measures( + lambda: UKFrameTargetAdapter(frame), + registry, + ProbeResolver(), + period=2025, + ) + + assert ("household", "probe_measure") in resolution.measure_inputs + for entity in frame.entities: + assert list(frame.table(entity).columns) == before[entity] diff --git a/packages/microcosm-build/tests/test_uk_national_targets.py b/packages/microcosm-build/tests/test_uk_national_targets.py index 1dd1c672c4..02ac5e428f 100644 --- a/packages/microcosm-build/tests/test_uk_national_targets.py +++ b/packages/microcosm-build/tests/test_uk_national_targets.py @@ -47,6 +47,7 @@ POLICYENGINE_BINDING_KEYS = { "affected_flag_variable", "band", + "band_filter_dimension", "band_period_factor", "count_of", "filters", @@ -58,6 +59,7 @@ "groupby_variable", "household_conditions", "kind", + "value_reduction", "map_to", "metric_name", "notes", @@ -278,9 +280,26 @@ def test_uk_national_targets_declare_chronicle_loader_guarantees(): "gate_comparison", } <= set(binding), target["target_id"] elif kind == "baseline_flag_crosstab": - assert {"affected_flag_variable", "count_of"} <= set(binding), target[ - "target_id" - ] + assert "affected_flag_variable" in binding, target["target_id"] + # The two-child-limit rebinding counts a real value_variable per + # record; count_of survives as the legacy spelling the provider + # still accepts. Either key must name the counted column. + assert {"value_variable", "count_of"} & set(binding), target["target_id"] + + reduction = binding.get("value_reduction") + if reduction is not None: + # A count over a member-level variable declares its reduction, so + # the materializer sums members instead of reading the variable at + # the target's own grain — a boolean read there would collapse to + # a household indicator and be published against a count. + assert set(reduction) == {"variable", "entity", "reduce"}, ( + target["target_id"] + ) + assert reduction["reduce"] in PREDICATE_REDUCERS, target["target_id"] + assert reduction["entity"] in PREDICATE_ENTITIES, target["target_id"] + assert reduction["variable"] == binding.get("value_variable"), ( + target["target_id"] + ) if "reduce" in binding: assert binding["reduce"] in BINDING_REDUCERS, target["target_id"] diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index a7c83e7428..117a9d6b0c 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -346,9 +346,9 @@ def test_write_is_atomic_when_metadata_write_fails( monkeypatch: pytest.MonkeyPatch, tmp_path ) -> None: pytest.importorskip("tables") - # The rowwise writer routes through the national seam's shared physical - # writer, so the metadata-failure seam now lives there. - from microcosm.build.uk_runtime import national_build as module + # The rowwise writer routes through the national frame seam's shared + # physical writer, so the metadata-failure seam lives there. + from microcosm.build.uk_runtime import national_frame as module result = clone_uk_dataset_tables_with_rowwise_geography( person=person_frame(), diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index ad3ec35066..379d4eda47 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -376,10 +376,10 @@ "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" + "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" + "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index a418a586cf..065832b827 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -151,10 +151,10 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "623f340ddde6f705717c3a6306522f8cf46c1c17a067f9e89df190ecc690f0fc" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "f2cc2af4ae41b3e84e0d7a6e8a1a4480fd0aa5f034b9e8eeefe8e0c2bc40d239" + "4f66eea7e593b795da93217e9ac8b3b53ca1f82375ec346b3a2ecbb558b89cb6" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "3601b4c77b5672205e14435d4833412f55a6b4bbabd5bbd2b6e33b29a63bf74e" + "26fdbcccfaa01e9afa116339cfaa870f76fc6ed3bd5db57fec498781ff3efc64" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" diff --git a/tools/build_uk_national_dataset.py b/tools/build_uk_national_dataset.py index 31d1959597..a600bf05dd 100644 --- a/tools/build_uk_national_dataset.py +++ b/tools/build_uk_national_dataset.py @@ -10,6 +10,7 @@ import time import uuid from datetime import UTC, datetime +from importlib import resources as importlib_resources from itertools import combinations from pathlib import Path @@ -38,6 +39,7 @@ from microcosm.build.uk_runtime.cgt_imputation import ( uk_capital_gains_imputation_stage, ) +from microcosm.build.uk_runtime.diagnostics import write_uk_calibration_diagnostics from microcosm.build.uk_runtime.frs_hmrc_leaves import ( UKFRSHMRCRetainedLeavesStageTransform, ) @@ -46,6 +48,7 @@ from microcosm.build.uk_runtime.hmrc_restoration import ( UKHMRCIncomeStageTransform, verify_certified_uk_candidate, + verify_staging_candidate_uk_input, ) from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry from microcosm.build.uk_runtime.national_build import build_uk_national_dataset @@ -60,6 +63,9 @@ UK_SAMPLE_RUNG_TOKENS, UK_SAMPLE_SEED_DEFAULT, ) +from microcosm.build.uk_runtime.parity_reference import ( + load_efrs_parity_reference, +) from microcosm.build.uk_runtime.release_identity import UK_RELEASE_TIERS from microcosm.build.uk_runtime.source_runtime import uk_stage_implementations from microcosm.build.uk_runtime.terminal_gates import ( @@ -131,6 +137,14 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Compact UK single-year H5 supplying the national base tables.", ) + parser.add_argument( + "--staging-candidate-input-sha256", + type=sha256_argument, + help=( + "Declare --input-h5 as a non-certified staging-candidate spine and " + "bind it to this SHA-256. Refused with --release-candidate." + ), + ) parser.add_argument( "--staging-h5", type=Path, @@ -335,6 +349,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "schema-1 alias is last-written over the report path and a " "candidate must keep its signed schema-4 report." ) + if args.release_candidate and args.staging_candidate_input_sha256 is not None: + parser.error( + "--release-candidate requires the certified input posture; " + "--staging-candidate-input-sha256 declares a non-certified spine." + ) if args.sample_seed < 0: parser.error("sample seed must be a non-negative integer.") if args.sample_fraction != 1.0 and ( @@ -430,10 +449,40 @@ def _source_pins( "cgt_ods": _artifact_pin(cgt_ods_path), } if ledger_artifact is not None: - pins["ledger_facts"] = dict(ledger_artifact.provenance()) + pins["ledger_facts"] = _ledger_facts_pin(ledger_artifact) return pins +def _ledger_facts_pin(ledger_artifact: object) -> dict[str, object]: + """Pin the consumer feed by content and size. + + Logbook role pins are exactly ``sha256`` and ``size_bytes``; the richer + Ledger identity block travels separately in ``safe_artifacts`` and + ``source_vintages``. The feed digest is already verified against the + manifest and the CLI pin at load, so it is reused rather than recomputed + over a multi-hundred-megabyte file. + """ + + path = Path(ledger_artifact.path) + facts_path = path / "consumer_facts.jsonl" if path.is_dir() else path + return { + "sha256": str(ledger_artifact.facts_sha256), + "size_bytes": int(facts_path.stat().st_size), + } + + +def _input_posture(candidate: object) -> dict[str, object]: + tier = str(getattr(candidate, "tier", "frs")) + return { + "posture": "staging_candidate" if tier == "staging_candidate" else "certified", + "filename": str(getattr(candidate, "filename", "")), + "tier": tier, + "revision": str(getattr(candidate, "revision", "")), + "sha256": str(candidate.sha256), + "size_bytes": int(candidate.size_bytes), + } + + def _gate_verdicts_from_report( report: dict[str, object], *, @@ -786,7 +835,13 @@ def _main_recording( for period in (2023, 2025) } ) - candidate = verify_certified_uk_candidate(args.input_h5) + if args.staging_candidate_input_sha256 is None: + candidate = verify_certified_uk_candidate(args.input_h5) + else: + candidate = verify_staging_candidate_uk_input( + args.input_h5, + expected_sha256=args.staging_candidate_input_sha256, + ) evidence_path.unlink(missing_ok=True) replay_path.unlink(missing_ok=True) build_record_path.unlink(missing_ok=True) @@ -836,11 +891,12 @@ def _main_recording( append_phase(state, "configured") append_phase(state, "candidate_verified") append_phase(state, "inputs_pinned") - # This staging path performs no calibration and therefore has no real - # target-surface or target-fit evidence; the schema-4 battery records - # the missing evidence explicitly. Input-mass evidence joins only when - # the caller supplies the licensed frozen reference sidecar; QRF-tail is - # spec-armed and runs whenever the frame evidence is present. + # Without Ledger facts, this staging path has no real target-surface or + # target-fit evidence; the schema-4 battery records the missing evidence + # explicitly. Armed calibration builds add target evidence from the solve. + # Input-mass evidence joins only when the caller supplies the licensed + # frozen reference sidecar; QRF-tail is spec-armed and runs whenever the + # frame evidence is present. gate_path_argument = ( {"input_coverage_path": legacy_input_coverage_path} if legacy_input_coverage_path is not None @@ -865,7 +921,12 @@ def _main_recording( calibration_stages: tuple[PlanStage, ...] = () if args.ledger_facts is not None: assert ledger_artifact is not None # resolved and pin-checked above - calibration_transform = UKNationalCalibrationStage(ledger_artifact.facts) + assert ledger_compilations is not None + calibration_year = load_uk_frs_release().calibration_year + calibration_transform = UKNationalCalibrationStage( + ledger_compilations[calibration_year], + period=calibration_year, + ) calibration_stages = ( PlanStage( name="national_calibration", @@ -910,14 +971,35 @@ def _main_recording( for period, compilation in ledger_compilations.items() } ), + # The frozen instrument supplies the parity trio's reference side; + # the candidate side comes from the staged frame and the solve. + parity_reference=( + load_efrs_parity_reference() if calibration_transform is not None else None + ), ) if calibration_transform is not None: - _write_json( + if calibration_transform.solve_result is None: + raise RuntimeError( + "national calibration diagnostics require the intact solve " + "result; checkpoint-resumed runs must rerun calibration before " + "writing calibration_diagnostics.json." + ) + assert ledger_artifact is not None + write_uk_calibration_diagnostics( + calibration_transform.solve_result, args.national_calibration_diagnostics_json, - { - "schema_version": 1, - "targets": list(calibration_transform.diagnostics), - "calibration": calibration_transform.manifest, + result.frame, + target_geography_levels=_uk_target_geography_levels( + calibration_transform.registry + ), + target_registry=calibration_transform.registry, + build={ + "build_id": state.build_id, + "ledger_facts": ledger_artifact.provenance(), + "code_pin": attempt_context["code_pin"], + "source_pins": source_pins, + "input_posture": _input_posture(candidate), + "score_vs_enhanced_frs": None, }, ) append_phase(state, "build_completed") @@ -959,6 +1041,7 @@ def _main_recording( ledger_artifact_provenance=( None if ledger_artifact is None else ledger_artifact.provenance() ), + input_posture=_input_posture(candidate), ) _write_json(build_record_path, build_record) append_phase(state, "build_record_written") @@ -1053,6 +1136,7 @@ def _staging_run_config( "rung_token": UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], }, "certified_candidate": dict(pins["certified_candidate"]), + "input_posture": _input_posture(candidate), "sources": { "adult_tab": dict(pins["adult_tab"]), "benefits_tab": dict(pins["benefits_tab"]), @@ -1100,6 +1184,7 @@ def _aggregate_build_record( sample_seed: int = UK_SAMPLE_SEED_DEFAULT, degenerate_exclusions_override: bool = False, ledger_artifact_provenance: dict[str, object] | None = None, + input_posture: dict[str, object] | None = None, ) -> dict[str, object]: """Return commit-safe aggregate evidence for one successful staging build.""" @@ -1132,9 +1217,9 @@ def _aggregate_build_record( ] release_evidence = dict(result.gate_report["release_evidence"]) source_vintages = dict(family_evidence.get("source_vintages", {})) + source_vintages["frs"] = load_uk_frs_release().vintage if ledger_artifact_provenance is not None: source_vintages["ledger_facts"] = ledger_artifact_provenance - source_vintages["frs"] = load_uk_frs_release().vintage return { "schema_version": 3, "build_kind": "uk_national_staging_dataset", @@ -1157,6 +1242,7 @@ def _aggregate_build_record( # honestly disagree, which is why they carry distinct names. "degenerate_exclusions_override_supplied": (degenerate_exclusions_override), }, + "input_posture": dict(input_posture or {}), "sampling": ( None if result.sampling_receipt is None else dict(result.sampling_receipt) ), @@ -1348,6 +1434,31 @@ def _validate_distinct_paths( ) +def _uk_target_geography_levels(registry) -> dict[str, str]: + payload = json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath("uk_national_targets.json") + .read_text(encoding="utf-8") + ) + targets = {row["target_id"]: row for row in payload["targets"]} + levels: dict[str, str] = {} + for spec in registry.specs: + target_id = spec.metadata.get("contract_target_id") + target = targets.get(str(target_id)) + if target is None: + raise ValueError( + f"UK calibration target {spec.name!r} references unknown " + f"contract target {target_id!r}." + ) + geography_levels = tuple(target.get("geography_levels") or ()) + if not geography_levels: + raise ValueError( + f"UK calibration target {spec.name!r} has no geography level." + ) + levels[spec.to_target().row_name] = str(geography_levels[0]) + return levels + + def _paths_alias(left: Path, right: Path) -> bool: """Conservatively identify aliases before any build path is unlinked.""" diff --git a/tools/calibrate_uk_national_dataset.py b/tools/calibrate_uk_national_dataset.py new file mode 100644 index 0000000000..815c8f3f25 --- /dev/null +++ b/tools/calibrate_uk_national_dataset.py @@ -0,0 +1,266 @@ +"""Calibrate an existing UK national spine H5 without replaying source stages. + +This driver is intentionally thin: it verifies pinned inputs, compiles the +Ledger target registry, applies the reviewed measure-exclusion register, records +any explicit doctrine overrides, and delegates the calibration/gate/logbook +work to :func:`microcosm.build.uk_runtime.calibration_run.run_uk_calibration`. + +Signed deviation for v1: no sampling rungs and no checkpointing. This seam runs +full-scale only; scale ladders and resumable source-stage checkpoints belong to +the spine build lane. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from itertools import combinations +from pathlib import Path +from typing import Any + +from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.uk_runtime.calibration_run import ( + UKCalibrationRunPaths, + run_uk_calibration, +) +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.ledger_targets import compile_uk_target_registry +from microcosm.build.uk_runtime.measure_simulation import ( + UKMeasureResolver, + apply_uk_calibration_measure_exclusions, + load_uk_calibration_measure_exclusions, +) +from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides +from microcosm.calibrate import TargetRegistry + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_CANONICAL_UK_RELEASE_ID = re.compile(r"populace-uk-[1-9][0-9]*-[a-z0-9_]+-k[1-9][0-9]*") +_UK_JUNE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + artifact = load_ledger_consumer_artifact( + args.ledger_facts, + expected_facts_sha256=args.ledger_facts_sha256, + expected_manifest_sha256=args.ledger_manifest_sha256, + ) + calibration_year = load_uk_frs_release().calibration_year + compilation = compile_uk_target_registry( + artifact.facts, target_period=calibration_year + ) + if compilation.unsupported: + raise SystemExit( + f"{len(compilation.unsupported)} target references failed to compile" + ) + _compare_frozen_register(args.register_json, compilation.registry) + exclusions = load_uk_calibration_measure_exclusions(args.measure_exclusions) + registry, exclusion_receipt = apply_uk_calibration_measure_exclusions( + compilation.registry, exclusions + ) + overrides = { + key: value + for key, value in { + "epochs": args.epochs, + "target_weight_rule": args.target_weight_rule, + "learning_rate": args.learning_rate, + "target_loss_cap": args.target_loss_cap, + }.items() + if value is not None + } + doctrine, doctrine_overrides = uk_doctrine_with_overrides(**overrides) + paths = UKCalibrationRunPaths( + input_h5=args.input_h5, + staging_h5=args.staging_h5, + diagnostics_json=args.diagnostics_json, + build_record_json=args.build_record_json, + terminal_gate_json=args.terminal_gate_json, + ) + # The resolver reads the input H5 to build its simulation, so the pin is + # verified here first — no bytes are consumed before they match the CLI sha. + measured_input_sha = _sha256_file(args.input_h5) + if measured_input_sha != args.input_sha256: + raise SystemExit( + "error: --input-h5 sha mismatch: " + f"measured {measured_input_sha}, pinned {args.input_sha256}" + ) + resolver = UKMeasureResolver( + simulation_source=args.input_h5, + scratch_dir=args.staging_h5.parent, + year=calibration_year, + frame=None, + ) + result = run_uk_calibration( + paths=paths, + input_sha256=args.input_sha256, + ledger_artifact=artifact, + register_registry=registry, + calibration_year=calibration_year, + exclusion_receipt=exclusion_receipt, + doctrine=doctrine, + doctrine_overrides=doctrine_overrides, + measure_resolver=resolver, + source_pins={ + "input_h5": { + "sha256": args.input_sha256, + "size_bytes": args.input_h5.stat().st_size, + }, + "ledger_facts": _ledger_facts_pin(artifact), + }, + run_config_extra={"calibration_year": calibration_year}, + release_candidate=args.release_candidate, + release_id=args.release_id, + logbook_prev_row_digest=args.logbook_prev_row_digest, + ) + summary = { + "staging_h5_sha256": result.staging_sha256, + "diagnostics_sha256": result.diagnostics_sha256, + "terminal_gate_sha256": result.terminal_gate_sha256, + "build_record_sha256": result.build_record_sha256, + "gate_verdicts": result.build_record["gate_summary"], + } + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-h5", required=True, type=Path) + parser.add_argument("--input-sha256", required=True, type=_sha256) + parser.add_argument("--ledger-facts", required=True, type=Path) + parser.add_argument("--ledger-facts-sha256", required=True, type=_sha256) + parser.add_argument("--ledger-manifest-sha256", required=True, type=_sha256) + parser.add_argument("--staging-h5", required=True, type=Path) + parser.add_argument("--diagnostics-json", required=True, type=Path) + parser.add_argument("--build-record-json", required=True, type=Path) + parser.add_argument("--release-id", required=True) + parser.add_argument("--terminal-gate-json", type=Path) + parser.add_argument("--register-json", type=Path) + parser.add_argument("--measure-exclusions", type=Path) + parser.add_argument("--release-candidate", action="store_true") + parser.add_argument("--logbook-prev-row-digest", type=_sha256) + parser.add_argument("--epochs", type=int) + parser.add_argument("--target-weight-rule") + parser.add_argument("--learning-rate", type=float) + parser.add_argument("--target-loss-cap", type=float) + args = parser.parse_args(argv) + args.terminal_gate_json = args.terminal_gate_json or args.staging_h5.with_suffix( + ".terminal_gates.json" + ) + override_flags = { + "--epochs": args.epochs, + "--target-weight-rule": args.target_weight_rule, + "--learning-rate": args.learning_rate, + "--target-loss-cap": args.target_loss_cap, + } + passed_overrides = [flag for flag, value in override_flags.items() if value is not None] + if args.release_candidate and passed_overrides: + parser.error( + "--release-candidate is refused with doctrine override flag(s): " + + ", ".join(passed_overrides) + ) + if args.release_candidate and args.measure_exclusions is not None: + # An exclusion register prunes the compiled target surface before the + # solve, and the calibration-scoped battery does not carry the + # target-surface gate — so an operator-supplied register would be an + # unreviewed narrowing of what a release candidate was measured + # against. Release candidates use the committed register only. + parser.error( + "--release-candidate is refused with --measure-exclusions: a " + "release candidate is measured against the committed target " + "surface, not an operator-supplied one" + ) + if ( + not args.release_candidate + and (_CANONICAL_UK_RELEASE_ID.fullmatch(args.release_id) or args.release_id == _UK_JUNE_RELEASE_ID) + ): + parser.error("canonical UK release ids require --release-candidate") + _validate_distinct_paths( + { + "--input-h5": args.input_h5, + "--ledger-facts": args.ledger_facts, + "--staging-h5": args.staging_h5, + "--diagnostics-json": args.diagnostics_json, + "--build-record-json": args.build_record_json, + "--terminal-gate-json": args.terminal_gate_json, + **({"--register-json": args.register_json} if args.register_json else {}), + **( + {"--measure-exclusions": args.measure_exclusions} + if args.measure_exclusions + else {} + ), + } + ) + return args + + +def _sha256(value: str) -> str: + if not _SHA256.fullmatch(value): + raise argparse.ArgumentTypeError("expected a lowercase SHA-256 digest") + return value + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _validate_distinct_paths(paths: dict[str, Path]) -> None: + resolved = {label: path.expanduser().resolve() for label, path in paths.items()} + for (left_label, left), (right_label, right) in combinations(resolved.items(), 2): + if _paths_alias(left, right): + raise SystemExit( + f"error: {left_label} and {right_label} must be distinct paths " + f"({left} aliases {right})" + ) + + +def _paths_alias(left: Path, right: Path) -> bool: + if str(left).casefold() == str(right).casefold(): + return True + try: + left_stat = left.stat() + right_stat = right.stat() + except FileNotFoundError: + return False + return (left_stat.st_dev, left_stat.st_ino) == (right_stat.st_dev, right_stat.st_ino) + + +def _compare_frozen_register(path: Path | None, registry: Any) -> None: + """Refuse when the compiled register is not the frozen scoring surface. + + Both sides are compared on ``TargetRegistry.version`` — the registry's own + content hash — through the validating loader, so the same artifact serves + this check and ``tools/score_uk_national_candidate.py``, and incidental + formatting cannot make two identical registers look different. + """ + + if path is None: + return + try: + frozen = TargetRegistry.from_json(path) + except ValueError as error: + raise SystemExit( + f"error: frozen scoring register is unusable: {error}" + ) from error + if frozen.version != registry.version: + raise SystemExit( + "re-derived register differs from the frozen scoring register: " + f"{registry.version} vs {frozen.version}" + ) + + +def _ledger_facts_pin(artifact: Any) -> dict[str, object]: + facts_path = ( + artifact.path / "consumer_facts.jsonl" if artifact.path.is_dir() else artifact.path + ) + return {"sha256": artifact.facts_sha256, "size_bytes": facts_path.stat().st_size} + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/generate_uk_target_references.py b/tools/generate_uk_target_references.py index 84b91fe0e3..1de3b4a7db 100644 --- a/tools/generate_uk_target_references.py +++ b/tools/generate_uk_target_references.py @@ -34,6 +34,7 @@ { "affected_flag_variable", "band", + "band_filter_dimension", "band_period_factor", "count_of", "filters", @@ -54,6 +55,7 @@ "source_lines", "threshold_price_base_year", "value_expression", + "value_reduction", "value_variable", "zeroed_input", } diff --git a/tools/score_uk_national_candidate.py b/tools/score_uk_national_candidate.py new file mode 100644 index 0000000000..451b4af233 --- /dev/null +++ b/tools/score_uk_national_candidate.py @@ -0,0 +1,333 @@ +"""Score a UK national candidate against the incumbent on one frozen register.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from microcosm.build.uk_runtime.national_calibration import prepare_uk_target_frame +from microcosm.build.uk_runtime.national_frame import load_uk_national_frame +from microcosm.calibrate import TargetRegistry, score_targets + +UK_SCORE_LOSS_CAP = 10.0 +UK_SCORE_HOLDOUT_BASIS = "none_declared" + + +def _holdout_loss(basis: str) -> float | None: + """The holdout loss for a declared basis, or None when none exists. + + June's frozen fixture carries a genuinely different holdout value + (0.1239 against a 0.0159 train loss) because it held rows out. This + register declares no split, so the holdout keys report absence rather + than the fitted loss wearing a holdout name — a copied value reads + downstream as perfect generalization from a measurement never made. + Declaring a basis without computing its split fails loudly here rather + than falling back to the fitted loss. + """ + + if basis == "none_declared": + return None + raise NotImplementedError( + f"holdout basis {basis!r} declares a split this scorer does not compute." + ) + + +def score_uk_national_candidate( + *, + candidate_h5: str | Path, + incumbent_h5: str | Path, + candidate_sha256: str, + incumbent_sha256: str, + target_registry: TargetRegistry, + calibration_year: int, + measure_resolver_factory: Callable[[Path, Any], Any] | None = None, + candidate_label: str = "populace_uk_2023", + incumbent_label: str = "enhanced_frs_2024_25", +) -> dict[str, Any]: + """Return the #578 rule-1 score block on a shared target registry. + + Both artifacts are verified against their declared digests before a byte + is read, and both sides are materialized through the same route the + calibration seam uses, so the score names the artifacts it actually + measured rather than two labels supplied on the command line. + """ + + if target_registry.country != "uk" or not target_registry.specs: + raise ValueError("UK candidate scoring requires a non-empty UK registry.") + candidate_pin = _verify_artifact("candidate", candidate_h5, candidate_sha256) + incumbent_pin = _verify_artifact("incumbent", incumbent_h5, incumbent_sha256) + candidate_frame, candidate_resolution = _scored_frame( + candidate_h5, target_registry, calibration_year, measure_resolver_factory + ) + incumbent_frame, incumbent_resolution = _scored_frame( + incumbent_h5, target_registry, calibration_year, measure_resolver_factory + ) + candidate = score_targets( + candidate_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + incumbent = score_targets( + incumbent_frame, + target_registry.to_target_set(), + target_loss_cap=UK_SCORE_LOSS_CAP, + ) + candidate_errors = _relative_errors(candidate) + incumbent_errors = _relative_errors(incumbent) + names = sorted(candidate_errors) + missing = sorted(set(names) ^ set(incumbent_errors)) + if missing: + raise RuntimeError( + "candidate and incumbent scores produced different target rows: " + f"{missing[:10]}." + ) + wins = _target_wins(candidate_errors, incumbent_errors) + return { + "candidate_train_loss": float(candidate.final_loss), + "candidate_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), + "candidate_full_loss": float(candidate.final_loss), + "incumbent_train_loss": float(incumbent.final_loss), + "incumbent_holdout_loss": _holdout_loss(UK_SCORE_HOLDOUT_BASIS), + "incumbent_full_loss": float(incumbent.final_loss), + "candidate_target_wins": wins["candidate"], + "incumbent_target_wins": wins["incumbent"], + "holdout_basis": UK_SCORE_HOLDOUT_BASIS, + "loss": { + "objective": "relative_error_loss", + "target_loss_cap": UK_SCORE_LOSS_CAP, + "train_equals_full": True, + }, + "register": { + "country": target_registry.country, + "version": target_registry.version, + "n_specs": len(target_registry), + }, + "artifacts": { + "candidate": {"label": candidate_label, **candidate_pin}, + "incumbent": {"label": incumbent_label, **incumbent_pin}, + }, + "measure_resolution": { + "candidate": candidate_resolution, + "incumbent": incumbent_resolution, + }, + "target_drift": _target_drift( + target_registry, candidate_errors, incumbent_errors + ), + "signed_asymmetries": [ + { + "id": "incumbent_own_registry", + "description": ( + "The incumbent was calibrated to its own registry; both " + "artifacts are rescored here on the supplied frozen register." + ), + }, + { + "id": "national_direct_vs_collapsed_local", + "description": ( + "Candidate national weights are a direct national solve; " + "incumbent national weights are the published national " + "surface being compared on the same frozen register." + ), + }, + ], + "target_wins_by_family": _target_wins_by_family( + target_registry, + candidate_errors, + incumbent_errors, + ), + } + + +def _relative_errors(result) -> dict[str, float]: + return { + diagnostic.name: float(diagnostic.relative_error) + for diagnostic in result.diagnostics + } + + +def _target_wins( + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, int]: + candidate = 0 + incumbent = 0 + for name, candidate_error in candidate_errors.items(): + incumbent_error = incumbent_errors[name] + if abs(candidate_error) < abs(incumbent_error): + candidate += 1 + elif abs(incumbent_error) < abs(candidate_error): + incumbent += 1 + return {"candidate": candidate, "incumbent": incumbent} + + +def _target_wins_by_family( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> dict[str, dict[str, int]]: + by_row_name = {spec.to_target().row_name: spec.family for spec in registry.specs} + result: dict[str, dict[str, int]] = {} + for name, family in by_row_name.items(): + bucket = result.setdefault( + family, + {"candidate_target_wins": 0, "incumbent_target_wins": 0, "ties": 0}, + ) + candidate_error = abs(candidate_errors[name]) + incumbent_error = abs(incumbent_errors[name]) + if candidate_error < incumbent_error: + bucket["candidate_target_wins"] += 1 + elif incumbent_error < candidate_error: + bucket["incumbent_target_wins"] += 1 + else: + bucket["ties"] += 1 + return result + + +def _load_registry(path: str | Path) -> TargetRegistry: + """Load the frozen register through its own validating loader. + + ``TargetRegistry.from_json`` checks the format revision and re-derives the + content hash, so a hand-edited or drifted register refuses here instead of + silently deciding rule 1 on a surface nobody reviewed. + """ + + return TargetRegistry.from_json(path) + + +def _sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_artifact(role: str, path: str | Path, expected: str) -> dict[str, object]: + measured = _sha256_file(path) + if measured != expected: + raise ValueError( + f"{role} artifact sha mismatch: measured {measured}, pinned {expected}" + ) + return { + "path": str(path), + "sha256": measured, + "size_bytes": Path(path).stat().st_size, + } + + +def _scored_frame( + h5_path: str | Path, + registry: TargetRegistry, + calibration_year: int, + factory: Callable[[Path, Any], Any] | None, +) -> tuple[Any, Any]: + frame, _provenance = load_uk_national_frame(h5_path) + resolver = None if factory is None else factory(Path(h5_path), frame) + return prepare_uk_target_frame( + frame, registry, period=calibration_year, measure_resolver=resolver + ) + + +def _default_measure_resolver_factory(scratch_dir: Path, year: int): + def build(h5_path: Path, frame: Any): + from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver + + return UKMeasureResolver( + simulation_source=h5_path, + scratch_dir=scratch_dir, + year=year, + frame=frame, + ) + + return build + + +def _target_drift( + registry: TargetRegistry, + candidate_errors: dict[str, float], + incumbent_errors: dict[str, float], +) -> list[dict[str, object]]: + """Per-target relative errors, the auditable half of the score block.""" + + families = {spec.to_target().row_name: spec.family for spec in registry.specs} + rows = [] + for name in sorted(candidate_errors): + candidate = candidate_errors[name] + incumbent = incumbent_errors[name] + # Relative errors are signed; the win is decided on magnitude, the + # same rule the aggregate counts use. + rows.append( + { + "target": name, + "family": families.get(name), + "candidate_relative_error": candidate, + "incumbent_relative_error": incumbent, + "winner": ( + "candidate" + if abs(candidate) < abs(incumbent) + else "incumbent" + if abs(incumbent) < abs(candidate) + else "tie" + ), + } + ) + return rows + + +def _write_json(path: str | Path, payload: dict[str, Any]) -> Path: + output = Path(path) + output.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + return output + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-h5", required=True, type=Path) + parser.add_argument("--candidate-sha256", required=True) + parser.add_argument("--incumbent-h5", required=True, type=Path) + parser.add_argument("--incumbent-sha256", required=True) + parser.add_argument("--registry-json", required=True, type=Path) + parser.add_argument("--output-json", required=True, type=Path) + parser.add_argument("--calibration-year", type=int) + parser.add_argument( + "--no-measure-resolution", + action="store_true", + help=( + "Score frames whose measures are already columns. Every packaged " + "UK reference binds a prepared measure, so this is for fixtures " + "only; a production register refuses on the first skipped target." + ), + ) + args = parser.parse_args(argv) + registry = _load_registry(args.registry_json) + calibration_year = args.calibration_year + if calibration_year is None: + from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + + calibration_year = load_uk_frs_release().calibration_year + factory = ( + None + if args.no_measure_resolution + else _default_measure_resolver_factory( + args.output_json.parent, int(calibration_year) + ) + ) + score = score_uk_national_candidate( + candidate_h5=args.candidate_h5, + incumbent_h5=args.incumbent_h5, + candidate_sha256=args.candidate_sha256, + incumbent_sha256=args.incumbent_sha256, + target_registry=registry, + calibration_year=int(calibration_year), + measure_resolver_factory=factory, + ) + _write_json(args.output_json, {"score_vs_enhanced_frs": score}) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())